2024-05-20T19:38:35.775 INFO:root:teuthology version: 0.0.1.dev276+g4fbe5e8 2024-05-20T19:38:35.775 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_leonidus@teuthology --archive /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929 --description fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mgr mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/no} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts pg_health session_timeout} ranks/multi/{balancer/random export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/yes 5-quiesce/with-quiesce 6-workunit/suites/iogen}} --name leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi --verbose -- /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/orig.config.yaml 2024-05-20T19:38:35.855 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-20T19:38:35.959 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929 branch: wip-lusov-quiesce description: fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mgr mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/no} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts pg_health session_timeout} ranks/multi/{balancer/random export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/yes 5-quiesce/with-quiesce 6-workunit/suites/iogen}} email: null first_in_suite: false job_id: '7715929' kernel: client: branch: testing kdb: true sha1: distro ktype: upstream last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi mgrmodules: sequential: - print: Enabling mgr modules - exec: mon.a: - ceph mgr module enable snap_schedule name: leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-lusov-quiesce ceph: cephfs: max_mds: 5 session_timeout: 300 standby_replay: true conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: ms die on skipped message: false mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds bal fragment size max: 10000 mds bal merge size: 5 mds bal split bits: 3 mds bal split size: 100 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true mds_export_ephemeral_random_max: 0.1 osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: cephsqlite lock renewal timeout: 900000 debug cephsqlite: 20 debug client: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 mon op complaint time: 120 osd: bluestore block size: 96636764160 bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug ms: 1 debug osd: 20 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 osd shutdown pgref assert: true osd_max_omap_entries_per_request: 10000 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - MDS_UP_LESS_THAN_MAX - filesystem is online with fewer MDS than max_mds - POOL_APP_NOT_ENABLED - do not have an application enabled - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - PG_AVAILABILITY - PG_DEGRADED - Reduced data availability - Degraded data redundancy - slow metadata IO - SLOW_OPS - slow request sha1: 4ebd1d5361d457738957dc0455bba90102296634 subvols: create: 2 subvol_options: --namespace-isolated --size 25000000000 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} ceph-fuse: client.0: mount_subvol_num: 0 check-counter: counters: mds: - mds.exported - mds.imported - mds.root_rsnaps - mds_server.req_mksnap_latency.avgcount - mds_server.req_rmsnap_latency.avgcount dry_run: true install: ceph: flavor: default sha1: 4ebd1d5361d457738957dc0455bba90102296634 kclient: client.0: mount_subvol_num: 1 mntopts: - ms_mode=secure - nowsync syntax: v1 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-lusov-quiesce sha1: 4ebd1d5361d457738957dc0455bba90102296634 owner: scheduled_leonidus@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.x - mds.a - mds.d - mds.g - mds.j - osd.0 - osd.3 - osd.6 - osd.9 - client.0 - - mon.b - mgr.y - mds.b - mds.e - mds.h - mds.k - osd.1 - osd.4 - osd.7 - osd.10 - - mon.c - mgr.z - mds.c - mds.f - mds.i - mds.l - osd.2 - osd.5 - osd.8 - osd.11 seed: 2891 sha1: 4ebd1d5361d457738957dc0455bba90102296634 sleep_before_teardown: 0 subset: 194/111111 suite: fs suite_branch: wip-lusov-quiesce suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_4ebd1d5361d457738957dc0455bba90102296634/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 4ebd1d5361d457738957dc0455bba90102296634 syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi028.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOPcS5m7utnbwfCmhOQUl1ZO33QU0VUb0Pja9sjo9Zt8zQFHTDEEy0eUJY6DyR+DEWWpPpZsdqpSeCoU+NZNzms= smithi033.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKOkX6y/wiTw/Ano6Op8C2NnTRtdRxrZz+Oz/FAvGSFEcxxcIm1ui48ii8H2ywfAdnCOoSfsoWsgY8TKrCNG+oA= smithi154.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5QxxiJLnGZ+H2p9eE3Uf+UXnXwFzBRVOlJlTX5XW0EFjMek6EcTMZojwu1vs8VT215EgPGzplqGTX9I6Wg9w0= tasks: - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - cephadm: roleless: false - cephadm.shell: mon.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: mon.a: - ceph fs dump - ceph osd dump - fs.ready: timeout: 300 - sequential: - mgrmodules - kclient: null - exec: mon.a: - ceph fs set cephfs balance_automate false - ceph fs subvolumegroup pin cephfs qa random 0.10 - check-counter: {} - fwd_scrub: scrub_timeout: 900 sleep_between_iterations: 1 - exec: mon.a: - ceph config set mgr mgr/snap_schedule/log_level debug - ceph config set mgr mgr/snap_schedule/allow_m_granularity true - ceph config set mgr mgr/snap_schedule/dump_on_update true - ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1m - ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6m3h - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - date +%s > START_TIME - full_sequential_finally: - exec: mon.a: - date +%s > END_TIME - START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - background_exec: mon.a: - while sleep 13; do ceph tell mds.cephfs:0 flush journal; done - quiescer: null - workunit: clients: all: - suites/iogen.sh teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return yaml.ktype == 'upstream' end - local function is_kdistro() return yaml.ktype == 'distro' end - local function is_fuse() return false end - local function syntax_version() return yaml.overrides.kclient.syntax end teuthology_branch: install-fix teuthology_sha1: 4fbe5e8dc899215f216853b8569a107d8be8861d timestamp: 2024-05-20_18:41:05 tube: smithi user: leonidus verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 2024-05-20T19:38:35.960 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_4ebd1d5361d457738957dc0455bba90102296634/qa; will attempt to use it 2024-05-20T19:38:35.960 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_4ebd1d5361d457738957dc0455bba90102296634/qa/tasks 2024-05-20T19:38:35.961 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-20T19:38:35.962 INFO:teuthology.task.internal:Checking packages... 2024-05-20T19:38:35.986 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '4ebd1d5361d457738957dc0455bba90102296634' 2024-05-20T19:38:35.987 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-20T19:38:35.987 INFO:teuthology.packaging:ref: None 2024-05-20T19:38:35.987 INFO:teuthology.packaging:tag: None 2024-05-20T19:38:35.987 INFO:teuthology.packaging:branch: wip-lusov-quiesce 2024-05-20T19:38:35.987 INFO:teuthology.packaging:sha1: 4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:38:35.987 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-lusov-quiesce 2024-05-20T19:38:36.157 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3830.g4ebd1d53 2024-05-20T19:38:36.160 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-20T19:38:36.166 INFO:teuthology.task.internal:no buildpackages task found 2024-05-20T19:38:36.166 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-20T19:38:36.173 INFO:teuthology.task.internal:Saving configuration 2024-05-20T19:38:36.196 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-20T19:38:36.202 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-20T19:38:36.227 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi028.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-20 19:33:35.956510', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOPcS5m7utnbwfCmhOQUl1ZO33QU0VUb0Pja9sjo9Zt8zQFHTDEEy0eUJY6DyR+DEWWpPpZsdqpSeCoU+NZNzms='} 2024-05-20T19:38:36.251 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi033.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-20 19:33:35.954509', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKOkX6y/wiTw/Ano6Op8C2NnTRtdRxrZz+Oz/FAvGSFEcxxcIm1ui48ii8H2ywfAdnCOoSfsoWsgY8TKrCNG+oA='} 2024-05-20T19:38:36.276 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi154.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-20 19:33:35.958251', 'locked_by': 'scheduled_leonidus@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5QxxiJLnGZ+H2p9eE3Uf+UXnXwFzBRVOlJlTX5XW0EFjMek6EcTMZojwu1vs8VT215EgPGzplqGTX9I6Wg9w0='} 2024-05-20T19:38:36.276 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-20T19:38:36.283 INFO:teuthology.task.internal:roles: ubuntu@smithi028.front.sepia.ceph.com - ['mon.a', 'mgr.x', 'mds.a', 'mds.d', 'mds.g', 'mds.j', 'osd.0', 'osd.3', 'osd.6', 'osd.9', 'client.0'] 2024-05-20T19:38:36.283 INFO:teuthology.task.internal:roles: ubuntu@smithi033.front.sepia.ceph.com - ['mon.b', 'mgr.y', 'mds.b', 'mds.e', 'mds.h', 'mds.k', 'osd.1', 'osd.4', 'osd.7', 'osd.10'] 2024-05-20T19:38:36.283 INFO:teuthology.task.internal:roles: ubuntu@smithi154.front.sepia.ceph.com - ['mon.c', 'mgr.z', 'mds.c', 'mds.f', 'mds.i', 'mds.l', 'osd.2', 'osd.5', 'osd.8', 'osd.11'] 2024-05-20T19:38:36.283 INFO:teuthology.run_tasks:Running task console_log... 2024-05-20T19:38:36.367 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7313087ac0>, signals=[15]) 2024-05-20T19:38:36.367 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-20T19:38:36.373 INFO:teuthology.task.internal:Opening connections... 2024-05-20T19:38:36.373 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T19:38:36.375 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:38:36.462 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi033.front.sepia.ceph.com 2024-05-20T19:38:36.463 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:38:36.543 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi154.front.sepia.ceph.com 2024-05-20T19:38:36.544 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:38:36.624 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-20T19:38:36.631 DEBUG:teuthology.orchestra.run.smithi028:> uname -m 2024-05-20T19:38:36.650 INFO:teuthology.orchestra.run.smithi028.stdout:x86_64 2024-05-20T19:38:36.651 DEBUG:teuthology.orchestra.run.smithi028:> cat /etc/os-release 2024-05-20T19:38:36.708 INFO:teuthology.orchestra.run.smithi028.stdout:NAME="CentOS Stream" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:VERSION="9" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:ID="centos" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:ID_LIKE="rhel fedora" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:VERSION_ID="9" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:PLATFORM_ID="platform:el9" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:ANSI_COLOR="0;31" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:LOGO="fedora-logo-icon" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:HOME_URL="https://centos.org/" 2024-05-20T19:38:36.709 INFO:teuthology.orchestra.run.smithi028.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-20T19:38:36.710 INFO:teuthology.orchestra.run.smithi028.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-20T19:38:36.710 INFO:teuthology.orchestra.run.smithi028.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-20T19:38:36.710 INFO:teuthology.lock.ops:Updating smithi028.front.sepia.ceph.com on lock server 2024-05-20T19:38:36.732 DEBUG:teuthology.orchestra.run.smithi033:> uname -m 2024-05-20T19:38:36.751 INFO:teuthology.orchestra.run.smithi033.stdout:x86_64 2024-05-20T19:38:36.751 DEBUG:teuthology.orchestra.run.smithi033:> cat /etc/os-release 2024-05-20T19:38:36.811 INFO:teuthology.orchestra.run.smithi033.stdout:NAME="CentOS Stream" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION="9" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:ID="centos" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:ID_LIKE="rhel fedora" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION_ID="9" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:PLATFORM_ID="platform:el9" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:ANSI_COLOR="0;31" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:LOGO="fedora-logo-icon" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-20T19:38:36.812 INFO:teuthology.orchestra.run.smithi033.stdout:HOME_URL="https://centos.org/" 2024-05-20T19:38:36.813 INFO:teuthology.orchestra.run.smithi033.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-20T19:38:36.813 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-20T19:38:36.813 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-20T19:38:36.813 INFO:teuthology.lock.ops:Updating smithi033.front.sepia.ceph.com on lock server 2024-05-20T19:38:36.832 DEBUG:teuthology.orchestra.run.smithi154:> uname -m 2024-05-20T19:38:36.850 INFO:teuthology.orchestra.run.smithi154.stdout:x86_64 2024-05-20T19:38:36.850 DEBUG:teuthology.orchestra.run.smithi154:> cat /etc/os-release 2024-05-20T19:38:36.907 INFO:teuthology.orchestra.run.smithi154.stdout:NAME="CentOS Stream" 2024-05-20T19:38:36.907 INFO:teuthology.orchestra.run.smithi154.stdout:VERSION="9" 2024-05-20T19:38:36.907 INFO:teuthology.orchestra.run.smithi154.stdout:ID="centos" 2024-05-20T19:38:36.907 INFO:teuthology.orchestra.run.smithi154.stdout:ID_LIKE="rhel fedora" 2024-05-20T19:38:36.907 INFO:teuthology.orchestra.run.smithi154.stdout:VERSION_ID="9" 2024-05-20T19:38:36.907 INFO:teuthology.orchestra.run.smithi154.stdout:PLATFORM_ID="platform:el9" 2024-05-20T19:38:36.907 INFO:teuthology.orchestra.run.smithi154.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-20T19:38:36.908 INFO:teuthology.orchestra.run.smithi154.stdout:ANSI_COLOR="0;31" 2024-05-20T19:38:36.908 INFO:teuthology.orchestra.run.smithi154.stdout:LOGO="fedora-logo-icon" 2024-05-20T19:38:36.908 INFO:teuthology.orchestra.run.smithi154.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-20T19:38:36.908 INFO:teuthology.orchestra.run.smithi154.stdout:HOME_URL="https://centos.org/" 2024-05-20T19:38:36.908 INFO:teuthology.orchestra.run.smithi154.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-20T19:38:36.908 INFO:teuthology.orchestra.run.smithi154.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-20T19:38:36.908 INFO:teuthology.orchestra.run.smithi154.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-20T19:38:36.908 INFO:teuthology.lock.ops:Updating smithi154.front.sepia.ceph.com on lock server 2024-05-20T19:38:36.932 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-20T19:38:36.943 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-20T19:38:36.949 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-20T19:38:36.949 DEBUG:teuthology.orchestra.run.smithi028:> test '!' -e /home/ubuntu/cephtest 2024-05-20T19:38:36.952 DEBUG:teuthology.orchestra.run.smithi033:> test '!' -e /home/ubuntu/cephtest 2024-05-20T19:38:36.954 DEBUG:teuthology.orchestra.run.smithi154:> test '!' -e /home/ubuntu/cephtest 2024-05-20T19:38:36.971 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-20T19:38:36.977 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-20T19:38:36.977 DEBUG:teuthology.orchestra.run.smithi028:> test -z $(ls -A /var/lib/ceph) 2024-05-20T19:38:37.009 DEBUG:teuthology.orchestra.run.smithi033:> test -z $(ls -A /var/lib/ceph) 2024-05-20T19:38:37.011 DEBUG:teuthology.orchestra.run.smithi154:> test -z $(ls -A /var/lib/ceph) 2024-05-20T19:38:37.043 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-20T19:38:37.125 INFO:teuthology.run_tasks:Running task kernel... 2024-05-20T19:38:37.142 INFO:teuthology.task.kernel:normalize config orig: {'client': {'branch': 'testing'}, 'kdb': True, 'sha1': 'distro'} 2024-05-20T19:38:37.142 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'branch': 'testing'}} 2024-05-20T19:38:37.142 INFO:teuthology.task.kernel:config {'client.0': {'branch': 'testing'}}, timeout 300 2024-05-20T19:38:37.143 DEBUG:teuthology.orchestra.run.smithi028:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-20T19:38:37.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T19:38:37.161 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F9%2Fx86_64&ref=testing 2024-05-20T19:38:37.339 DEBUG:teuthology.task.kernel:sha1 for client.0 is 0fe9556839d28e65b29a64cc05a26a6884e8775d 2024-05-20T19:38:37.340 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "0fe9556839d28e65b29a64cc05a26a6884e8775d"... 2024-05-20T19:38:37.340 DEBUG:teuthology.orchestra.run.smithi028:> uname -r 2024-05-20T19:38:37.358 INFO:teuthology.orchestra.run.smithi028.stdout:5.14.0-437.el9.x86_64 2024-05-20T19:38:37.358 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-437.el9.x86_64 vs 0fe9556839d28e65b29a64cc05a26a6884e8775d 2024-05-20T19:38:37.358 DEBUG:teuthology.task.kernel:failed to parse current kernel version 2024-05-20T19:38:37.359 DEBUG:teuthology.orchestra.run.smithi028:> sudo yum upgrade -y linux-firmware 2024-05-20T19:38:40.937 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - BaseOS 4.1 MB/s | 8.1 MB 00:01 2024-05-20T19:38:44.221 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - AppStream 8.8 MB/s | 19 MB 00:02 2024-05-20T19:38:50.017 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - CRB 2.9 MB/s | 6.2 MB 00:02 2024-05-20T19:38:51.479 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - Extras packages 59 kB/s | 16 kB 00:00 2024-05-20T19:38:53.766 INFO:teuthology.orchestra.run.smithi028.stdout:Extra Packages for Enterprise Linux 9.8 MB/s | 21 MB 00:02 2024-05-20T19:38:59.048 INFO:teuthology.orchestra.run.smithi028.stdout:lab-extras 43 kB/s | 1.7 kB 00:00 2024-05-20T19:39:01.024 INFO:teuthology.orchestra.run.smithi028.stdout:Dependencies resolved. 2024-05-20T19:39:01.025 INFO:teuthology.orchestra.run.smithi028.stdout:Nothing to do. 2024-05-20T19:39:01.025 INFO:teuthology.orchestra.run.smithi028.stdout:Complete! 2024-05-20T19:39:01.120 INFO:teuthology.task.kernel:Downloading kernel 0fe9556839d28e65b29a64cc05a26a6884e8775d on client.0... 2024-05-20T19:39:01.121 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F9%2Fx86_64&sha1=0fe9556839d28e65b29a64cc05a26a6884e8775d 2024-05-20T19:39:01.344 INFO:teuthology.task.kernel:fetching, builder baseurl is https://2.chacra.ceph.com/r/kernel/testing/0fe9556839d28e65b29a64cc05a26a6884e8775d/centos/9/flavors/default/x86_64/ 2024-05-20T19:39:01.344 DEBUG:teuthology.orchestra.run.smithi028:> rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.9.0_rc3_g0fe9556839d2-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://2.chacra.ceph.com/r/kernel/testing/0fe9556839d28e65b29a64cc05a26a6884e8775d/centos/9/flavors/default/x86_64/ --input-file=- 2024-05-20T19:39:01.346 DEBUG:teuthology.task.kernel:Waiting for download/copy to ubuntu@smithi028.front.sepia.ceph.com to complete... 2024-05-20T19:39:19.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20 19:39:19 URL:https://2.chacra.ceph.com/r/kernel/testing/0fe9556839d28e65b29a64cc05a26a6884e8775d/centos/9/flavors/default/x86_64/kernel-6.9.0_rc3_g0fe9556839d2-1.x86_64.rpm [357365859/357365859] -> "/tmp/kernel.x86_64.rpm" [1] 2024-05-20T19:39:19.129 INFO:teuthology.orchestra.run.smithi028.stderr:FINISHED --2024-05-20 19:39:19-- 2024-05-20T19:39:19.129 INFO:teuthology.orchestra.run.smithi028.stderr:Total wall clock time: 18s 2024-05-20T19:39:19.129 INFO:teuthology.orchestra.run.smithi028.stderr:Downloaded: 1 files, 341M in 17s (19.6 MB/s) 2024-05-20T19:39:19.132 INFO:teuthology.task.kernel:Installing kernel 0fe9556839d28e65b29a64cc05a26a6884e8775d on client.0... 2024-05-20T19:39:19.132 DEBUG:teuthology.orchestra.run.smithi028:> sudo rpm -ivh --oldpackage --replacefiles --replacepkgs /tmp/kernel.x86_64.rpm 2024-05-20T19:39:21.315 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying... ######################################## 2024-05-20T19:39:21.447 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing... ######################################## 2024-05-20T19:39:21.597 INFO:teuthology.orchestra.run.smithi028.stdout:Updating / installing... 2024-05-20T19:39:29.971 INFO:teuthology.orchestra.run.smithi028.stdout:kernel-6.9.0_rc3_g0fe9556839d2-1 ######################################## 2024-05-20T19:40:09.795 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi028.front.sepia.ceph.com, path=/tmp/kernel.x86_64.rpm, version=None) 2024-05-20T19:40:09.796 DEBUG:teuthology.orchestra.run.smithi028:> rpm -qlp /tmp/kernel.x86_64.rpm 2024-05-20T19:40:09.883 INFO:teuthology.orchestra.run.smithi028.stdout:/boot/System.map-6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/boot/config-6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/boot/vmlinuz-6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/System.map 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/config 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/blowfish-x86_64.ko 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/camellia-aesni-avx-x86_64.ko 2024-05-20T19:40:09.884 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/camellia-aesni-avx2.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/camellia-x86_64.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/cast5-avx-x86_64.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/cast6-avx-x86_64.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/crc32-pclmul.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/crc32c-intel.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/crct10dif-pclmul.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/des3_ede-x86_64.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/ghash-clmulni-intel.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/serpent-avx-x86_64.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/serpent-avx2.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/serpent-sse2-x86_64.ko 2024-05-20T19:40:09.885 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/sha1-ssse3.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/sha256-ssse3.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/sha512-ssse3.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/twofish-avx-x86_64.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/twofish-x86_64-3way.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/crypto/twofish-x86_64.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kernel 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kernel/cpu 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kernel/cpu/mce 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kernel/cpu/mce/mce-inject.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kvm 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kvm/kvm-amd.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kvm/kvm-intel.ko 2024-05-20T19:40:09.886 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/arch/x86/kvm/kvm.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/ansi_cprng.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/anubis.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/arc4.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/async_tx 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/async_tx/async_memcpy.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/async_tx/async_pq.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/async_tx/async_raid6_recov.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/async_tx/async_tx.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/async_tx/async_xor.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/async_tx/raid6test.ko 2024-05-20T19:40:09.887 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/authenc.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/authencesn.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/blake2b_generic.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/blowfish_common.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/blowfish_generic.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/camellia_generic.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/cast5_generic.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/cast6_generic.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/cast_common.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/ccm.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/cmac.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/crc32_generic.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/crypto_engine.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/crypto_user.ko 2024-05-20T19:40:09.888 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/cts.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/des_generic.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/dh_generic.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/echainiv.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/essiv.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/fcrypt.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/khazad.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/lz4.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/lz4hc.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/md4.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/michael_mic.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/pcbc.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/pcrypt.ko 2024-05-20T19:40:09.889 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/rmd160.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/seed.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/serpent_generic.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/tcrypt.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/tea.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/twofish_common.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/twofish_generic.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/vmac.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/wp512.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/xcbc.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/xor.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/crypto/xxhash_generic.ko 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers 2024-05-20T19:40:09.890 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi/acpi_ipmi.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi/acpi_pad.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi/ec_sys.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi/platform_profile.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi/sbs.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi/sbshc.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/acpi/video.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/acard-ahci.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/ahci_platform.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/ata_generic.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/libahci_platform.ko 2024-05-20T19:40:09.891 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_acpi.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_ali.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_amd.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_artop.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_atiixp.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_atp867x.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_cmd640.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_cmd64x.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_cypress.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_efar.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_hpt366.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_hpt37x.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_hpt3x2n.ko 2024-05-20T19:40:09.892 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_hpt3x3.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_it8213.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_it821x.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_jmicron.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_marvell.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_mpiix.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_netcell.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_ninja32.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_ns87410.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_ns87415.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_oldpiix.ko 2024-05-20T19:40:09.893 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_opti.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_optidma.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_pcmcia.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_pdc2027x.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_pdc202xx_old.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_piccolo.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_rdc.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_sch.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_serverworks.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_sil680.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_sis.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_sl82c105.ko 2024-05-20T19:40:09.894 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_triflex.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pata_via.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/pdc_adma.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_inic162x.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_mv.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_nv.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_promise.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_qstor.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_sil.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_sil24.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_sis.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_svw.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_sx4.ko 2024-05-20T19:40:09.895 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_uli.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_via.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ata/sata_vsc.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/auxdisplay 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/auxdisplay/ks0108.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/base 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/base/regmap 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/base/regmap/regmap-i2c.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/base/regmap/regmap-mmio.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/bcma 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/bcma/bcma.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/aoe 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/aoe/aoe.ko 2024-05-20T19:40:09.896 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/brd.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/drbd 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/drbd/drbd.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/floppy.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/loop.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/mtip32xx 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/mtip32xx/mtip32xx.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/nbd.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/null_blk 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/null_blk/null_blk.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/pktcdvd.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/rbd.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/virtio_blk.ko 2024-05-20T19:40:09.897 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/xen-blkback 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/xen-blkback/xen-blkback.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/xen-blkfront.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/zram 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/block/zram/zram.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/hangcheck-timer.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/hw_random 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/hw_random/amd-rng.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/hw_random/intel-rng.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/hw_random/timeriomem-rng.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/hw_random/via-rng.ko 2024-05-20T19:40:09.898 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/hw_random/virtio-rng.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ipmi 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ipmi/ipmi_devintf.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ipmi/ipmi_msghandler.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ipmi/ipmi_poweroff.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ipmi/ipmi_si.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ipmi/ipmi_ssif.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ipmi/ipmi_watchdog.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/lp.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/mwave 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/mwave/mwave.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/ppdev.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tlclk.ko 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tpm 2024-05-20T19:40:09.899 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tpm/tpm.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tpm/tpm_atmel.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tpm/tpm_infineon.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tpm/tpm_nsc.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tpm/tpm_tis.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/tpm/tpm_tis_core.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/uv_mmtimer.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/char/virtio_console.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/cpufreq 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/cpufreq/acpi-cpufreq.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/cpufreq/amd_freq_sensitivity.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/cpufreq/p4-clockmod.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/cpufreq/pcc-cpufreq.ko 2024-05-20T19:40:09.900 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/cpufreq/powernow-k8.ko 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/cpufreq/speedstep-lib.ko 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/ccp 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/ccp/ccp-crypto.ko 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/ccp/ccp.ko 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/intel 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/intel/qat 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/intel/qat/qat_common 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/intel/qat/qat_common/intel_qat.ko 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/intel/qat/qat_dh895xcc 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/intel/qat/qat_dh895xcc/qat_dh895xcc.ko 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/padlock-aes.ko 2024-05-20T19:40:09.901 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/padlock-sha.ko 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/virtio 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/crypto/virtio/virtio_crypto.ko 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/dca 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/dca/dca.ko 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/devfreq 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/devfreq/governor_simpleondemand.ko 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/dma 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/dma/dw 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/dma/dw/dw_dmac.ko 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/dma/ioat 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/dma/ioat/ioatdma.ko 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac 2024-05-20T19:40:09.902 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/amd64_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/e752x_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/edac_mce_amd.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/i3000_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/i3200_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/i5100_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/i5400_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/i7300_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/i7core_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/i82975x_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/ie31200_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/sb_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/edac/x38_edac.ko 2024-05-20T19:40:09.903 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/firmware 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/firmware/edd.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/firmware/iscsi_ibft.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/gpio 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/gpio/gpio-ich.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/gpio/gpio-viperboard.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-appleir.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-aureal.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-axff.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-dr.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-elecom.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-elo.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-emsff.ko 2024-05-20T19:40:09.904 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-gaff.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-gt683r.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-gyration.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-holtek-kbd.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-holtek-mouse.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-holtekff.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-hyperv.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-icade.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-keytouch.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-kye.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-lcpower.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-led.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-lenovo.ko 2024-05-20T19:40:09.905 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-logitech-dj.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-logitech-hidpp.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-multitouch.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-ortek.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-penmount.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-petalynx.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-picolcd.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-pl.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-plantronics.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-primax.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-rmi.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-arvo.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-common.ko 2024-05-20T19:40:09.906 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-isku.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-kone.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-koneplus.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-konepure.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-kovaplus.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-lua.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-pyra.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-ryos.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat-savu.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-roccat.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-saitek.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-samsung.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-sensor-hub.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-sjoy.ko 2024-05-20T19:40:09.907 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-sony.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-speedlink.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-steelseries.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-sunplus.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-thrustmaster.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-tivo.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-tmff.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-topseed.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-twinhan.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-uclogic.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-waltop.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-wiimote.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-xinmo.ko 2024-05-20T19:40:09.908 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-zpff.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/hid-zydacron.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/uhid.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hid/wacom.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hv 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hv/hv_balloon.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hv/hv_utils.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hv/hv_vmbus.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/abituguru.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/abituguru3.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/acpi_power_meter.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ad7414.ko 2024-05-20T19:40:09.909 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ad7418.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adc128d818.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adm1025.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adm1026.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adm1029.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adm1031.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adm9240.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ads7828.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adt7410.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adt7411.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adt7462.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adt7470.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adt7475.ko 2024-05-20T19:40:09.910 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/adt7x10.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/amc6821.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/applesmc.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/asb100.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/asc7621.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/asus_atk0110.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/atxp1.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/coretemp.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/dell-smm-hwmon.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/dme1737.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ds1621.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ds620.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/emc1403.ko 2024-05-20T19:40:09.911 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/emc6w201.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/f71805f.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/f71882fg.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/f75375s.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/fam15h_power.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/fschmd.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/g760a.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/g762.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/gl518sm.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/gl520sm.ko 2024-05-20T19:40:09.912 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/hwmon-vid.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/i5500_temp.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/i5k_amb.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ibmaem.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ibmpex.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ina209.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ina2xx.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/it87.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/k10temp.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/k8temp.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lineage-pem.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm63.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm73.ko 2024-05-20T19:40:09.913 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm75.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm77.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm78.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm80.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm83.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm85.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm87.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm90.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm92.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm93.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm95234.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm95241.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/lm95245.ko 2024-05-20T19:40:09.914 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ltc2945.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ltc4151.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ltc4215.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ltc4222.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ltc4245.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ltc4260.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/ltc4261.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/max16065.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/max1619.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/max1668.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/max197.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/max6639.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/max6650.ko 2024-05-20T19:40:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/max6697.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/mcp3021.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/nct6683.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/nct6775-core.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/nct6775.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/nct7802.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pc87360.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pc87427.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pcf8591.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/adm1275.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/lm25066.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/ltc2978.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/max16064.ko 2024-05-20T19:40:09.916 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/max34440.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/max8688.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/pmbus.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/pmbus_core.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/tps40422.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/ucd9000.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/ucd9200.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/pmbus/zl6100.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/powr1220.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/sch5627.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/sch5636.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/sch56xx-common.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/sht15.ko 2024-05-20T19:40:09.917 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/sht21.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/shtc1.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/sis5595.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/smsc47b397.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/smsc47m1.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/smsc47m192.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/thmc50.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/tmp102.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/tmp103.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/tmp401.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/tmp421.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/via-cputemp.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/via686a.ko 2024-05-20T19:40:09.918 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/vt1211.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/vt8231.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83627ehf.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83627hf.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83781d.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83791d.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83792d.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83793.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83795.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83l785ts.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/hwmon/w83l786ng.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/algos 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/algos/i2c-algo-bit.ko 2024-05-20T19:40:09.919 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/algos/i2c-algo-pca.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-amd756-s4882.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-amd756.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-amd8111.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-ccgx-ucsi.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-designware-core.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-designware-pci.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-designware-platform.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-diolan-u2c.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-i801.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-isch.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-ismt.ko 2024-05-20T19:40:09.920 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-nforce2-s4985.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-nforce2.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-parport.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-pca-platform.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-piix4.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-scmi.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-simtec.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-sis96x.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-tiny-usb.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-via.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-viapro.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/busses/i2c-viperboard.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/i2c-dev.ko 2024-05-20T19:40:09.921 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/i2c-mux.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/i2c-slave-eeprom.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/i2c-smbus.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/i2c/i2c-stub.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core/ib_cm.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core/ib_core.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core/ib_umad.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core/ib_uverbs.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core/iw_cm.ko 2024-05-20T19:40:09.922 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core/rdma_cm.ko 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/core/rdma_ucm.ko 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/cxgb4 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/cxgb4/iw_cxgb4.ko 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/mlx4 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/mlx4/mlx4_ib.ko 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/mlx5 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/mthca 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/hw/mthca/ib_mthca.ko 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/ipoib 2024-05-20T19:40:09.923 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/iser 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/iser/ib_iser.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/isert 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/isert/ib_isert.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/srp 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/srp/ib_srp.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/srpt 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/infiniband/ulp/srpt/ib_srpt.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/gameport 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/gameport/emu10k1-gp.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/gameport/fm801-gp.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/gameport/gameport.ko 2024-05-20T19:40:09.924 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/joydev.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/keyboard 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/keyboard/gpio_keys.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/appletouch.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/bcm5974.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/cyapatp.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/elan_i2c.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/sermouse.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/synaptics_i2c.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/synaptics_usb.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/mouse/vsxxxaa.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/rmi4 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/rmi4/rmi_core.ko 2024-05-20T19:40:09.925 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/serio 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/serio/altera_ps2.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/serio/arc_ps2.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/serio/hyperv-keyboard.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/serio/serio_raw.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/input/sparse-keymap.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/leds-blinkm.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/leds-lm3530.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/leds-lp3944.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/leds-ss4200.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-audio.ko 2024-05-20T19:40:09.926 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-backlight.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-camera.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-default-on.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-gpio.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-heartbeat.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-oneshot.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-timer.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/leds/trigger/ledtrig-transient.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/bcache 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/bcache/bcache.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-bio-prison.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-cache-smq.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-cache.ko 2024-05-20T19:40:09.927 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-crypt.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-delay.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-flakey.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-log-userspace.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-multipath.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-queue-length.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-raid.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-round-robin.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-service-time.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-switch.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-thin-pool.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/dm-verity.ko 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/persistent-data 2024-05-20T19:40:09.928 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/persistent-data/dm-persistent-data.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/raid0.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/raid1.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/raid10.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/md/raid456.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion/mptbase.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion/mptctl.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion/mptfc.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion/mptlan.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion/mptsas.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion/mptscsih.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/message/fusion/mptspi.ko 2024-05-20T19:40:09.929 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd/lpc_ich.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd/lpc_sch.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd/mfd-core.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd/sm501.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd/viperboard.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd/vx855.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/mfd/wl1273-core.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/altera-stapl 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/altera-stapl/altera-stapl.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/apds9802als.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/apds990x.ko 2024-05-20T19:40:09.930 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/bh1770glc.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/cb710 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/cb710/cb710.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/eeprom 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/eeprom/at24.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/eeprom/eeprom_93cx6.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/eeprom/max6875.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/enclosure.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/hpilo.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/isl29003.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/isl29020.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/lis3lv02d 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/lis3lv02d/lis3lv02d.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/lis3lv02d/lis3lv02d_i2c.ko 2024-05-20T19:40:09.931 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/mei 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/mei/mei-me.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/mei/mei-txe.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/mei/mei.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/sgi-gru 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/sgi-gru/gru.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/sgi-xp 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/sgi-xp/xp.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/sgi-xp/xpc.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/sgi-xp/xpnet.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/tifm_7xx1.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/tifm_core.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/tsl2550.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/vmw_balloon.ko 2024-05-20T19:40:09.932 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/vmw_vmci 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/misc/vmw_vmci/vmw_vmci.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/bonding 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/bonding/bonding.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dsa 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dsa/b53 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dsa/b53/b53_common.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dsa/bcm-sf2.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dsa/mv88e6060.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dsa/mv88e6xxx 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dsa/mv88e6xxx/mv88e6xxx.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/dummy.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/eql.ko 2024-05-20T19:40:09.933 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/3com 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/3com/3c574_cs.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/3com/3c589_cs.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/3com/3c59x.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/3com/typhoon.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/8390 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/8390/8390.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/8390/axnet_cs.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/8390/ne2k-pci.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/8390/pcnet_cs.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/adaptec 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/adaptec/starfire.ko 2024-05-20T19:40:09.934 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/agere 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/agere/et131x.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/alteon 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/alteon/acenic.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/altera 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/altera/altera_tse.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/amd 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/amd/amd8111e.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/amd/nmclan_cs.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/amd/pcnet32.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/alx 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/alx/alx.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/atl1c 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/atl1c/atl1c.ko 2024-05-20T19:40:09.935 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/atl1e 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/atl1e/atl1e.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/atlx 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/atlx/atl1.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/atheros/atlx/atl2.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/b44.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/bnx2.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/bnx2x 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/bnx2x/bnx2x.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/cnic.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/genet 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/genet/genet.ko 2024-05-20T19:40:09.936 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/broadcom/tg3.ko 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/brocade 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/brocade/bna 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/brocade/bna/bna.ko 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb/cxgb.ko 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb3 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb3/cxgb3.ko 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb4 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb4/cxgb4.ko 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb4vf 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf.ko 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/libcxgb 2024-05-20T19:40:09.937 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/chelsio/libcxgb/libcxgb.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/cisco 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/cisco/enic 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/cisco/enic/enic.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec/tulip 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec/tulip/de2104x.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec/tulip/dmfe.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec/tulip/tulip.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec/tulip/uli526x.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec/tulip/winbond-840.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dec/tulip/xircom_cb.ko 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dlink 2024-05-20T19:40:09.938 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dlink/dl2k.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dlink/sundance.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/dnet.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/emulex 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/emulex/benet 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/emulex/benet/be2net.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/ethoc.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/fealnx.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/e100.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/e1000 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/e1000/e1000.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/e1000e 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/e1000e/e1000e.ko 2024-05-20T19:40:09.939 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/fm10k 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/fm10k/fm10k.ko 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/i40e 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/i40e/i40e.ko 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/iavf 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/iavf/iavf.ko 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/igb 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/igb/igb.ko 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/igbvf 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/igbvf/igbvf.ko 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/ixgbe 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/ixgbe/ixgbe.ko 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/ixgbevf 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/intel/ixgbevf/ixgbevf.ko 2024-05-20T19:40:09.940 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/jme.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/marvell 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/marvell/mvmdio.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/marvell/skge.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/marvell/sky2.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/mellanox 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/mellanox/mlx4 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_core.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_en.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/mellanox/mlx5 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/mellanox/mlx5/core 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/micrel 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/micrel/ksz884x.ko 2024-05-20T19:40:09.941 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/myricom 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/myricom/myri10ge 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/myricom/myri10ge/myri10ge.ko 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/natsemi 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/natsemi/natsemi.ko 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/natsemi/ns83820.ko 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/neterion 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/neterion/s2io.ko 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/nvidia 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/nvidia/forcedeth.ko 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/packetengines 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/packetengines/hamachi.ko 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/packetengines/yellowfin.ko 2024-05-20T19:40:09.942 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/qlogic 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/qlogic/netxen 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/qlogic/netxen/netxen_nic.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/qlogic/qla3xxx.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/qlogic/qlcnic 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/qlogic/qlcnic/qlcnic.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/rdc 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/rdc/r6040.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/realtek 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/realtek/8139cp.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/realtek/8139too.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/realtek/atp.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/realtek/r8169.ko 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/rocker 2024-05-20T19:40:09.943 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/rocker/rocker.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sfc 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sfc/sfc.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/silan 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/silan/sc92031.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sis 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sis/sis190.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sis/sis900.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/smsc 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/smsc/epic100.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/smsc/smc91c92_cs.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/smsc/smsc911x.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/smsc/smsc9420.ko 2024-05-20T19:40:09.944 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/stmicro 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/stmicro/stmmac 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/stmicro/stmmac/stmmac.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sun 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sun/cassini.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sun/niu.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sun/sungem.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/sun/sunhme.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/tehuti 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/tehuti/tehuti.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/ti 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/ti/tlan.ko 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/via 2024-05-20T19:40:09.945 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/via/via-rhine.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/via/via-velocity.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/wiznet 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/wiznet/w5100.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/wiznet/w5300.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/xircom 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ethernet/xircom/xirc2ps_cs.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/geneve.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/hyperv 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/hyperv/hv_netvsc.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ieee802154 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ieee802154/fakelb.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ifb.ko 2024-05-20T19:40:09.946 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ipvlan 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ipvlan/ipvlan.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/macvlan.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/macvtap.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/mdio 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/mdio.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/mdio/mdio-bcm-unimac.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/mdio/mdio-bitbang.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/mdio/mdio-regmap.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/mii.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/net_failover.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/netconsole.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/nlmon.ko 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/pcs 2024-05-20T19:40:09.947 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/pcs/pcs-lynx.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/pcs/pcs_xpcs.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/amd.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/bcm-phy-lib.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/bcm-phy-ptp.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/bcm7xxx.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/bcm87xx.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/broadcom.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/cicada.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/davicom.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/dp83640.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/et1011c.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/icplus.ko 2024-05-20T19:40:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/lxt.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/marvell.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/micrel.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/national.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/phylink.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/qsemi.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/realtek.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/smsc.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/ste10Xp.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/phy/vitesse.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/bsd_comp.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/ppp_async.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/ppp_deflate.ko 2024-05-20T19:40:09.949 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/ppp_generic.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/ppp_mppe.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/ppp_synctty.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/pppoe.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/pppox.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/ppp/pptp.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/slip 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/slip/slhc.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/slip/slip.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/sungem_phy.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/tap.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/team 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/team/team.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/team/team_mode_activebackup.ko 2024-05-20T19:40:09.950 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/team/team_mode_broadcast.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/team/team_mode_loadbalance.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/team/team_mode_random.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/team/team_mode_roundrobin.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/tun.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/veth.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/virtio_net.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/vmxnet3 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/vmxnet3/vmxnet3.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/vxlan 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/vxlan/vxlan.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/xen-netback 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/xen-netback/xen-netback.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/net/xen-netfront.ko 2024-05-20T19:40:09.951 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ntb 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ntb/ntb.ko 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/nvme 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/nvme/host 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/nvme/host/nvme-core.ko 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/nvme/host/nvme.ko 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/parport 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/parport/parport.ko 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/parport/parport_cs.ko 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/parport/parport_pc.ko 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/parport/parport_serial.ko 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pci 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pci/hotplug 2024-05-20T19:40:09.952 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pci/hotplug/acpiphp_ibm.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pci/pcie 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pci/pcie/aer_inject.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pci/xen-pcifront.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pcmcia 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pcmcia/i82092.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pcmcia/pd6729.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pcmcia/yenta_socket.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pinctrl 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pinctrl/intel 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pinctrl/intel/pinctrl-cherryview.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/acer-wmi.ko 2024-05-20T19:40:09.953 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/acerhdf.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/apple-gmux.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/asus-laptop.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/asus-nb-wmi.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/asus-wmi.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/classmate-laptop.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/eeepc-laptop.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/eeepc-wmi.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/fujitsu-laptop.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/fujitsu-tablet.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/hdaps.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/intel 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/intel/intel-rst.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/intel_ips.ko 2024-05-20T19:40:09.954 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/msi-wmi.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/mxm-wmi.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/panasonic-laptop.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/samsung-laptop.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/samsung-q10.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/topstar-laptop.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/toshiba_bluetooth.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/toshiba_haps.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/wmi-bmof.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/platform/x86/wmi.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/powercap 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/powercap/intel_rapl_common.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/powercap/intel_rapl_msr.ko 2024-05-20T19:40:09.955 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pps 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pps/clients 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pps/clients/pps-gpio.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pps/clients/pps-ldisc.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pps/clients/pps_parport.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/pps/pps_core.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ptp 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ptp/ptp.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ptp/ptp_kvm.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ras 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ras/amd 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ras/amd/atl 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ras/amd/atl/amd_atl.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ras/amd/fmpm.ko 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc 2024-05-20T19:40:09.956 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-bq32k.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds1286.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds1307.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds1374.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds1511.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds1553.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds1672.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds1742.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds2404.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-ds3232.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-em3027.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-fm3130.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-isl12022.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-isl1208.ko 2024-05-20T19:40:09.957 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-m41t80.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-m48t35.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-m48t59.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-max6900.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-msm6242.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-pcf2127.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-pcf85063.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-pcf8523.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-pcf8563.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-pcf8583.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-rp5c01.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-rs5c372.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-rv3029c2.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-rx8025.ko 2024-05-20T19:40:09.958 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-rx8581.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-stk17ta8.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/rtc/rtc-x1205.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/3w-9xxx.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/3w-sas.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/3w-xxxx.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/BusLogic.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/a100u2w.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/aacraid 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/aacraid/aacraid.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/advansys.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/aic7xxx 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/aic7xxx/aic79xx.ko 2024-05-20T19:40:09.959 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/aic7xxx/aic7xxx.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/am53c974.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/arcmsr 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/arcmsr/arcmsr.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/atp870u.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/be2iscsi 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/be2iscsi/be2iscsi.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/bfa 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/bfa/bfa.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/bnx2fc 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/bnx2fc/bnx2fc.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/bnx2i 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/bnx2i/bnx2i.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/ch.ko 2024-05-20T19:40:09.960 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/csiostor 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/csiostor/csiostor.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/cxgbi 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/cxgbi/cxgb3i 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/cxgbi/cxgb3i/cxgb3i.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/cxgbi/cxgb4i 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/cxgbi/cxgb4i/cxgb4i.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/cxgbi/libcxgbi.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/dc395x.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/device_handler 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/device_handler/scsi_dh_alua.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/device_handler/scsi_dh_emc.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/device_handler/scsi_dh_hp_sw.ko 2024-05-20T19:40:09.961 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/device_handler/scsi_dh_rdac.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/dmx3191d.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/esas2r 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/esas2r/esas2r.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/esp_scsi.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/fcoe 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/fcoe/fcoe.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/fcoe/libfcoe.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/fnic 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/fnic/fnic.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/hpsa.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/hptiop.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/hv_storvsc.ko 2024-05-20T19:40:09.962 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/initio.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/ipr.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/ips.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/isci 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/isci/isci.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/iscsi_boot_sysfs.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/iscsi_tcp.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/libfc 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/libfc/libfc.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/libiscsi.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/libiscsi_tcp.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/libsas 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/libsas/libsas.ko 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/lpfc 2024-05-20T19:40:09.963 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/lpfc/lpfc.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/megaraid 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/megaraid.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/megaraid/megaraid_mbox.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/megaraid/megaraid_mm.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/megaraid/megaraid_sas.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/mpt3sas 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/mpt3sas/mpt3sas.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/mvsas 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/mvsas/mvsas.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/mvumi.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/pm8001 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/pm8001/pm80xx.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/pmcraid.ko 2024-05-20T19:40:09.964 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/qla1280.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/qla2xxx 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/qla2xxx/qla2xxx.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/qla2xxx/tcm_qla2xxx.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/qla4xxx 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/qla4xxx/qla4xxx.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/raid_class.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/scsi_debug.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/scsi_transport_fc.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/scsi_transport_iscsi.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/scsi_transport_sas.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/scsi_transport_spi.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/scsi_transport_srp.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/ses.ko 2024-05-20T19:40:09.965 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/st.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/stex.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/sym53c8xx_2 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/sym53c8xx_2/sym53c8xx.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/virtio_scsi.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/vmw_pvscsi.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/scsi/wd719x.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ssb 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ssb/ssb.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/iscsi 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/iscsi/iscsi_target_mod.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/loopback 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/loopback/tcm_loop.ko 2024-05-20T19:40:09.966 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/target_core_file.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/target_core_iblock.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/target_core_mod.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/target_core_pscsi.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/target_core_user.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/tcm_fc 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/target/tcm_fc/tcm_fc.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/acpi_thermal_rel.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/int3400_thermal.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/int3401_thermal.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/int3402_thermal.ko 2024-05-20T19:40:09.967 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/int3403_thermal.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_power_floor.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_wt_hint.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_wt_req.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/intel_soc_dts_iosf.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/intel_soc_dts_thermal.ko 2024-05-20T19:40:09.968 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/thermal/intel/x86_pkg_temp_thermal.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/ipwireless 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/ipwireless/ipwireless.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/n_gsm.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/n_hdlc.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/nozomi.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/serial 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/serial/8250 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/serial/8250/serial_cs.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/serial/arc_uart.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/serial/jsm 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/tty/serial/jsm/jsm.ko 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ufs 2024-05-20T19:40:09.969 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ufs/core 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ufs/core/ufshcd-core.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ufs/host 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/ufs/host/ufshcd-pci.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/uio 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/uio/uio.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/uio/uio_aec.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/uio/uio_cif.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/uio/uio_pci_generic.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/uio/uio_sercos3.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/class 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/class/cdc-acm.ko 2024-05-20T19:40:09.970 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/class/cdc-wdm.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/class/usblp.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/class/usbtmc.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/host 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/host/sl811-hcd.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/image 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/image/mdc800.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/image/microtek.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/adutux.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/appledisplay.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/emi26.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/emi62.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/ezusb.ko 2024-05-20T19:40:09.971 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/idmouse.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/iowarrior.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/isight_firmware.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/ldusb.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/legousbtower.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/sisusbvga 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/sisusbvga/sisusbvga.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/trancevibrator.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/usb3503.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/usblcd.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/usbsevseg.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/uss720.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/misc/yurex.ko 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/phy 2024-05-20T19:40:09.972 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/phy/phy-generic.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/aircable.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ark3116.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/belkin_sa.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ch341.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/cp210x.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/cyberjack.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/cypress_m8.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/digi_acceleport.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/empeg.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ftdi_sio.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/garmin_gps.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/io_edgeport.ko 2024-05-20T19:40:09.973 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/io_ti.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ipaq.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ipw.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ir-usb.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/iuu_phoenix.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/keyspan.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/keyspan_pda.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/kl5kusb105.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/kobil_sct.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/mct_u232.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/mos7720.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/mos7840.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/navman.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/omninet.ko 2024-05-20T19:40:09.974 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/opticon.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/option.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/oti6858.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/pl2303.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/qcaux.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/qcserial.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/quatech2.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/safe_serial.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/sierra.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/spcp8x5.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ssu100.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/symbolserial.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/ti_usb_3410_5052.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/usb-serial-simple.ko 2024-05-20T19:40:09.975 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/usb_debug.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/usb_wwan.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/visor.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/whiteheat.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/serial/xsens_mt.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/uas.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-alauda.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-cypress.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-datafab.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-eneub6250.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-freecom.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-isd200.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-jumpshot.ko 2024-05-20T19:40:09.976 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-karma.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-onetouch.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-realtek.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-sddr09.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-sddr55.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/ums-usbat.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/storage/usb-storage.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/usbip 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/usbip/usbip-core.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/usbip/usbip-host.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/usb/usbip/vhci-hcd.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vfio 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vfio/pci 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vfio/pci/vfio-pci-core.ko 2024-05-20T19:40:09.977 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vfio/pci/vfio-pci.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vfio/vfio.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vfio/vfio_iommu_type1.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vhost 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vhost/vhost.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vhost/vhost_iotlb.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vhost/vhost_net.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/vhost/vhost_scsi.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/video 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/video/backlight 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/video/backlight/apple_bl.ko 2024-05-20T19:40:09.978 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/video/backlight/lcd.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/video/backlight/platform_lcd.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio/virtio.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio/virtio_balloon.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio/virtio_mmio.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio/virtio_pci.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio/virtio_pci_legacy_dev.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio/virtio_pci_modern_dev.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/virtio/virtio_ring.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/masters 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/masters/ds2482.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/masters/ds2490.ko 2024-05-20T19:40:09.979 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2406.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2408.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2413.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2423.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2431.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2433.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2780.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds2781.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_ds28e04.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_smem.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/slaves/w1_therm.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/w1/wire.ko 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog 2024-05-20T19:40:09.980 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/alim1535_wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/alim7101_wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/f71808e_wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/hpwdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/i6300esb.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/iTCO_vendor_support.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/iTCO_wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/ib700wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/ibmasr.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/ie6xx_wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/it8712f_wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/it87_wdt.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/machzwd.ko 2024-05-20T19:40:09.981 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/nv_tco.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/pcwd_pci.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/pcwd_usb.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/sbc_fitpc2_wdt.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/sch311x_wdt.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/softdog.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/sp5100_tco.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/via_wdt.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/w83627hf_wdt.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/w83877f_wdt.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/w83977f_wdt.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/wdt_pci.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/watchdog/xen_wdt.ko 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen 2024-05-20T19:40:09.982 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xen-acpi-processor.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xen-evtchn.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xen-gntalloc.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xen-gntdev.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xen-pciback 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xen-pciback/xen-pciback.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xen-privcmd.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xenfs 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/drivers/xen/xenfs/xenfs.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/9p 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/9p/9p.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/affs 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/affs/affs.ko 2024-05-20T19:40:09.983 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/befs 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/befs/befs.ko 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/binfmt_misc.ko 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/btrfs 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/btrfs/btrfs.ko 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/cachefiles 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/cachefiles/cachefiles.ko 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ceph 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ceph/ceph.ko 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/coda 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/coda/coda.ko 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/cramfs 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/cramfs/cramfs.ko 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/dlm 2024-05-20T19:40:09.984 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/dlm/dlm.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ecryptfs 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ecryptfs/ecryptfs.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/f2fs 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/f2fs/f2fs.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/fat 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/fat/fat.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/fat/msdos.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/fat/vfat.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/fuse 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/fuse/cuse.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/fuse/fuse.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/gfs2 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/gfs2/gfs2.ko 2024-05-20T19:40:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/hfs 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/hfs/hfs.ko 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/hfsplus 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/hfsplus/hfsplus.ko 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/isofs 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/isofs/isofs.ko 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/jfs 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/jfs/jfs.ko 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/lockd 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/lockd/lockd.ko 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/minix 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/minix/minix.ko 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/netfs 2024-05-20T19:40:09.986 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/netfs/netfs.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/blocklayout 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/blocklayout/blocklayoutdriver.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/filelayout 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/filelayout/nfs_layout_nfsv41_files.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/flexfilelayout 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/flexfilelayout/nfs_layout_flexfiles.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/nfs.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/nfsv3.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs/nfsv4.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs_common 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs_common/grace.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfs_common/nfs_acl.ko 2024-05-20T19:40:09.987 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfsd 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nfsd/nfsd.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nilfs2 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nilfs2/nilfs2.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-celtic.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-centeuro.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-croatian.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-cyrillic.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-gaelic.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-greek.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-iceland.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-inuit.ko 2024-05-20T19:40:09.988 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-roman.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-romanian.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/mac-turkish.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp1250.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp1251.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp1255.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp737.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp775.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp850.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp852.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp855.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp857.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp860.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp861.ko 2024-05-20T19:40:09.989 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp862.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp863.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp864.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp865.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp866.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp869.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp874.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp932.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp936.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp949.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_cp950.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_euc-jp.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-1.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-13.ko 2024-05-20T19:40:09.990 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-14.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-15.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-2.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-3.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-4.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-5.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-6.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-7.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_iso8859-9.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_koi8-r.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_koi8-ru.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_koi8-u.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_ucs2_utils.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/nls/nls_utf8.ko 2024-05-20T19:40:09.991 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/cluster 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/cluster/ocfs2_nodemanager.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/dlm 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/dlm/ocfs2_dlm.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/dlmfs 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/dlmfs/ocfs2_dlmfs.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/ocfs2.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/ocfs2_stack_o2cb.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/ocfs2_stack_user.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ocfs2/ocfs2_stackglue.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/overlayfs 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/overlayfs/overlay.ko 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/pstore 2024-05-20T19:40:09.992 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/pstore/ramoops.ko 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/reiserfs 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/reiserfs/reiserfs.ko 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/romfs 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/romfs/romfs.ko 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/smb 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/smb/client 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/smb/client/cifs.ko 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/smb/common 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/smb/common/cifs_arc4.ko 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/smb/common/cifs_md4.ko 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/squashfs 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/squashfs/squashfs.ko 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/sysv 2024-05-20T19:40:09.993 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/sysv/sysv.ko 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/udf 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/udf/udf.ko 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ufs 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/ufs/ufs.ko 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/xfs 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/fs/xfs/xfs.ko 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/kernel 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/kernel/rcu 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/kernel/rcu/rcutorture.ko 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/kernel/torture.ko 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/kernel/trace 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/kernel/trace/ring_buffer_benchmark.ko 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib 2024-05-20T19:40:09.994 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/asn1_encoder.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/crc-itu-t.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/crc8.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/crypto 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/crypto/libarc4.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/crypto/libdes.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/libcrc32c.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/lru_cache.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/lz4 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/lz4/lz4_compress.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/lz4/lz4hc_compress.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/math 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/math/cordic.ko 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/raid6 2024-05-20T19:40:09.995 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/raid6/raid6_pq.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/reed_solomon 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/reed_solomon/reed_solomon.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/ts_bm.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/ts_fsm.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/ts_kmp.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/zstd 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/lib/zstd/zstd_compress.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/mm 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/mm/hwpoison-inject.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/6lowpan.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/nhc_dest.ko 2024-05-20T19:40:09.996 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/nhc_fragment.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/nhc_hop.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/nhc_ipv6.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/nhc_mobility.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/nhc_routing.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/6lowpan/nhc_udp.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/802 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/802/garp.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/802/mrp.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/802/p8022.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/802/psnap.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/802/stp.ko 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/8021q 2024-05-20T19:40:09.997 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/8021q/8021q.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/9p 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/9p/9pnet.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/9p/9pnet_fd.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/9p/9pnet_rdma.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/9p/9pnet_virtio.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/batman-adv 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/batman-adv/batman-adv.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/br_netfilter.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/bridge.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_802_3.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_among.ko 2024-05-20T19:40:09.998 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_arp.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_arpreply.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_dnat.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_ip.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_ip6.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_limit.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_log.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_mark.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_mark_m.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_nflog.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_pkttype.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_redirect.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_snat.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_stp.ko 2024-05-20T19:40:09.999 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebt_vlan.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebtable_broute.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebtable_filter.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebtable_nat.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/ebtables.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/nf_conntrack_bridge.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/nft_meta_bridge.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/bridge/netfilter/nft_reject_bridge.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ceph 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ceph/libceph.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/core 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/core/failover.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/core/pktgen.ko 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dccp 2024-05-20T19:40:10.000 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dccp/dccp.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dccp/dccp_diag.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dccp/dccp_ipv4.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dccp/dccp_ipv6.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dns_resolver 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dns_resolver/dns_resolver.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dsa 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dsa/dsa_core.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dsa/tag_brcm.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dsa/tag_dsa.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dsa/tag_none.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/dsa/tag_trailer.ko 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ieee802154 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ieee802154/6lowpan 2024-05-20T19:40:10.001 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ieee802154/6lowpan/ieee802154_6lowpan.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ieee802154/ieee802154.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ieee802154/ieee802154_socket.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/ah4.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/esp4.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/fou.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/gre.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/inet_diag.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/ip_gre.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/ip_tunnel.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/ip_vti.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/ipcomp.ko 2024-05-20T19:40:10.002 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/ipip.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/arp_tables.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/arpt_mangle.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/arptable_filter.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/ipt_ECN.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/ipt_SYNPROXY.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/ipt_ah.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/ipt_rpfilter.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/iptable_mangle.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/iptable_nat.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/iptable_raw.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/iptable_security.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko 2024-05-20T19:40:10.003 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nf_dup_ipv4.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nf_nat_h323.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nf_nat_pptp.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nf_nat_snmp_basic.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nf_socket_ipv4.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nf_tproxy_ipv4.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nft_dup_ipv4.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nft_fib_ipv4.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/netfilter/nft_reject_ipv4.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_bic.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_dctcp.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_diag.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_highspeed.ko 2024-05-20T19:40:10.004 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_htcp.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_hybla.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_illinois.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_lp.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_scalable.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_vegas.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_veno.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_westwood.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tcp_yeah.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/tunnel4.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/udp_diag.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/udp_tunnel.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv4/xfrm4_tunnel.ko 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6 2024-05-20T19:40:10.005 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/ah6.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/esp6.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/fou6.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/ip6_tunnel.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/ip6_udp_tunnel.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/ip6_vti.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/ipcomp6.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6_tables.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_REJECT.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_SYNPROXY.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_ah.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_eui64.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_frag.ko 2024-05-20T19:40:10.006 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_hbh.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_ipv6header.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_mh.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_rpfilter.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6t_rt.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6table_filter.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6table_mangle.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6table_nat.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6table_raw.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/ip6table_security.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nf_dup_ipv6.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nf_reject_ipv6.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nf_socket_ipv6.ko 2024-05-20T19:40:10.007 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nf_tproxy_ipv6.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nft_dup_ipv6.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nft_fib_ipv6.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/netfilter/nft_reject_ipv6.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/sit.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/tunnel6.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/ipv6/xfrm6_tunnel.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/key 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/key/af_key.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp/l2tp_core.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp/l2tp_debugfs.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp/l2tp_eth.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp/l2tp_ip.ko 2024-05-20T19:40:10.008 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp/l2tp_ip6.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp/l2tp_netlink.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/l2tp/l2tp_ppp.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/llc 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/llc/llc.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/mac802154 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/mac802154/mac802154.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/mpls 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/mpls/mpls_gso.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_bitmap_ip.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_bitmap_ipmac.ko 2024-05-20T19:40:10.009 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_bitmap_port.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_ip.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_ipmark.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_ipport.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_ipportip.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_ipportnet.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_mac.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_net.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_netiface.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_netnet.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_netport.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_hash_netportnet.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipset/ip_set_list_set.ko 2024-05-20T19:40:10.010 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_dh.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_fo.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_ftp.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_lblc.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_lblcr.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_lc.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_nq.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_pe_sip.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_rr.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_sed.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_sh.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_wlc.ko 2024-05-20T19:40:10.011 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/ipvs/ip_vs_wrr.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conncount.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_amanda.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_broadcast.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_ftp.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_h323.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_irc.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_netbios_ns.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_netlink.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_pptp.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_sane.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_sip.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_snmp.ko 2024-05-20T19:40:10.012 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_conntrack_tftp.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_dup_netdev.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_flow_table.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_flow_table_inet.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_log_syslog.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_nat.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_nat_amanda.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_nat_ftp.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_nat_irc.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_nat_sip.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_nat_tftp.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_synproxy_core.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nf_tables.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nfnetlink.ko 2024-05-20T19:40:10.013 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nfnetlink_acct.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nfnetlink_log.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nfnetlink_osf.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nfnetlink_queue.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_chain_nat.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_compat.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_ct.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_dup_netdev.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_fib.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_fwd_netdev.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_hash.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_limit.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_log.ko 2024-05-20T19:40:10.014 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_masq.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_nat.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_numgen.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_queue.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_quota.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_redir.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_reject.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_reject_inet.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_socket.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/nft_tunnel.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_AUDIT.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_CHECKSUM.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_CLASSIFY.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_CONNSECMARK.ko 2024-05-20T19:40:10.015 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_CT.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_DSCP.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_HL.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_HMARK.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_IDLETIMER.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_LED.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_LOG.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_MASQUERADE.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_NETMAP.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_NFLOG.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_NFQUEUE.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_RATEEST.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_REDIRECT.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_SECMARK.ko 2024-05-20T19:40:10.016 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_TCPMSS.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_TCPOPTSTRIP.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_TEE.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_TPROXY.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_TRACE.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_addrtype.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_bpf.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_cgroup.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_cluster.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_comment.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_connbytes.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_connlabel.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_connlimit.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_connmark.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_conntrack.ko 2024-05-20T19:40:10.017 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_cpu.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_dccp.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_devgroup.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_dscp.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_ecn.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_esp.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_hashlimit.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_helper.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_hl.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_ipcomp.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_iprange.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_ipvs.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_l2tp.ko 2024-05-20T19:40:10.018 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_length.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_limit.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_mac.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_mark.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_multiport.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_nat.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_nfacct.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_osf.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_owner.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_physdev.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_pkttype.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_policy.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_quota.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_rateest.ko 2024-05-20T19:40:10.019 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_realm.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_recent.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_sctp.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_set.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_socket.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_state.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_statistic.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_string.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_tcpmss.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_time.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netfilter/xt_u32.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netlink 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/netlink/netlink_diag.ko 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/nsh 2024-05-20T19:40:10.020 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/nsh/nsh.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/openvswitch 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/openvswitch/openvswitch.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/openvswitch/vport-geneve.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/openvswitch/vport-gre.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/openvswitch/vport-vxlan.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/packet 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/packet/af_packet_diag.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/rds 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/rds/rds.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/rds/rds_rdma.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/rds/rds_tcp.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_csum.ko 2024-05-20T19:40:10.021 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_gact.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_mirred.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_nat.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_pedit.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_police.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_simple.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_skbedit.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/act_vlan.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/cls_basic.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/cls_bpf.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/cls_flow.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/cls_fw.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/cls_route.ko 2024-05-20T19:40:10.022 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/cls_u32.ko 2024-05-20T19:40:10.023 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/em_cmp.ko 2024-05-20T19:40:10.023 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/em_ipset.ko 2024-05-20T19:40:10.023 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/em_meta.ko 2024-05-20T19:40:10.023 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/em_nbyte.ko 2024-05-20T19:40:10.023 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/em_text.ko 2024-05-20T19:40:10.023 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/em_u32.ko 2024-05-20T19:40:10.025 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_choke.ko 2024-05-20T19:40:10.025 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_codel.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_drr.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_fq.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_gred.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_hfsc.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_hhf.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_htb.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_ingress.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_mqprio.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_mqprio_lib.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_multiq.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_netem.ko 2024-05-20T19:40:10.026 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_pie.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_plug.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_prio.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_qfq.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_red.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_sfb.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_sfq.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_tbf.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sched/sch_teql.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sctp 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sctp/sctp.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sctp/sctp_diag.ko 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sunrpc 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sunrpc/auth_gss 2024-05-20T19:40:10.027 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sunrpc/auth_gss/auth_rpcgss.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sunrpc/auth_gss/rpcsec_gss_krb5.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sunrpc/sunrpc.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sunrpc/xprtrdma 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/sunrpc/xprtrdma/rpcrdma.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/tipc 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/tipc/tipc.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/tipc/tipc_diag.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/unix 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/unix/unix_diag.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/vmw_vsock 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/vmw_vsock/vmw_vsock_virtio_transport_common.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/vmw_vsock/vmw_vsock_vmci_transport.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/vmw_vsock/vsock.ko 2024-05-20T19:40:10.028 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/vmw_vsock/vsock_diag.ko 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/vmw_vsock/vsock_loopback.ko 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/xfrm 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/net/xfrm/xfrm_ipcomp.ko 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/security 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/security/keys 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/security/keys/encrypted-keys 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/security/keys/encrypted-keys/encrypted-keys.ko 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/security/keys/trusted-keys 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/kernel/security/keys/trusted-keys/trusted.ko 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.alias 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.alias.bin 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.builtin 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.builtin.alias.bin 2024-05-20T19:40:10.029 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.builtin.bin 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.builtin.modinfo 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.dep 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.dep.bin 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.devname 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.order 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.softdep 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.symbols 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.symbols.bin 2024-05-20T19:40:10.030 INFO:teuthology.orchestra.run.smithi028.stdout:/lib/modules/6.9.0-rc3-g0fe9556839d2/vmlinuz 2024-05-20T19:40:10.034 DEBUG:teuthology.task.kernel:get_image_version: 6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:10.034 DEBUG:teuthology.orchestra.run.smithi028:> rpm --scripts -qp /tmp/kernel.x86_64.rpm 2024-05-20T19:40:10.060 INFO:teuthology.orchestra.run.smithi028.stdout:postinstall scriptlet (using /bin/sh): 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout:if [ -x /usr/bin/kernel-install ]; then 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout: /usr/bin/kernel-install add 6.9.0-rc3-g0fe9556839d2 /lib/modules/6.9.0-rc3-g0fe9556839d2/vmlinuz 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout:fi 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout:for file in vmlinuz System.map config; do 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout: if ! cmp --silent "/lib/modules/6.9.0-rc3-g0fe9556839d2/${file}" "/boot/${file}-6.9.0-rc3-g0fe9556839d2"; then 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout: cp "/lib/modules/6.9.0-rc3-g0fe9556839d2/${file}" "/boot/${file}-6.9.0-rc3-g0fe9556839d2" 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout: fi 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout:done 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout:if [ -d "/lib/modules/6.9.0-rc3-g0fe9556839d2/dtb" ] && \ 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout: ! diff -rq "/lib/modules/6.9.0-rc3-g0fe9556839d2/dtb" "/boot/dtb-6.9.0-rc3-g0fe9556839d2" >/dev/null 2>&1; then 2024-05-20T19:40:10.061 INFO:teuthology.orchestra.run.smithi028.stdout: rm -rf "/boot/dtb-6.9.0-rc3-g0fe9556839d2" 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout: cp -r "/lib/modules/6.9.0-rc3-g0fe9556839d2/dtb" "/boot/dtb-6.9.0-rc3-g0fe9556839d2" 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout:fi 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout:if [ ! -e "/lib/modules/6.9.0-rc3-g0fe9556839d2/modules.dep" ]; then 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout: /usr/sbin/depmod 6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout:fi 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout:preuninstall scriptlet (using /bin/sh): 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout:if [ -x /usr/bin/kernel-install ]; then 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout:kernel-install remove 6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:10.062 INFO:teuthology.orchestra.run.smithi028.stdout:fi 2024-05-20T19:40:10.063 DEBUG:teuthology.orchestra.run.smithi028:> sudo rpm -qi grub2-tools 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Name : grub2-tools 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Epoch : 1 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Version : 2.06 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Release : 68.el9 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Architecture: x86_64 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Group : Unspecified 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Size : 8264254 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:License : GPLv3+ 2024-05-20T19:40:10.141 INFO:teuthology.orchestra.run.smithi028.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:Packager : builder@centos.org 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:Vendor : CentOS 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:Summary : Support tools for GRUB. 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:Description : 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-20T19:40:10.142 INFO:teuthology.orchestra.run.smithi028.stdout:hardware devices. 2024-05-20T19:40:10.143 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:40:10.143 INFO:teuthology.orchestra.run.smithi028.stdout:This subpackage provides tools for support of all platforms. 2024-05-20T19:40:10.143 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-20T19:40:10.144 INFO:teuthology.task.kernel:Updating grub on smithi028 to boot 6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:10.144 DEBUG:teuthology.orchestra.run.smithi028:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-20T19:40:10.723 INFO:teuthology.orchestra.run.smithi028.stderr:Generating grub configuration file ... 2024-05-20T19:40:12.137 INFO:teuthology.orchestra.run.smithi028.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-20T19:40:12.166 INFO:teuthology.orchestra.run.smithi028.stderr:done 2024-05-20T19:40:12.168 DEBUG:teuthology.orchestra.run.smithi028:> mktemp 2024-05-20T19:40:12.189 INFO:teuthology.orchestra.run.smithi028.stdout:/tmp/tmp.PVsjCOPbXc 2024-05-20T19:40:12.189 DEBUG:teuthology.orchestra.run.smithi028:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.PVsjCOPbXc 2024-05-20T19:40:12.255 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0666 /tmp/tmp.PVsjCOPbXc 2024-05-20T19:40:12.402 DEBUG:teuthology.orchestra.remote:smithi028:/tmp/tmp.PVsjCOPbXc is 6KB 2024-05-20T19:40:12.444 DEBUG:teuthology.orchestra.run.smithi028:> rm -fr /tmp/tmp.PVsjCOPbXc 2024-05-20T19:40:12.462 DEBUG:teuthology.orchestra.run.smithi028:> sudo /bin/ls /boot/loader/entries || true 2024-05-20T19:40:12.524 INFO:teuthology.orchestra.run.smithi028.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-20T19:40:12.524 INFO:teuthology.orchestra.run.smithi028.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-20T19:40:12.524 INFO:teuthology.orchestra.run.smithi028.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-20T19:40:12.524 INFO:teuthology.orchestra.run.smithi028.stdout:7fc8828e06d0482694bcf4a447d13cb6-6.9.0-rc3-g0fe9556839d2.conf 2024-05-20T19:40:12.525 DEBUG:teuthology.orchestra.run.smithi028:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-6.9.0-rc3-g0fe9556839d2 2024-05-20T19:40:12.695 DEBUG:teuthology.orchestra.run.smithi028:> sudo shutdown -r now 2024-05-20T19:40:12.781 ERROR:paramiko.transport:Socket exception: Connection reset by peer (104) 2024-05-20T19:40:42.705 INFO:teuthology.task.kernel:Checking client client.0 for new kernel version... 2024-05-20T19:40:42.705 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-05-20T19:40:42.706 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:41:42.708 DEBUG:teuthology.orchestra.remote:timed out 2024-05-20T19:41:51.710 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-05-20T19:41:51.711 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:41:52.272 DEBUG:teuthology.orchestra.run.smithi028:> true 2024-05-20T19:41:53.110 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-05-20T19:41:53.110 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "6.9.0-rc3-g0fe9556839d2"... 2024-05-20T19:41:53.111 DEBUG:teuthology.orchestra.run.smithi028:> uname -r 2024-05-20T19:41:53.152 INFO:teuthology.orchestra.run.smithi028.stdout:6.9.0-rc3-g0fe9556839d2 2024-05-20T19:41:53.152 DEBUG:teuthology.task.kernel:current kernel version is 6.9.0-rc3-g0fe9556839d2 vs 6.9.0-rc3-g0fe9556839d2 2024-05-20T19:41:53.152 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-20T19:41:53.152 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-20T19:41:54.155 DEBUG:teuthology.parallel:result is None 2024-05-20T19:41:54.170 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-20T19:41:54.937 INFO:teuthology.task.internal:Creating test directory... 2024-05-20T19:41:54.937 DEBUG:teuthology.orchestra.run.smithi028:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-20T19:41:54.946 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-20T19:41:54.959 DEBUG:teuthology.orchestra.run.smithi154:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-20T19:41:54.985 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-20T19:41:55.024 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-20T19:41:55.055 INFO:teuthology.task.internal:Creating archive directory... 2024-05-20T19:41:55.055 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-20T19:41:55.061 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-20T19:41:55.064 DEBUG:teuthology.orchestra.run.smithi154:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-20T19:41:55.107 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-20T19:41:55.119 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-20T19:41:55.119 DEBUG:teuthology.orchestra.run.smithi033:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-20T19:41:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T19:41:55.147 DEBUG:teuthology.orchestra.run.smithi154:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-20T19:41:55.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T19:41:55.164 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-20T19:41:55.171 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-20T19:41:55.191 DEBUG:teuthology.orchestra.run.smithi154:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-20T19:41:55.227 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T19:41:55.243 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T19:41:55.244 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T19:41:55.279 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T19:41:55.290 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T19:41:55.381 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T19:41:55.385 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-20T19:41:55.636 INFO:teuthology.task.internal:Configuring sudo... 2024-05-20T19:41:55.636 DEBUG:teuthology.orchestra.run.smithi028:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-20T19:41:55.642 DEBUG:teuthology.orchestra.run.smithi033:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-20T19:41:55.645 DEBUG:teuthology.orchestra.run.smithi154:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-20T19:41:55.707 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-20T19:41:55.927 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-20T19:41:55.927 DEBUG:teuthology.orchestra.run.smithi028:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-20T19:41:55.934 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-20T19:41:55.937 DEBUG:teuthology.orchestra.run.smithi154:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-20T19:41:55.983 DEBUG:teuthology.orchestra.run.smithi028:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T19:41:56.062 DEBUG:teuthology.orchestra.run.smithi028:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T19:41:56.151 DEBUG:teuthology.orchestra.run.smithi028:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T19:41:56.228 DEBUG:teuthology.orchestra.run.smithi028:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T19:41:56.303 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:41:56.303 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-20T19:41:56.383 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T19:41:56.429 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T19:41:56.476 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T19:41:56.557 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T19:41:56.605 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:41:56.605 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-20T19:41:56.673 DEBUG:teuthology.orchestra.run.smithi154:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T19:41:56.718 DEBUG:teuthology.orchestra.run.smithi154:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T19:41:56.770 DEBUG:teuthology.orchestra.run.smithi154:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T19:41:56.852 DEBUG:teuthology.orchestra.run.smithi154:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T19:41:56.895 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:41:56.895 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-20T19:41:56.960 DEBUG:teuthology.orchestra.run.smithi028:> sudo service rsyslog restart 2024-05-20T19:41:56.964 DEBUG:teuthology.orchestra.run.smithi033:> sudo service rsyslog restart 2024-05-20T19:41:56.967 DEBUG:teuthology.orchestra.run.smithi154:> sudo service rsyslog restart 2024-05-20T19:41:57.001 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T19:41:57.042 INFO:teuthology.orchestra.run.smithi154.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T19:41:57.044 INFO:teuthology.orchestra.run.smithi028.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T19:41:57.639 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-20T19:41:57.647 INFO:teuthology.task.internal:Starting timer... 2024-05-20T19:41:57.647 INFO:teuthology.run_tasks:Running task pcp... 2024-05-20T19:41:57.657 INFO:teuthology.run_tasks:Running task selinux... 2024-05-20T19:41:57.667 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-05-20T19:41:57.668 DEBUG:teuthology.orchestra.run.smithi028:> sudo service auditd rotate 2024-05-20T19:41:57.775 INFO:teuthology.orchestra.run.smithi028.stdout:Rotating logs: 2024-05-20T19:41:57.779 DEBUG:teuthology.orchestra.run.smithi033:> sudo service auditd rotate 2024-05-20T19:41:57.845 INFO:teuthology.orchestra.run.smithi033.stdout:Rotating logs: 2024-05-20T19:41:57.847 DEBUG:teuthology.orchestra.run.smithi154:> sudo service auditd rotate 2024-05-20T19:41:57.915 INFO:teuthology.orchestra.run.smithi154.stdout:Rotating logs: 2024-05-20T19:41:57.917 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-20T19:41:57.918 DEBUG:teuthology.orchestra.run.smithi028:> /usr/sbin/getenforce 2024-05-20T19:41:57.959 INFO:teuthology.orchestra.run.smithi028.stdout:Permissive 2024-05-20T19:41:57.959 DEBUG:teuthology.orchestra.run.smithi033:> /usr/sbin/getenforce 2024-05-20T19:41:57.983 INFO:teuthology.orchestra.run.smithi033.stdout:Permissive 2024-05-20T19:41:57.983 DEBUG:teuthology.orchestra.run.smithi154:> /usr/sbin/getenforce 2024-05-20T19:41:58.008 INFO:teuthology.orchestra.run.smithi154.stdout:Permissive 2024-05-20T19:41:58.008 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi028.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi033.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi154.front.sepia.ceph.com': 'permissive'} 2024-05-20T19:41:58.009 DEBUG:teuthology.orchestra.run.smithi028:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T19:41:58.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T19:41:58.072 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T19:41:58.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T19:41:58.099 DEBUG:teuthology.orchestra.run.smithi154:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T19:41:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T19:41:58.125 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-20T19:41:58.125 DEBUG:teuthology.orchestra.run.smithi028:> sudo /usr/sbin/setenforce permissive 2024-05-20T19:41:58.187 DEBUG:teuthology.orchestra.run.smithi033:> sudo /usr/sbin/setenforce permissive 2024-05-20T19:41:58.219 DEBUG:teuthology.orchestra.run.smithi154:> sudo /usr/sbin/setenforce permissive 2024-05-20T19:41:58.246 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-20T19:41:58.256 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-05-20T19:41:58.259 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-05-20T19:41:58.408 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-20T19:41:58.421 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-20T19:41:58.426 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi028.front.sepia.ceph.com,smithi033.front.sepia.ceph.com,smithi154.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-20T19:48:17.955 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi028.front.sepia.ceph.com'), Remote(name='ubuntu@smithi033.front.sepia.ceph.com'), Remote(name='ubuntu@smithi154.front.sepia.ceph.com')] 2024-05-20T19:48:17.957 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-05-20T19:48:17.958 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:48:18.072 DEBUG:teuthology.orchestra.run.smithi028:> true 2024-05-20T19:48:18.226 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-05-20T19:48:18.227 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-05-20T19:48:18.227 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:48:18.308 DEBUG:teuthology.orchestra.run.smithi033:> true 2024-05-20T19:48:18.392 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-05-20T19:48:18.393 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-05-20T19:48:18.393 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T19:48:18.473 DEBUG:teuthology.orchestra.run.smithi154:> true 2024-05-20T19:48:18.555 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-05-20T19:48:18.556 INFO:teuthology.run_tasks:Running task clock... 2024-05-20T19:48:18.569 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-20T19:48:18.570 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-20T19:48:18.570 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T19:48:18.578 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-20T19:48:18.579 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T19:48:18.582 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-20T19:48:18.582 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T19:48:18.613 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-20T19:48:18.628 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-20T19:48:18.628 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-20T19:48:18.642 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-20T19:48:18.654 INFO:teuthology.orchestra.run.smithi033.stderr:sudo: ntpd: command not found 2024-05-20T19:48:18.665 INFO:teuthology.orchestra.run.smithi033.stdout:506 Cannot talk to daemon 2024-05-20T19:48:18.666 INFO:teuthology.orchestra.run.smithi154.stderr:sudo: ntpd: command not found 2024-05-20T19:48:18.669 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-20T19:48:18.677 INFO:teuthology.orchestra.run.smithi154.stdout:506 Cannot talk to daemon 2024-05-20T19:48:18.680 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-20T19:48:18.691 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-20T19:48:18.694 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-20T19:48:18.704 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-20T19:48:18.714 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-20T19:48:18.749 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-05-20T19:48:18.754 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T19:48:18.754 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-05-20T19:48:18.754 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.755 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.755 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.755 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.758 INFO:teuthology.orchestra.run.smithi154.stderr:bash: line 1: ntpq: command not found 2024-05-20T19:48:18.763 INFO:teuthology.orchestra.run.smithi154.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T19:48:18.763 INFO:teuthology.orchestra.run.smithi154.stdout:=============================================================================== 2024-05-20T19:48:18.763 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.763 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.763 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.764 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:18.792 INFO:teuthology.orchestra.run.smithi028.stderr:sudo: ntpd: command not found 2024-05-20T19:48:18.825 INFO:teuthology.orchestra.run.smithi028.stdout:506 Cannot talk to daemon 2024-05-20T19:48:18.864 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-20T19:48:18.904 INFO:teuthology.orchestra.run.smithi028.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-20T19:48:19.036 INFO:teuthology.orchestra.run.smithi028.stderr:bash: line 1: ntpq: command not found 2024-05-20T19:48:19.046 INFO:teuthology.orchestra.run.smithi028.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T19:48:19.046 INFO:teuthology.orchestra.run.smithi028.stdout:=============================================================================== 2024-05-20T19:48:19.046 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:19.046 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:19.046 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:19.046 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T19:48:19.048 INFO:teuthology.run_tasks:Running task install... 2024-05-20T19:48:19.057 DEBUG:teuthology.task.install:project ceph 2024-05-20T19:48:19.057 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '4ebd1d5361d457738957dc0455bba90102296634'}} 2024-05-20T19:48:19.057 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '4ebd1d5361d457738957dc0455bba90102296634'} 2024-05-20T19:48:19.058 INFO:teuthology.task.install:Using flavor: default 2024-05-20T19:48:19.063 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-05-20T19:48:19.063 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']} 2024-05-20T19:48:19.064 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '4ebd1d5361d457738957dc0455bba90102296634', 'tag': None, 'wait_for_package': False} 2024-05-20T19:48:19.064 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:48:19.066 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '4ebd1d5361d457738957dc0455bba90102296634', 'tag': None, 'wait_for_package': False} 2024-05-20T19:48:19.066 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:48:19.067 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '4ebd1d5361d457738957dc0455bba90102296634', 'tag': None, 'wait_for_package': False} 2024-05-20T19:48:19.067 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:48:19.212 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/ 2024-05-20T19:48:19.212 INFO:teuthology.task.install.rpm:Package version is 19.0.0-3830.g4ebd1d53 2024-05-20T19:48:19.217 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/ 2024-05-20T19:48:19.217 INFO:teuthology.task.install.rpm:Package version is 19.0.0-3830.g4ebd1d53 2024-05-20T19:48:19.282 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/ 2024-05-20T19:48:19.282 INFO:teuthology.task.install.rpm:Package version is 19.0.0-3830.g4ebd1d53 2024-05-20T19:48:19.332 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-05-20T19:48:19.332 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:48:19.332 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-05-20T19:48:19.360 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-05-20T19:48:19.361 DEBUG:teuthology.orchestra.run.smithi033:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/4ebd1d5361d457738957dc0455bba90102296634/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-05-20T19:48:19.416 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-05-20T19:48:19.416 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:48:19.416 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-05-20T19:48:19.423 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-05-20T19:48:19.423 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:48:19.423 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-05-20T19:48:19.430 DEBUG:teuthology.orchestra.run.smithi033:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-05-20T19:48:19.442 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-05-20T19:48:19.442 DEBUG:teuthology.orchestra.run.smithi154:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/4ebd1d5361d457738957dc0455bba90102296634/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-05-20T19:48:19.507 DEBUG:teuthology.orchestra.run.smithi033:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-05-20T19:48:19.508 DEBUG:teuthology.orchestra.run.smithi154:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-05-20T19:48:19.520 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-shell, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-05-20T19:48:19.520 DEBUG:teuthology.orchestra.run.smithi028:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/4ebd1d5361d457738957dc0455bba90102296634/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-05-20T19:48:19.572 INFO:teuthology.orchestra.run.smithi033.stdout:check_obsoletes = 1 2024-05-20T19:48:19.574 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum clean all 2024-05-20T19:48:19.584 DEBUG:teuthology.orchestra.run.smithi154:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-05-20T19:48:19.647 INFO:teuthology.orchestra.run.smithi154.stdout:check_obsoletes = 1 2024-05-20T19:48:19.648 DEBUG:teuthology.orchestra.run.smithi028:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-05-20T19:48:19.649 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum clean all 2024-05-20T19:48:19.774 DEBUG:teuthology.orchestra.run.smithi028:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-05-20T19:48:19.814 INFO:teuthology.orchestra.run.smithi033.stdout:57 files removed 2024-05-20T19:48:19.842 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-05-20T19:48:19.860 INFO:teuthology.orchestra.run.smithi028.stdout:check_obsoletes = 1 2024-05-20T19:48:19.865 DEBUG:teuthology.orchestra.run.smithi028:> sudo yum clean all 2024-05-20T19:48:19.877 INFO:teuthology.orchestra.run.smithi154.stdout:57 files removed 2024-05-20T19:48:19.904 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-05-20T19:48:20.221 INFO:teuthology.orchestra.run.smithi028.stdout:57 files removed 2024-05-20T19:48:20.261 DEBUG:teuthology.orchestra.run.smithi028:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-shell cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-05-20T19:48:20.487 INFO:teuthology.orchestra.run.smithi033.stdout:ceph packages for x86_64 226 kB/s | 84 kB 00:00 2024-05-20T19:48:20.638 INFO:teuthology.orchestra.run.smithi154.stdout:ceph packages for x86_64 178 kB/s | 84 kB 00:00 2024-05-20T19:48:20.871 INFO:teuthology.orchestra.run.smithi033.stdout:ceph noarch packages 35 kB/s | 12 kB 00:00 2024-05-20T19:48:21.056 INFO:teuthology.orchestra.run.smithi154.stdout:ceph noarch packages 31 kB/s | 12 kB 00:00 2024-05-20T19:48:21.175 INFO:teuthology.orchestra.run.smithi028.stdout:ceph packages for x86_64 181 kB/s | 84 kB 00:00 2024-05-20T19:48:21.263 INFO:teuthology.orchestra.run.smithi033.stdout:ceph source packages 5.3 kB/s | 1.9 kB 00:00 2024-05-20T19:48:21.398 INFO:teuthology.orchestra.run.smithi154.stdout:ceph source packages 6.2 kB/s | 1.9 kB 00:00 2024-05-20T19:48:21.751 INFO:teuthology.orchestra.run.smithi028.stdout:ceph noarch packages 25 kB/s | 12 kB 00:00 2024-05-20T19:48:22.250 INFO:teuthology.orchestra.run.smithi028.stdout:ceph source packages 4.6 kB/s | 1.9 kB 00:00 2024-05-20T19:48:23.193 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.1 MB 00:01 2024-05-20T19:48:23.294 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.1 MB 00:01 2024-05-20T19:48:25.087 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - BaseOS 2.9 MB/s | 8.1 MB 00:02 2024-05-20T19:48:26.362 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - AppStream 9.3 MB/s | 19 MB 00:02 2024-05-20T19:48:26.611 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - AppStream 8.5 MB/s | 19 MB 00:02 2024-05-20T19:48:27.376 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - AppStream 19 MB/s | 19 MB 00:01 2024-05-20T19:48:31.983 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - CRB 3.2 MB/s | 6.2 MB 00:01 2024-05-20T19:48:32.649 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - CRB 2.6 MB/s | 6.2 MB 00:02 2024-05-20T19:48:33.362 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - Extras packages 92 kB/s | 16 kB 00:00 2024-05-20T19:48:33.965 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - CRB 2.4 MB/s | 6.2 MB 00:02 2024-05-20T19:48:34.169 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - Extras packages 42 kB/s | 16 kB 00:00 2024-05-20T19:48:35.329 INFO:teuthology.orchestra.run.smithi154.stdout:Extra Packages for Enterprise Linux 20 MB/s | 21 MB 00:01 2024-05-20T19:48:35.578 INFO:teuthology.orchestra.run.smithi028.stdout:CentOS Stream 9 - Extras packages 73 kB/s | 16 kB 00:00 2024-05-20T19:48:37.617 INFO:teuthology.orchestra.run.smithi033.stdout:Extra Packages for Enterprise Linux 5.1 MB/s | 21 MB 00:04 2024-05-20T19:48:37.769 INFO:teuthology.orchestra.run.smithi028.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:02 2024-05-20T19:48:40.554 INFO:teuthology.orchestra.run.smithi154.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-05-20T19:48:42.688 INFO:teuthology.orchestra.run.smithi154.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-05-20T19:48:42.689 INFO:teuthology.orchestra.run.smithi154.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-05-20T19:48:42.700 INFO:teuthology.orchestra.run.smithi154.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-05-20T19:48:42.701 INFO:teuthology.orchestra.run.smithi154.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-05-20T19:48:42.702 INFO:teuthology.orchestra.run.smithi154.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-05-20T19:48:42.702 INFO:teuthology.orchestra.run.smithi154.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-05-20T19:48:42.704 INFO:teuthology.orchestra.run.smithi154.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-05-20T19:48:42.708 INFO:teuthology.orchestra.run.smithi154.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-05-20T19:48:42.708 INFO:teuthology.orchestra.run.smithi154.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-05-20T19:48:42.769 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-05-20T19:48:42.789 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-05-20T19:48:42.789 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-05-20T19:48:42.789 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-05-20T19:48:42.789 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 6.6 k 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-base x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 5.4 M 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-fuse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 890 k 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 145 k 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 1.5 M 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-cephadm noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 143 k 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-dashboard noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 3.5 M 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 7.4 M 2024-05-20T19:48:42.790 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-rook noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 49 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-radosgw x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 11 M 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-test x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 49 M 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-volume noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 273 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: cephadm noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 764 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: cephfs-mirror x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 225 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: cephfs-shell x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 27 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: cephfs-top noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 18 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-libelf-devel x86_64 0.191-4.el9 appstream 22 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-05-20T19:48:42.791 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs-devel x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 32 k 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 747 k 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: librados-devel x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 127 k 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: libtool x86_64 2.4.6-45.el9 appstream 582 k 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: openssl-devel x86_64 1:3.2.1-1.el9 appstream 4.4 M 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cephfs x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 161 k 2024-05-20T19:48:42.792 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rados x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 320 k 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rbd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 300 k 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rgw x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 100 k 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-fuse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 85 k 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-mirror x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.1 M 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-nbd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 171 k 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout:Upgrading: 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager x86_64 1:1.47.90-1.el9 baseos 2.3 M 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager-libnm x86_64 1:1.47.90-1.el9 baseos 1.8 M 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager-team x86_64 1:1.47.90-1.el9 baseos 41 k 2024-05-20T19:48:42.793 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager-tui x86_64 1:1.47.90-1.el9 baseos 248 k 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-debuginfod-client x86_64 0.191-4.el9 baseos 37 k 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-libelf x86_64 0.191-4.el9 baseos 207 k 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-libs x86_64 0.191-4.el9 baseos 260 k 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: librados2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.4 M 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: librbd1 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.1 M 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: openssl x86_64 1:3.2.1-1.el9 baseos 1.3 M 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: openssl-libs x86_64 1:3.2.1-1.el9 baseos 2.4 M 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-05-20T19:48:42.794 INFO:teuthology.orchestra.run.smithi154.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-common x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 21 M 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-grafana-dashboards noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 34 k 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mds x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 2.4 M 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-modules-core noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 274 k 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mon x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 4.7 M 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-osd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 17 M 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-prometheus-alerts noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 16 k 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-selinux x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 25 k 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-05-20T19:48:42.795 INFO:teuthology.orchestra.run.smithi154.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: libcephsqlite x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 163 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-05-20T19:48:42.796 INFO:teuthology.orchestra.run.smithi154.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-05-20T19:48:42.797 INFO:teuthology.orchestra.run.smithi154.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-05-20T19:48:42.797 INFO:teuthology.orchestra.run.smithi154.stdout: libradosstriper1 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 501 k 2024-05-20T19:48:42.797 INFO:teuthology.orchestra.run.smithi154.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-05-20T19:48:42.797 INFO:teuthology.orchestra.run.smithi154.stdout: librgw2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 5.4 M 2024-05-20T19:48:42.797 INFO:teuthology.orchestra.run.smithi154.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-05-20T19:48:42.798 INFO:teuthology.orchestra.run.smithi154.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-05-20T19:48:42.798 INFO:teuthology.orchestra.run.smithi154.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-05-20T19:48:42.798 INFO:teuthology.orchestra.run.smithi154.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-05-20T19:48:42.798 INFO:teuthology.orchestra.run.smithi154.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-05-20T19:48:42.798 INFO:teuthology.orchestra.run.smithi154.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-05-20T19:48:42.799 INFO:teuthology.orchestra.run.smithi154.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-05-20T19:48:42.799 INFO:teuthology.orchestra.run.smithi154.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-05-20T19:48:42.799 INFO:teuthology.orchestra.run.smithi154.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-05-20T19:48:42.799 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-05-20T19:48:42.799 INFO:teuthology.orchestra.run.smithi154.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-05-20T19:48:42.800 INFO:teuthology.orchestra.run.smithi154.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-05-20T19:48:42.800 INFO:teuthology.orchestra.run.smithi154.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-05-20T19:48:42.800 INFO:teuthology.orchestra.run.smithi154.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-05-20T19:48:42.800 INFO:teuthology.orchestra.run.smithi154.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-05-20T19:48:42.801 INFO:teuthology.orchestra.run.smithi154.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-05-20T19:48:42.801 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-05-20T19:48:42.801 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-argparse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 45 k 2024-05-20T19:48:42.801 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-common x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 136 k 2024-05-20T19:48:42.801 INFO:teuthology.orchestra.run.smithi154.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-05-20T19:48:42.802 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-05-20T19:48:42.802 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-05-20T19:48:42.802 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-05-20T19:48:42.802 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-05-20T19:48:42.802 INFO:teuthology.orchestra.run.smithi154.stdout: python3-colorama noarch 0.4.6-3.el9 appstream 61 k 2024-05-20T19:48:42.803 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-05-20T19:48:42.803 INFO:teuthology.orchestra.run.smithi154.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-05-20T19:48:42.803 INFO:teuthology.orchestra.run.smithi154.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-05-20T19:48:42.803 INFO:teuthology.orchestra.run.smithi154.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-05-20T19:48:42.803 INFO:teuthology.orchestra.run.smithi154.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-05-20T19:48:42.804 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-05-20T19:48:42.804 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-05-20T19:48:42.804 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-05-20T19:48:42.804 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-05-20T19:48:42.804 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-05-20T19:48:42.805 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-05-20T19:48:42.805 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-05-20T19:48:42.805 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-05-20T19:48:42.805 INFO:teuthology.orchestra.run.smithi154.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-05-20T19:48:42.805 INFO:teuthology.orchestra.run.smithi154.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-05-20T19:48:42.806 INFO:teuthology.orchestra.run.smithi154.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-05-20T19:48:42.806 INFO:teuthology.orchestra.run.smithi154.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-05-20T19:48:42.806 INFO:teuthology.orchestra.run.smithi154.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-05-20T19:48:42.806 INFO:teuthology.orchestra.run.smithi154.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-05-20T19:48:42.806 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-05-20T19:48:42.807 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-05-20T19:48:42.807 INFO:teuthology.orchestra.run.smithi154.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-05-20T19:48:42.807 INFO:teuthology.orchestra.run.smithi154.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-05-20T19:48:42.807 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-05-20T19:48:42.807 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-05-20T19:48:42.807 INFO:teuthology.orchestra.run.smithi154.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-05-20T19:48:42.808 INFO:teuthology.orchestra.run.smithi154.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-05-20T19:48:42.808 INFO:teuthology.orchestra.run.smithi154.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-05-20T19:48:42.808 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-05-20T19:48:42.808 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-05-20T19:48:42.808 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-05-20T19:48:42.808 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-05-20T19:48:42.809 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-05-20T19:48:42.809 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-05-20T19:48:42.809 INFO:teuthology.orchestra.run.smithi154.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-05-20T19:48:42.809 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-05-20T19:48:42.809 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-05-20T19:48:42.809 INFO:teuthology.orchestra.run.smithi154.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-05-20T19:48:42.810 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-05-20T19:48:42.810 INFO:teuthology.orchestra.run.smithi154.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-05-20T19:48:42.810 INFO:teuthology.orchestra.run.smithi154.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-05-20T19:48:42.810 INFO:teuthology.orchestra.run.smithi154.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-05-20T19:48:42.810 INFO:teuthology.orchestra.run.smithi154.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-05-20T19:48:42.810 INFO:teuthology.orchestra.run.smithi154.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-05-20T19:48:42.811 INFO:teuthology.orchestra.run.smithi154.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-05-20T19:48:42.811 INFO:teuthology.orchestra.run.smithi154.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-05-20T19:48:42.811 INFO:teuthology.orchestra.run.smithi154.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-05-20T19:48:42.811 INFO:teuthology.orchestra.run.smithi154.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-05-20T19:48:42.811 INFO:teuthology.orchestra.run.smithi154.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-05-20T19:48:42.811 INFO:teuthology.orchestra.run.smithi154.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-05-20T19:48:42.812 INFO:teuthology.orchestra.run.smithi154.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-05-20T19:48:42.812 INFO:teuthology.orchestra.run.smithi154.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-05-20T19:48:42.812 INFO:teuthology.orchestra.run.smithi154.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-05-20T19:48:42.812 INFO:teuthology.orchestra.run.smithi154.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-05-20T19:48:42.812 INFO:teuthology.orchestra.run.smithi154.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-05-20T19:48:42.813 INFO:teuthology.orchestra.run.smithi154.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-05-20T19:48:42.813 INFO:teuthology.orchestra.run.smithi154.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-05-20T19:48:42.813 INFO:teuthology.orchestra.run.smithi154.stdout:Installing weak dependencies: 2024-05-20T19:48:42.813 INFO:teuthology.orchestra.run.smithi154.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-05-20T19:48:42.813 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:48:42.814 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-05-20T19:48:42.814 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-05-20T19:48:42.814 INFO:teuthology.orchestra.run.smithi154.stdout:Install 148 Packages 2024-05-20T19:48:42.814 INFO:teuthology.orchestra.run.smithi154.stdout:Upgrade 11 Packages 2024-05-20T19:48:42.814 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:48:42.815 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 227 M 2024-05-20T19:48:42.816 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-05-20T19:48:42.888 INFO:teuthology.orchestra.run.smithi033.stdout:lab-extras 49 kB/s | 1.7 kB 00:00 2024-05-20T19:48:43.689 INFO:teuthology.orchestra.run.smithi154.stdout:(1/159): ceph-19.0.0-3830.g4ebd1d53.el9.x86_64. 17 kB/s | 6.6 kB 00:00 2024-05-20T19:48:44.230 INFO:teuthology.orchestra.run.smithi028.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-05-20T19:48:44.256 INFO:teuthology.orchestra.run.smithi154.stdout:(2/159): ceph-fuse-19.0.0-3830.g4ebd1d53.el9.x8 1.5 MB/s | 890 kB 00:00 2024-05-20T19:48:44.372 INFO:teuthology.orchestra.run.smithi154.stdout:(3/159): ceph-immutable-object-cache-19.0.0-383 1.2 MB/s | 145 kB 00:00 2024-05-20T19:48:44.731 INFO:teuthology.orchestra.run.smithi154.stdout:(4/159): ceph-mds-19.0.0-3830.g4ebd1d53.el9.x86 6.6 MB/s | 2.4 MB 00:00 2024-05-20T19:48:44.973 INFO:teuthology.orchestra.run.smithi154.stdout:(5/159): ceph-base-19.0.0-3830.g4ebd1d53.el9.x8 3.2 MB/s | 5.4 MB 00:01 2024-05-20T19:48:45.015 INFO:teuthology.orchestra.run.smithi154.stdout:(6/159): ceph-mgr-19.0.0-3830.g4ebd1d53.el9.x86 5.3 MB/s | 1.5 MB 00:00 2024-05-20T19:48:45.106 INFO:teuthology.orchestra.run.smithi033.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-05-20T19:48:45.107 INFO:teuthology.orchestra.run.smithi033.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-05-20T19:48:45.118 INFO:teuthology.orchestra.run.smithi033.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-05-20T19:48:45.119 INFO:teuthology.orchestra.run.smithi033.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-05-20T19:48:45.120 INFO:teuthology.orchestra.run.smithi033.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-05-20T19:48:45.121 INFO:teuthology.orchestra.run.smithi033.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-05-20T19:48:45.123 INFO:teuthology.orchestra.run.smithi033.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-05-20T19:48:45.125 INFO:teuthology.orchestra.run.smithi033.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-05-20T19:48:45.126 INFO:teuthology.orchestra.run.smithi033.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-05-20T19:48:45.194 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-05-20T19:48:45.205 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-05-20T19:48:45.205 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-05-20T19:48:45.205 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-05-20T19:48:45.205 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-05-20T19:48:45.205 INFO:teuthology.orchestra.run.smithi033.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-05-20T19:48:45.205 INFO:teuthology.orchestra.run.smithi033.stdout: ceph x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 6.6 k 2024-05-20T19:48:45.205 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-base x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 5.4 M 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-fuse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 890 k 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 145 k 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 1.5 M 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-cephadm noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 143 k 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-dashboard noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 3.5 M 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 7.4 M 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-rook noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 49 k 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-radosgw x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 11 M 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-test x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 49 M 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-volume noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 273 k 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: cephadm noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 764 k 2024-05-20T19:48:45.206 INFO:teuthology.orchestra.run.smithi033.stdout: cephfs-mirror x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 225 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: cephfs-shell x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 27 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: cephfs-top noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 18 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-libelf-devel x86_64 0.191-4.el9 appstream 22 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs-devel x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 32 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 747 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: librados-devel x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 127 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: libtool x86_64 2.4.6-45.el9 appstream 582 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: openssl-devel x86_64 1:3.2.1-1.el9 appstream 4.4 M 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-05-20T19:48:45.207 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cephfs x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 161 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rados x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 320 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rbd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 300 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rgw x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 100 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-fuse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 85 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-mirror x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.1 M 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-nbd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 171 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout:Upgrading: 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager x86_64 1:1.47.90-1.el9 baseos 2.3 M 2024-05-20T19:48:45.208 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager-libnm x86_64 1:1.47.90-1.el9 baseos 1.8 M 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager-team x86_64 1:1.47.90-1.el9 baseos 41 k 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager-tui x86_64 1:1.47.90-1.el9 baseos 248 k 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-debuginfod-client x86_64 0.191-4.el9 baseos 37 k 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-libelf x86_64 0.191-4.el9 baseos 207 k 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-libs x86_64 0.191-4.el9 baseos 260 k 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: librados2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.4 M 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: librbd1 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.1 M 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: openssl x86_64 1:3.2.1-1.el9 baseos 1.3 M 2024-05-20T19:48:45.209 INFO:teuthology.orchestra.run.smithi033.stdout: openssl-libs x86_64 1:3.2.1-1.el9 baseos 2.4 M 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-common x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 21 M 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-grafana-dashboards noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 34 k 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mds x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 2.4 M 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-modules-core noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 274 k 2024-05-20T19:48:45.210 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mon x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 4.7 M 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-osd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 17 M 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-prometheus-alerts noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 16 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-selinux x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 25 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-05-20T19:48:45.211 INFO:teuthology.orchestra.run.smithi033.stdout: libcephsqlite x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 163 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: libradosstriper1 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 501 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: librgw2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 5.4 M 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-05-20T19:48:45.212 INFO:teuthology.orchestra.run.smithi033.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-05-20T19:48:45.213 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-argparse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 45 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-common x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 136 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-colorama noarch 0.4.6-3.el9 appstream 61 k 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-05-20T19:48:45.214 INFO:teuthology.orchestra.run.smithi033.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-05-20T19:48:45.215 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-05-20T19:48:45.216 INFO:teuthology.orchestra.run.smithi033.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-05-20T19:48:45.217 INFO:teuthology.orchestra.run.smithi033.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-05-20T19:48:45.218 INFO:teuthology.orchestra.run.smithi033.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout:Installing weak dependencies: 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout:Install 148 Packages 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout:Upgrade 11 Packages 2024-05-20T19:48:45.219 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:48:45.220 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 227 M 2024-05-20T19:48:45.220 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-05-20T19:48:45.893 INFO:teuthology.orchestra.run.smithi154.stdout:(7/159): ceph-mon-19.0.0-3830.g4ebd1d53.el9.x86 5.1 MB/s | 4.7 MB 00:00 2024-05-20T19:48:45.939 INFO:teuthology.orchestra.run.smithi033.stdout:(1/159): ceph-19.0.0-3830.g4ebd1d53.el9.x86_64. 14 kB/s | 6.6 kB 00:00 2024-05-20T19:48:46.091 INFO:teuthology.orchestra.run.smithi154.stdout:(8/159): ceph-common-19.0.0-3830.g4ebd1d53.el9. 7.6 MB/s | 21 MB 00:02 2024-05-20T19:48:46.166 INFO:teuthology.orchestra.run.smithi154.stdout:(9/159): ceph-selinux-19.0.0-3830.g4ebd1d53.el9 336 kB/s | 25 kB 00:00 2024-05-20T19:48:46.740 INFO:teuthology.orchestra.run.smithi028.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-05-20T19:48:46.742 INFO:teuthology.orchestra.run.smithi028.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-05-20T19:48:46.763 INFO:teuthology.orchestra.run.smithi028.stdout:Package NetworkManager-1:1.47.4-1.el9.x86_64 is already installed. 2024-05-20T19:48:46.764 INFO:teuthology.orchestra.run.smithi028.stdout:Package iproute-6.2.0-5.el9.x86_64 is already installed. 2024-05-20T19:48:46.766 INFO:teuthology.orchestra.run.smithi028.stdout:Package util-linux-2.37.4-18.el9.x86_64 is already installed. 2024-05-20T19:48:46.767 INFO:teuthology.orchestra.run.smithi028.stdout:Package libacl-devel-2.3.1-4.el9.x86_64 is already installed. 2024-05-20T19:48:46.770 INFO:teuthology.orchestra.run.smithi028.stdout:Package libattr-devel-2.5.1-3.el9.x86_64 is already installed. 2024-05-20T19:48:46.773 INFO:teuthology.orchestra.run.smithi028.stdout:Package xfsdump-3.1.12-4.el9.x86_64 is already installed. 2024-05-20T19:48:46.774 INFO:teuthology.orchestra.run.smithi028.stdout:Package xfsprogs-6.3.0-1.el9.x86_64 is already installed. 2024-05-20T19:48:46.853 INFO:teuthology.orchestra.run.smithi028.stdout:Dependencies resolved. 2024-05-20T19:48:46.875 INFO:teuthology.orchestra.run.smithi154.stdout:(10/159): ceph-osd-19.0.0-3830.g4ebd1d53.el9.x8 9.1 MB/s | 17 MB 00:01 2024-05-20T19:48:46.876 INFO:teuthology.orchestra.run.smithi028.stdout:======================================================================================= 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: Package Arch Version Repository Size 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout:======================================================================================= 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout:Installing: 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: bison x86_64 3.7.4-5.el9 appstream 943 k 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: ceph x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 6.6 k 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-base x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 5.4 M 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-fuse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 890 k 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-immutable-object-cache x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 145 k 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 1.5 M 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-cephadm noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 143 k 2024-05-20T19:48:46.877 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-dashboard noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 3.5 M 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-diskprediction-local noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 7.4 M 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-rook noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 49 k 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-radosgw x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 11 M 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-test x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 49 M 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-volume noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 273 k 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: cephadm noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 764 k 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: cephfs-mirror x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 225 k 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: cephfs-shell x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 27 k 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: cephfs-top noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 18 k 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-libelf-devel x86_64 0.191-4.el9 appstream 22 k 2024-05-20T19:48:46.878 INFO:teuthology.orchestra.run.smithi028.stdout: flex x86_64 2.6.4-9.el9 appstream 313 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: libaio-devel x86_64 0.3.111-13.el9 appstream 11 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: libcephfs-devel x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 32 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: libcephfs2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 747 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: librados-devel x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 127 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: libtool x86_64 2.4.6-45.el9 appstream 582 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: libuuid-devel x86_64 2.37.4-18.el9 appstream 25 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: openssl-devel x86_64 1:3.2.1-1.el9 appstream 4.4 M 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql x86_64 13.14-1.el9 appstream 1.6 M 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql-contrib x86_64 13.14-1.el9 appstream 911 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql-server x86_64 13.14-1.el9 appstream 5.8 M 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cephfs x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 161 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rados x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 320 k 2024-05-20T19:48:46.879 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rbd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 300 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rgw x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 100 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: rbd-fuse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 85 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: rbd-mirror x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.1 M 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: rbd-nbd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 171 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: xfsprogs-devel x86_64 6.3.0-1.el9 appstream 94 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout:Upgrading: 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager x86_64 1:1.47.90-1.el9 baseos 2.3 M 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager-libnm x86_64 1:1.47.90-1.el9 baseos 1.8 M 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager-team x86_64 1:1.47.90-1.el9 baseos 41 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager-tui x86_64 1:1.47.90-1.el9 baseos 248 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-debuginfod-client x86_64 0.191-4.el9 baseos 37 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-libelf x86_64 0.191-4.el9 baseos 207 k 2024-05-20T19:48:46.880 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-libs x86_64 0.191-4.el9 baseos 260 k 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: librados2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.4 M 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: librbd1 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 3.1 M 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: openssl x86_64 1:3.2.1-1.el9 baseos 1.3 M 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: openssl-libs x86_64 1:3.2.1-1.el9 baseos 2.4 M 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout:Installing dependencies: 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: automake noarch 1.16.2-8.el9 appstream 689 k 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-common x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 21 M 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-grafana-dashboards noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 34 k 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mds x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 2.4 M 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-modules-core noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 274 k 2024-05-20T19:48:46.881 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mon x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 4.7 M 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-osd x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 17 M 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-prometheus-alerts noarch 2:19.0.0-3830.g4ebd1d53.el9 ceph-noarch 16 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-selinux x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 25 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: libcephsqlite x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 163 k 2024-05-20T19:48:46.882 INFO:teuthology.orchestra.run.smithi028.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: libradosstriper1 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 501 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: librgw2 x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 5.4 M 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: libzstd-devel x86_64 1.5.1-2.el9 appstream 47 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-05-20T19:48:46.883 INFO:teuthology.orchestra.run.smithi028.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql-private-libs x86_64 13.14-1.el9 appstream 133 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ceph-argparse x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 45 k 2024-05-20T19:48:46.884 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ceph-common x86_64 2:19.0.0-3830.g4ebd1d53.el9 ceph 136 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cmd2 noarch 2.4.2-2.el9 epel 303 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-colorama noarch 0.4.6-3.el9 appstream 61 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-05-20T19:48:46.885 INFO:teuthology.orchestra.run.smithi028.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-05-20T19:48:46.886 INFO:teuthology.orchestra.run.smithi028.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-05-20T19:48:46.887 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyperclip noarch 1.8.2-2.el9 epel 28 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-05-20T19:48:46.888 INFO:teuthology.orchestra.run.smithi028.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: python3-wcwidth noarch 0.2.5-8.el9 appstream 44 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: uuid x86_64 1.6.2-55.el9 appstream 58 k 2024-05-20T19:48:46.889 INFO:teuthology.orchestra.run.smithi028.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout: zlib-devel x86_64 1.2.11-41.el9 appstream 45 k 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout:Installing weak dependencies: 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout:Transaction Summary 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout:======================================================================================= 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout:Install 148 Packages 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout:Upgrade 11 Packages 2024-05-20T19:48:46.890 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:48:46.891 INFO:teuthology.orchestra.run.smithi028.stdout:Total download size: 227 M 2024-05-20T19:48:46.891 INFO:teuthology.orchestra.run.smithi028.stdout:Downloading Packages: 2024-05-20T19:48:46.915 INFO:teuthology.orchestra.run.smithi033.stdout:(2/159): ceph-fuse-19.0.0-3830.g4ebd1d53.el9.x8 913 kB/s | 890 kB 00:00 2024-05-20T19:48:46.958 INFO:teuthology.orchestra.run.smithi154.stdout:(11/159): cephfs-mirror-19.0.0-3830.g4ebd1d53.e 2.6 MB/s | 225 kB 00:00 2024-05-20T19:48:47.023 INFO:teuthology.orchestra.run.smithi033.stdout:(3/159): ceph-immutable-object-cache-19.0.0-383 1.3 MB/s | 145 kB 00:00 2024-05-20T19:48:47.025 INFO:teuthology.orchestra.run.smithi154.stdout:(12/159): cephfs-shell-19.0.0-3830.g4ebd1d53.el 406 kB/s | 27 kB 00:00 2024-05-20T19:48:47.118 INFO:teuthology.orchestra.run.smithi154.stdout:(13/159): libcephfs-devel-19.0.0-3830.g4ebd1d53 346 kB/s | 32 kB 00:00 2024-05-20T19:48:47.276 INFO:teuthology.orchestra.run.smithi154.stdout:(14/159): libcephfs2-19.0.0-3830.g4ebd1d53.el9. 4.6 MB/s | 747 kB 00:00 2024-05-20T19:48:47.351 INFO:teuthology.orchestra.run.smithi154.stdout:(15/159): libcephsqlite-19.0.0-3830.g4ebd1d53.e 2.1 MB/s | 163 kB 00:00 2024-05-20T19:48:47.451 INFO:teuthology.orchestra.run.smithi154.stdout:(16/159): librados-devel-19.0.0-3830.g4ebd1d53. 1.2 MB/s | 127 kB 00:00 2024-05-20T19:48:47.551 INFO:teuthology.orchestra.run.smithi154.stdout:(17/159): libradosstriper1-19.0.0-3830.g4ebd1d5 4.9 MB/s | 501 kB 00:00 2024-05-20T19:48:48.115 INFO:teuthology.orchestra.run.smithi033.stdout:(4/159): ceph-base-19.0.0-3830.g4ebd1d53.el9.x8 2.0 MB/s | 5.4 MB 00:02 2024-05-20T19:48:48.123 INFO:teuthology.orchestra.run.smithi028.stdout:(1/159): ceph-19.0.0-3830.g4ebd1d53.el9.x86_64. 11 kB/s | 6.6 kB 00:00 2024-05-20T19:48:48.169 INFO:teuthology.orchestra.run.smithi154.stdout:(18/159): librgw2-19.0.0-3830.g4ebd1d53.el9.x86 8.7 MB/s | 5.4 MB 00:00 2024-05-20T19:48:48.260 INFO:teuthology.orchestra.run.smithi154.stdout:(19/159): python3-ceph-argparse-19.0.0-3830.g4e 493 kB/s | 45 kB 00:00 2024-05-20T19:48:48.298 INFO:teuthology.orchestra.run.smithi033.stdout:(5/159): ceph-mds-19.0.0-3830.g4ebd1d53.el9.x86 1.9 MB/s | 2.4 MB 00:01 2024-05-20T19:48:48.335 INFO:teuthology.orchestra.run.smithi154.stdout:(20/159): python3-ceph-common-19.0.0-3830.g4ebd 1.8 MB/s | 136 kB 00:00 2024-05-20T19:48:48.428 INFO:teuthology.orchestra.run.smithi154.stdout:(21/159): python3-cephfs-19.0.0-3830.g4ebd1d53. 1.7 MB/s | 161 kB 00:00 2024-05-20T19:48:48.490 INFO:teuthology.orchestra.run.smithi033.stdout:(6/159): ceph-mgr-19.0.0-3830.g4ebd1d53.el9.x86 4.0 MB/s | 1.5 MB 00:00 2024-05-20T19:48:48.536 INFO:teuthology.orchestra.run.smithi154.stdout:(22/159): python3-rados-19.0.0-3830.g4ebd1d53.e 2.9 MB/s | 320 kB 00:00 2024-05-20T19:48:48.628 INFO:teuthology.orchestra.run.smithi154.stdout:(23/159): python3-rbd-19.0.0-3830.g4ebd1d53.el9 3.2 MB/s | 300 kB 00:00 2024-05-20T19:48:48.712 INFO:teuthology.orchestra.run.smithi154.stdout:(24/159): python3-rgw-19.0.0-3830.g4ebd1d53.el9 1.2 MB/s | 100 kB 00:00 2024-05-20T19:48:48.796 INFO:teuthology.orchestra.run.smithi154.stdout:(25/159): rbd-fuse-19.0.0-3830.g4ebd1d53.el9.x8 1.0 MB/s | 85 kB 00:00 2024-05-20T19:48:48.856 INFO:teuthology.orchestra.run.smithi028.stdout:(2/159): ceph-fuse-19.0.0-3830.g4ebd1d53.el9.x8 1.2 MB/s | 890 kB 00:00 2024-05-20T19:48:49.074 INFO:teuthology.orchestra.run.smithi028.stdout:(3/159): ceph-immutable-object-cache-19.0.0-383 669 kB/s | 145 kB 00:00 2024-05-20T19:48:49.171 INFO:teuthology.orchestra.run.smithi154.stdout:(26/159): ceph-radosgw-19.0.0-3830.g4ebd1d53.el 3.2 MB/s | 11 MB 00:03 2024-05-20T19:48:49.288 INFO:teuthology.orchestra.run.smithi154.stdout:(27/159): rbd-nbd-19.0.0-3830.g4ebd1d53.el9.x86 1.4 MB/s | 171 kB 00:00 2024-05-20T19:48:49.405 INFO:teuthology.orchestra.run.smithi154.stdout:(28/159): ceph-grafana-dashboards-19.0.0-3830.g 294 kB/s | 34 kB 00:00 2024-05-20T19:48:49.514 INFO:teuthology.orchestra.run.smithi154.stdout:(29/159): ceph-mgr-cephadm-19.0.0-3830.g4ebd1d5 1.3 MB/s | 143 kB 00:00 2024-05-20T19:48:49.597 INFO:teuthology.orchestra.run.smithi154.stdout:(30/159): rbd-mirror-19.0.0-3830.g4ebd1d53.el9. 3.9 MB/s | 3.1 MB 00:00 2024-05-20T19:48:49.666 INFO:teuthology.orchestra.run.smithi033.stdout:(7/159): ceph-mon-19.0.0-3830.g4ebd1d53.el9.x86 3.4 MB/s | 4.7 MB 00:01 2024-05-20T19:48:49.849 INFO:teuthology.orchestra.run.smithi028.stdout:(4/159): ceph-mds-19.0.0-3830.g4ebd1d53.el9.x86 3.1 MB/s | 2.4 MB 00:00 2024-05-20T19:48:49.966 INFO:teuthology.orchestra.run.smithi028.stdout:(5/159): ceph-base-19.0.0-3830.g4ebd1d53.el9.x8 2.2 MB/s | 5.4 MB 00:02 2024-05-20T19:48:50.358 INFO:teuthology.orchestra.run.smithi028.stdout:(6/159): ceph-mgr-19.0.0-3830.g4ebd1d53.el9.x86 3.0 MB/s | 1.5 MB 00:00 2024-05-20T19:48:50.614 INFO:teuthology.orchestra.run.smithi154.stdout:(31/159): ceph-mgr-dashboard-19.0.0-3830.g4ebd1 3.2 MB/s | 3.5 MB 00:01 2024-05-20T19:48:50.691 INFO:teuthology.orchestra.run.smithi033.stdout:(8/159): ceph-common-19.0.0-3830.g4ebd1d53.el9. 4.0 MB/s | 21 MB 00:05 2024-05-20T19:48:50.740 INFO:teuthology.orchestra.run.smithi154.stdout:(32/159): ceph-mgr-modules-core-19.0.0-3830.g4e 2.2 MB/s | 274 kB 00:00 2024-05-20T19:48:50.808 INFO:teuthology.orchestra.run.smithi033.stdout:(9/159): ceph-selinux-19.0.0-3830.g4ebd1d53.el9 216 kB/s | 25 kB 00:00 2024-05-20T19:48:50.890 INFO:teuthology.orchestra.run.smithi154.stdout:(33/159): ceph-mgr-rook-19.0.0-3830.g4ebd1d53.e 329 kB/s | 49 kB 00:00 2024-05-20T19:48:51.057 INFO:teuthology.orchestra.run.smithi154.stdout:(34/159): ceph-prometheus-alerts-19.0.0-3830.g4 99 kB/s | 16 kB 00:00 2024-05-20T19:48:51.190 INFO:teuthology.orchestra.run.smithi154.stdout:(35/159): ceph-volume-19.0.0-3830.g4ebd1d53.el9 2.0 MB/s | 273 kB 00:00 2024-05-20T19:48:51.441 INFO:teuthology.orchestra.run.smithi028.stdout:(7/159): ceph-mon-19.0.0-3830.g4ebd1d53.el9.x86 3.2 MB/s | 4.7 MB 00:01 2024-05-20T19:48:51.466 INFO:teuthology.orchestra.run.smithi154.stdout:(36/159): cephadm-19.0.0-3830.g4ebd1d53.el9.noa 2.7 MB/s | 764 kB 00:00 2024-05-20T19:48:51.574 INFO:teuthology.orchestra.run.smithi154.stdout:(37/159): cephfs-top-19.0.0-3830.g4ebd1d53.el9. 168 kB/s | 18 kB 00:00 2024-05-20T19:48:51.683 INFO:teuthology.orchestra.run.smithi154.stdout:(38/159): fuse-2.9.9-15.el9.x86_64.rpm 742 kB/s | 80 kB 00:00 2024-05-20T19:48:51.792 INFO:teuthology.orchestra.run.smithi154.stdout:(39/159): libgfortran-11.4.1-3.el9.x86_64.rpm 7.3 MB/s | 803 kB 00:00 2024-05-20T19:48:51.834 INFO:teuthology.orchestra.run.smithi154.stdout:(40/159): libquadmath-11.4.1-3.el9.x86_64.rpm 4.6 MB/s | 195 kB 00:00 2024-05-20T19:48:51.876 INFO:teuthology.orchestra.run.smithi154.stdout:(41/159): python3-cffi-1.14.5-5.el9.x86_64.rpm 6.0 MB/s | 253 kB 00:00 2024-05-20T19:48:51.951 INFO:teuthology.orchestra.run.smithi154.stdout:(42/159): python3-cryptography-36.0.1-4.el9.x86 16 MB/s | 1.2 MB 00:00 2024-05-20T19:48:51.993 INFO:teuthology.orchestra.run.smithi154.stdout:(43/159): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-05-20T19:48:52.035 INFO:teuthology.orchestra.run.smithi154.stdout:(44/159): python3-pycparser-2.20-6.el9.noarch.r 3.2 MB/s | 135 kB 00:00 2024-05-20T19:48:52.077 INFO:teuthology.orchestra.run.smithi154.stdout:(45/159): python3-requests-2.25.1-8.el9.noarch. 2.9 MB/s | 125 kB 00:00 2024-05-20T19:48:52.119 INFO:teuthology.orchestra.run.smithi154.stdout:(46/159): python3-urllib3-1.26.5-5.el9.noarch.r 5.0 MB/s | 215 kB 00:00 2024-05-20T19:48:52.286 INFO:teuthology.orchestra.run.smithi154.stdout:(47/159): ceph-mgr-diskprediction-local-19.0.0- 2.7 MB/s | 7.4 MB 00:02 2024-05-20T19:48:52.311 INFO:teuthology.orchestra.run.smithi154.stdout:(48/159): automake-1.16.2-8.el9.noarch.rpm 3.5 MB/s | 689 kB 00:00 2024-05-20T19:48:52.378 INFO:teuthology.orchestra.run.smithi154.stdout:(49/159): boost-program-options-1.75.0-8.el9.x8 1.6 MB/s | 107 kB 00:00 2024-05-20T19:48:52.430 INFO:teuthology.orchestra.run.smithi154.stdout:(50/159): bison-3.7.4-5.el9.x86_64.rpm 6.5 MB/s | 943 kB 00:00 2024-05-20T19:48:52.431 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:52.488 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:52.521 INFO:teuthology.orchestra.run.smithi154.stdout:(51/159): flex-2.6.4-9.el9.x86_64.rpm 3.3 MB/s | 313 kB 00:00 2024-05-20T19:48:52.562 INFO:teuthology.orchestra.run.smithi154.stdout:(52/159): flexiblas-3.0.4-8.el9.x86_64.rpm 788 kB/s | 33 kB 00:00 2024-05-20T19:48:52.713 INFO:teuthology.orchestra.run.smithi154.stdout:(53/159): flexiblas-netlib-3.0.4-8.el9.x86_64.r 20 MB/s | 3.0 MB 00:00 2024-05-20T19:48:52.713 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:52.763 INFO:teuthology.orchestra.run.smithi154.stdout:(54/159): flexiblas-openblas-openmp-3.0.4-8.el9 363 kB/s | 18 kB 00:00 2024-05-20T19:48:52.796 INFO:teuthology.orchestra.run.smithi154.stdout:(55/159): libaio-devel-0.3.111-13.el9.x86_64.rp 336 kB/s | 11 kB 00:00 2024-05-20T19:48:52.880 INFO:teuthology.orchestra.run.smithi154.stdout:(56/159): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.9 MB/s | 160 kB 00:00 2024-05-20T19:48:52.931 INFO:teuthology.orchestra.run.smithi154.stdout:(57/159): librabbitmq-0.11.0-7.el9.x86_64.rpm 902 kB/s | 45 kB 00:00 2024-05-20T19:48:53.022 INFO:teuthology.orchestra.run.smithi154.stdout:(58/159): librdkafka-1.6.1-102.el9.x86_64.rpm 7.1 MB/s | 662 kB 00:00 2024-05-20T19:48:53.115 INFO:teuthology.orchestra.run.smithi154.stdout:(59/159): libtool-2.4.6-45.el9.x86_64.rpm 6.2 MB/s | 582 kB 00:00 2024-05-20T19:48:53.118 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:53.135 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:48:53.165 INFO:teuthology.orchestra.run.smithi154.stdout:(60/159): libuuid-devel-2.37.4-18.el9.x86_64.rp 509 kB/s | 25 kB 00:00 2024-05-20T19:48:53.232 INFO:teuthology.orchestra.run.smithi154.stdout:(61/159): libzstd-devel-1.5.1-2.el9.x86_64.rpm 713 kB/s | 47 kB 00:00 2024-05-20T19:48:53.274 INFO:teuthology.orchestra.run.smithi154.stdout:(62/159): elfutils-libelf-devel-0.191-4.el9.x86 24 kB/s | 22 kB 00:00 2024-05-20T19:48:53.324 INFO:teuthology.orchestra.run.smithi154.stdout:(63/159): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.2 MB/s | 292 kB 00:00 2024-05-20T19:48:53.366 INFO:teuthology.orchestra.run.smithi154.stdout:(64/159): lua-5.4.4-4.el9.x86_64.rpm 2.0 MB/s | 188 kB 00:00 2024-05-20T19:48:53.391 INFO:teuthology.orchestra.run.smithi154.stdout:(65/159): openblas-0.3.21-2.el9.x86_64.rpm 526 kB/s | 35 kB 00:00 2024-05-20T19:48:53.583 INFO:teuthology.orchestra.run.smithi154.stdout:(66/159): openblas-openmp-0.3.21-2.el9.x86_64.r 22 MB/s | 4.7 MB 00:00 2024-05-20T19:48:53.651 INFO:teuthology.orchestra.run.smithi154.stdout:(67/159): openssl-devel-3.2.1-1.el9.x86_64.rpm 17 MB/s | 4.4 MB 00:00 2024-05-20T19:48:53.751 INFO:teuthology.orchestra.run.smithi154.stdout:(68/159): postgresql-13.14-1.el9.x86_64.rpm 9.5 MB/s | 1.6 MB 00:00 2024-05-20T19:48:53.776 INFO:teuthology.orchestra.run.smithi154.stdout:(69/159): postgresql-contrib-13.14-1.el9.x86_64 7.1 MB/s | 911 kB 00:00 2024-05-20T19:48:53.835 INFO:teuthology.orchestra.run.smithi154.stdout:(70/159): postgresql-private-libs-13.14-1.el9.x 1.6 MB/s | 133 kB 00:00 2024-05-20T19:48:53.943 INFO:teuthology.orchestra.run.smithi154.stdout:(71/159): protobuf-3.14.0-13.el9.x86_64.rpm 9.3 MB/s | 1.0 MB 00:00 2024-05-20T19:48:54.010 INFO:teuthology.orchestra.run.smithi154.stdout:(72/159): python3-attrs-20.3.0-7.el9.noarch.rpm 1.3 MB/s | 87 kB 00:00 2024-05-20T19:48:54.103 INFO:teuthology.orchestra.run.smithi154.stdout:(73/159): postgresql-server-13.14-1.el9.x86_64. 18 MB/s | 5.8 MB 00:00 2024-05-20T19:48:54.120 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:48:54.124 INFO:teuthology.orchestra.run.smithi033.stdout:(10/159): ceph-radosgw-19.0.0-3830.g4ebd1d53.el 2.4 MB/s | 11 MB 00:04 2024-05-20T19:48:54.174 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:48:54.254 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:48:54.308 INFO:teuthology.orchestra.run.smithi028.stdout:(8/159): ceph-osd-19.0.0-3830.g4ebd1d53.el9.x86 4.3 MB/s | 17 MB 00:03 2024-05-20T19:48:54.336 INFO:teuthology.orchestra.run.smithi154.stdout:(74/159): python3-babel-2.9.1-2.el9.noarch.rpm 18 MB/s | 6.0 MB 00:00 2024-05-20T19:48:54.336 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:48:54.353 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 199.223.246.113) 2024-05-20T19:48:54.403 INFO:teuthology.orchestra.run.smithi154.stdout:(75/159): python3-colorama-0.4.6-3.el9.noarch.r 204 kB/s | 61 kB 00:00 2024-05-20T19:48:54.436 INFO:teuthology.orchestra.run.smithi154.stdout:(76/159): python3-devel-3.9.18-3.el9.x86_64.rpm 2.4 MB/s | 244 kB 00:00 2024-05-20T19:48:54.444 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:48:54.466 INFO:teuthology.orchestra.run.smithi033.stdout:(11/159): ceph-osd-19.0.0-3830.g4ebd1d53.el9.x8 2.8 MB/s | 17 MB 00:05 2024-05-20T19:48:54.475 INFO:teuthology.orchestra.run.smithi028.stdout:(9/159): ceph-radosgw-19.0.0-3830.g4ebd1d53.el9 3.5 MB/s | 11 MB 00:03 2024-05-20T19:48:54.492 INFO:teuthology.orchestra.run.smithi033.stdout:(12/159): cephfs-mirror-19.0.0-3830.g4ebd1d53.e 612 kB/s | 225 kB 00:00 2024-05-20T19:48:54.492 INFO:teuthology.orchestra.run.smithi028.stdout:(10/159): ceph-selinux-19.0.0-3830.g4ebd1d53.el 138 kB/s | 25 kB 00:00 2024-05-20T19:48:54.503 INFO:teuthology.orchestra.run.smithi154.stdout:(77/159): python3-jsonpatch-1.21-16.el9.noarch. 384 kB/s | 26 kB 00:00 2024-05-20T19:48:54.513 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:48:54.553 INFO:teuthology.orchestra.run.smithi154.stdout:(78/159): python3-jsonpointer-2.0-4.el9.noarch. 389 kB/s | 19 kB 00:00 2024-05-20T19:48:54.567 INFO:teuthology.orchestra.run.smithi033.stdout:(13/159): cephfs-shell-19.0.0-3830.g4ebd1d53.el 269 kB/s | 27 kB 00:00 2024-05-20T19:48:54.592 INFO:teuthology.orchestra.run.smithi033.stdout:(14/159): libcephfs-devel-19.0.0-3830.g4ebd1d53 317 kB/s | 32 kB 00:00 2024-05-20T19:48:54.593 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:48:54.620 INFO:teuthology.orchestra.run.smithi154.stdout:(79/159): python3-mako-1.1.4-6.el9.noarch.rpm 2.5 MB/s | 172 kB 00:00 2024-05-20T19:48:54.671 INFO:teuthology.orchestra.run.smithi154.stdout:(80/159): python3-markupsafe-1.1.1-12.el9.x86_6 693 kB/s | 35 kB 00:00 2024-05-20T19:48:54.673 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:48:54.689 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 199.223.246.113) 2024-05-20T19:48:54.750 INFO:teuthology.orchestra.run.smithi028.stdout:(11/159): ceph-common-19.0.0-3830.g4ebd1d53.el9 2.9 MB/s | 21 MB 00:07 2024-05-20T19:48:54.754 INFO:teuthology.orchestra.run.smithi154.stdout:(81/159): python3-jinja2-2.11.3-6.el9.noarch.rp 706 kB/s | 248 kB 00:00 2024-05-20T19:48:54.784 INFO:teuthology.orchestra.run.smithi028.stdout:(12/159): cephfs-mirror-19.0.0-3830.g4ebd1d53.e 771 kB/s | 225 kB 00:00 2024-05-20T19:48:54.784 INFO:teuthology.orchestra.run.smithi033.stdout:(15/159): libcephsqlite-19.0.0-3830.g4ebd1d53.e 852 kB/s | 163 kB 00:00 2024-05-20T19:48:54.826 INFO:teuthology.orchestra.run.smithi033.stdout:(16/159): libcephfs2-19.0.0-3830.g4ebd1d53.el9. 2.8 MB/s | 747 kB 00:00 2024-05-20T19:48:54.830 INFO:teuthology.orchestra.run.smithi154.stdout:(82/159): python3-numpy-f2py-1.20.1-5.el9.x86_6 3.3 MB/s | 250 kB 00:00 2024-05-20T19:48:54.881 INFO:teuthology.orchestra.run.smithi028.stdout:(13/159): cephfs-shell-19.0.0-3830.g4ebd1d53.el 212 kB/s | 27 kB 00:00 2024-05-20T19:48:54.884 INFO:teuthology.orchestra.run.smithi033.stdout:(17/159): librados-devel-19.0.0-3830.g4ebd1d53. 1.2 MB/s | 127 kB 00:00 2024-05-20T19:48:54.914 INFO:teuthology.orchestra.run.smithi154.stdout:(83/159): python3-numpy-1.20.1-5.el9.x86_64.rpm 21 MB/s | 5.1 MB 00:00 2024-05-20T19:48:54.964 INFO:teuthology.orchestra.run.smithi154.stdout:(84/159): python3-oauthlib-3.1.1-5.el9.noarch.r 1.6 MB/s | 222 kB 00:00 2024-05-20T19:48:55.006 INFO:teuthology.orchestra.run.smithi154.stdout:(85/159): python3-packaging-20.9-5.el9.noarch.r 838 kB/s | 77 kB 00:00 2024-05-20T19:48:55.018 INFO:teuthology.orchestra.run.smithi028.stdout:(14/159): libcephfs-devel-19.0.0-3830.g4ebd1d53 136 kB/s | 32 kB 00:00 2024-05-20T19:48:55.018 INFO:teuthology.orchestra.run.smithi033.stdout:(18/159): libradosstriper1-19.0.0-3830.g4ebd1d5 2.5 MB/s | 501 kB 00:00 2024-05-20T19:48:55.065 INFO:teuthology.orchestra.run.smithi154.stdout:(86/159): python3-prettytable-0.7.2-27.el9.noar 420 kB/s | 42 kB 00:00 2024-05-20T19:48:55.115 INFO:teuthology.orchestra.run.smithi154.stdout:(87/159): python3-protobuf-3.14.0-13.el9.noarch 2.4 MB/s | 269 kB 00:00 2024-05-20T19:48:55.126 INFO:teuthology.orchestra.run.smithi028.stdout:(15/159): libcephsqlite-19.0.0-3830.g4ebd1d53.e 1.5 MB/s | 163 kB 00:00 2024-05-20T19:48:55.157 INFO:teuthology.orchestra.run.smithi154.stdout:(88/159): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.7 MB/s | 159 kB 00:00 2024-05-20T19:48:55.169 INFO:teuthology.orchestra.run.smithi033.stdout:(19/159): python3-ceph-argparse-19.0.0-3830.g4e 300 kB/s | 45 kB 00:00 2024-05-20T19:48:55.193 INFO:teuthology.orchestra.run.smithi028.stdout:(16/159): libcephfs2-19.0.0-3830.g4ebd1d53.el9. 2.3 MB/s | 747 kB 00:00 2024-05-20T19:48:55.216 INFO:teuthology.orchestra.run.smithi154.stdout:(89/159): python3-pyasn1-modules-0.4.8-6.el9.no 2.7 MB/s | 279 kB 00:00 2024-05-20T19:48:55.235 INFO:teuthology.orchestra.run.smithi028.stdout:(17/159): librados-devel-19.0.0-3830.g4ebd1d53. 1.1 MB/s | 127 kB 00:00 2024-05-20T19:48:55.241 INFO:teuthology.orchestra.run.smithi154.stdout:(90/159): python3-pytz-2021.1-5.el9.noarch.rpm 609 kB/s | 51 kB 00:00 2024-05-20T19:48:55.269 INFO:teuthology.orchestra.run.smithi033.stdout:(20/159): python3-ceph-common-19.0.0-3830.g4ebd 1.3 MB/s | 136 kB 00:00 2024-05-20T19:48:55.291 INFO:teuthology.orchestra.run.smithi154.stdout:(91/159): python3-requests-oauthlib-1.3.0-12.el 712 kB/s | 54 kB 00:00 2024-05-20T19:48:55.350 INFO:teuthology.orchestra.run.smithi154.stdout:(92/159): python3-toml-0.10.2-6.el9.noarch.rpm 712 kB/s | 42 kB 00:00 2024-05-20T19:48:55.378 INFO:teuthology.orchestra.run.smithi033.stdout:(21/159): python3-cephfs-19.0.0-3830.g4ebd1d53. 1.5 MB/s | 161 kB 00:00 2024-05-20T19:48:55.385 INFO:teuthology.orchestra.run.smithi028.stdout:(18/159): libradosstriper1-19.0.0-3830.g4ebd1d5 2.6 MB/s | 501 kB 00:00 2024-05-20T19:48:55.408 INFO:teuthology.orchestra.run.smithi154.stdout:(93/159): python3-wcwidth-0.2.5-8.el9.noarch.rp 759 kB/s | 44 kB 00:00 2024-05-20T19:48:55.523 INFO:teuthology.orchestra.run.smithi028.stdout:(19/159): python3-ceph-argparse-19.0.0-3830.g4e 331 kB/s | 45 kB 00:00 2024-05-20T19:48:55.553 INFO:teuthology.orchestra.run.smithi154.stdout:(94/159): qatlib-24.02.0-1.el9.x86_64.rpm 1.5 MB/s | 221 kB 00:00 2024-05-20T19:48:55.569 INFO:teuthology.orchestra.run.smithi033.stdout:(22/159): python3-rados-19.0.0-3830.g4ebd1d53.e 1.6 MB/s | 320 kB 00:00 2024-05-20T19:48:55.609 INFO:teuthology.orchestra.run.smithi154.stdout:(95/159): qatlib-service-24.02.0-1.el9.x86_64.r 637 kB/s | 36 kB 00:00 2024-05-20T19:48:55.636 INFO:teuthology.orchestra.run.smithi028.stdout:(20/159): python3-ceph-common-19.0.0-3830.g4ebd 1.2 MB/s | 136 kB 00:00 2024-05-20T19:48:55.685 INFO:teuthology.orchestra.run.smithi154.stdout:(96/159): qatzip-libs-1.2.0-1.el9.x86_64.rpm 632 kB/s | 47 kB 00:00 2024-05-20T19:48:55.761 INFO:teuthology.orchestra.run.smithi033.stdout:(23/159): python3-rbd-19.0.0-3830.g4ebd1d53.el9 1.5 MB/s | 300 kB 00:00 2024-05-20T19:48:55.765 INFO:teuthology.orchestra.run.smithi028.stdout:(21/159): python3-cephfs-19.0.0-3830.g4ebd1d53. 1.2 MB/s | 161 kB 00:00 2024-05-20T19:48:55.860 INFO:teuthology.orchestra.run.smithi154.stdout:(97/159): python3-scipy-1.6.2-8.el9.x86_64.rpm 26 MB/s | 16 MB 00:00 2024-05-20T19:48:55.862 INFO:teuthology.orchestra.run.smithi033.stdout:(24/159): python3-rgw-19.0.0-3830.g4ebd1d53.el9 996 kB/s | 100 kB 00:00 2024-05-20T19:48:55.895 INFO:teuthology.orchestra.run.smithi028.stdout:(22/159): python3-rados-19.0.0-3830.g4ebd1d53.e 2.4 MB/s | 320 kB 00:00 2024-05-20T19:48:55.910 INFO:teuthology.orchestra.run.smithi154.stdout:(98/159): socat-1.7.4.1-5.el9.x86_64.rpm 1.3 MB/s | 305 kB 00:00 2024-05-20T19:48:55.935 INFO:teuthology.orchestra.run.smithi154.stdout:(99/159): uuid-1.6.2-55.el9.x86_64.rpm 771 kB/s | 58 kB 00:00 2024-05-20T19:48:55.970 INFO:teuthology.orchestra.run.smithi033.stdout:(25/159): rbd-fuse-19.0.0-3830.g4ebd1d53.el9.x8 787 kB/s | 85 kB 00:00 2024-05-20T19:48:55.986 INFO:teuthology.orchestra.run.smithi154.stdout:(100/159): xfsprogs-devel-6.3.0-1.el9.x86_64.rp 1.2 MB/s | 94 kB 00:00 2024-05-20T19:48:56.011 INFO:teuthology.orchestra.run.smithi154.stdout:(101/159): xmlstarlet-1.6.1-20.el9.x86_64.rpm 846 kB/s | 64 kB 00:00 2024-05-20T19:48:56.012 INFO:teuthology.orchestra.run.smithi028.stdout:(23/159): python3-rbd-19.0.0-3830.g4ebd1d53.el9 2.5 MB/s | 300 kB 00:00 2024-05-20T19:48:56.061 INFO:teuthology.orchestra.run.smithi154.stdout:(102/159): zlib-devel-1.2.11-41.el9.x86_64.rpm 594 kB/s | 45 kB 00:00 2024-05-20T19:48:56.087 INFO:teuthology.orchestra.run.smithi154.stdout:(103/159): lua-devel-5.4.4-4.el9.x86_64.rpm 296 kB/s | 22 kB 00:00 2024-05-20T19:48:56.121 INFO:teuthology.orchestra.run.smithi028.stdout:(24/159): python3-rgw-19.0.0-3830.g4ebd1d53.el9 925 kB/s | 100 kB 00:00 2024-05-20T19:48:56.187 INFO:teuthology.orchestra.run.smithi154.stdout:(104/159): protobuf-compiler-3.14.0-13.el9.x86_ 6.7 MB/s | 863 kB 00:00 2024-05-20T19:48:56.229 INFO:teuthology.orchestra.run.smithi028.stdout:(25/159): rbd-fuse-19.0.0-3830.g4ebd1d53.el9.x8 791 kB/s | 85 kB 00:00 2024-05-20T19:48:56.321 INFO:teuthology.orchestra.run.smithi154.stdout:(105/159): abseil-cpp-20211102.0-3.el9.x86_64.r 2.3 MB/s | 548 kB 00:00 2024-05-20T19:48:56.362 INFO:teuthology.orchestra.run.smithi154.stdout:(106/159): grpc-data-1.46.7-10.el9.noarch.rpm 471 kB/s | 19 kB 00:00 2024-05-20T19:48:56.404 INFO:teuthology.orchestra.run.smithi154.stdout:(107/159): gperftools-libs-2.9.1-2.el9.x86_64.r 1.4 MB/s | 309 kB 00:00 2024-05-20T19:48:56.455 INFO:teuthology.orchestra.run.smithi154.stdout:(108/159): libarrow-doc-9.0.0-11.el9.noarch.rpm 519 kB/s | 26 kB 00:00 2024-05-20T19:48:56.530 INFO:teuthology.orchestra.run.smithi154.stdout:(109/159): libarrow-9.0.0-11.el9.x86_64.rpm 26 MB/s | 4.4 MB 00:00 2024-05-20T19:48:56.555 INFO:teuthology.orchestra.run.smithi154.stdout:(110/159): liboath-2.6.7-2.el9.x86_64.rpm 484 kB/s | 49 kB 00:00 2024-05-20T19:48:56.581 INFO:teuthology.orchestra.run.smithi154.stdout:(111/159): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-05-20T19:48:56.605 INFO:teuthology.orchestra.run.smithi028.stdout:(26/159): librgw2-19.0.0-3830.g4ebd1d53.el9.x86 3.9 MB/s | 5.4 MB 00:01 2024-05-20T19:48:56.648 INFO:teuthology.orchestra.run.smithi154.stdout:(112/159): parquet-libs-9.0.0-11.el9.x86_64.rpm 12 MB/s | 839 kB 00:00 2024-05-20T19:48:56.675 INFO:teuthology.orchestra.run.smithi154.stdout:(113/159): luarocks-3.9.2-5.el9.noarch.rpm 1.3 MB/s | 151 kB 00:00 2024-05-20T19:48:56.706 INFO:teuthology.orchestra.run.smithi154.stdout:(114/159): python3-asyncssh-2.13.2-5.el9.noarch 9.1 MB/s | 548 kB 00:00 2024-05-20T19:48:56.732 INFO:teuthology.orchestra.run.smithi154.stdout:(115/159): python3-bcrypt-3.2.2-1.el9.x86_64.rp 739 kB/s | 43 kB 00:00 2024-05-20T19:48:56.738 INFO:teuthology.orchestra.run.smithi028.stdout:(27/159): rbd-nbd-19.0.0-3830.g4ebd1d53.el9.x86 1.3 MB/s | 171 kB 00:00 2024-05-20T19:48:56.757 INFO:teuthology.orchestra.run.smithi154.stdout:(116/159): python3-cachetools-4.2.4-1.el9.noarc 639 kB/s | 32 kB 00:00 2024-05-20T19:48:56.782 INFO:teuthology.orchestra.run.smithi154.stdout:(117/159): python3-certifi-2021.10.8-2.el9.noar 299 kB/s | 15 kB 00:00 2024-05-20T19:48:56.833 INFO:teuthology.orchestra.run.smithi154.stdout:(118/159): python3-cheroot-8.6.0-4.el9.noarch.r 2.2 MB/s | 172 kB 00:00 2024-05-20T19:48:56.864 INFO:teuthology.orchestra.run.smithi028.stdout:(28/159): ceph-grafana-dashboards-19.0.0-3830.g 276 kB/s | 34 kB 00:00 2024-05-20T19:48:56.875 INFO:teuthology.orchestra.run.smithi154.stdout:(119/159): python3-cherrypy-18.6.1-2.el9.noarch 3.8 MB/s | 358 kB 00:00 2024-05-20T19:48:56.908 INFO:teuthology.orchestra.run.smithi154.stdout:(120/159): python3-cmd2-2.4.2-2.el9.noarch.rpm 3.9 MB/s | 303 kB 00:00 2024-05-20T19:48:56.967 INFO:teuthology.orchestra.run.smithi154.stdout:(121/159): python3-google-auth-2.29.0-1.el9.noa 2.2 MB/s | 206 kB 00:00 2024-05-20T19:48:56.997 INFO:teuthology.orchestra.run.smithi028.stdout:(29/159): ceph-mgr-cephadm-19.0.0-3830.g4ebd1d5 1.1 MB/s | 143 kB 00:00 2024-05-20T19:48:57.017 INFO:teuthology.orchestra.run.smithi154.stdout:(122/159): python3-grpcio-1.46.7-10.el9.x86_64. 19 MB/s | 2.0 MB 00:00 2024-05-20T19:48:57.043 INFO:teuthology.orchestra.run.smithi154.stdout:(123/159): python3-grpcio-tools-1.46.7-10.el9.x 1.9 MB/s | 144 kB 00:00 2024-05-20T19:48:57.068 INFO:teuthology.orchestra.run.smithi154.stdout:(124/159): python3-jaraco-8.2.1-3.el9.noarch.rp 212 kB/s | 11 kB 00:00 2024-05-20T19:48:57.079 INFO:teuthology.orchestra.run.smithi033.stdout:(26/159): librgw2-19.0.0-3830.g4ebd1d53.el9.x86 2.4 MB/s | 5.4 MB 00:02 2024-05-20T19:48:57.089 INFO:teuthology.orchestra.run.smithi028.stdout:(30/159): rbd-mirror-19.0.0-3830.g4ebd1d53.el9. 3.6 MB/s | 3.1 MB 00:00 2024-05-20T19:48:57.093 INFO:teuthology.orchestra.run.smithi154.stdout:(125/159): python3-jaraco-classes-3.2.1-5.el9.n 351 kB/s | 18 kB 00:00 2024-05-20T19:48:57.118 INFO:teuthology.orchestra.run.smithi154.stdout:(126/159): python3-jaraco-collections-3.0.0-8.e 460 kB/s | 23 kB 00:00 2024-05-20T19:48:57.129 INFO:teuthology.orchestra.run.smithi033.stdout:(27/159): rbd-mirror-19.0.0-3830.g4ebd1d53.el9. 2.7 MB/s | 3.1 MB 00:01 2024-05-20T19:48:57.144 INFO:teuthology.orchestra.run.smithi154.stdout:(127/159): python3-jaraco-functools-3.5.0-2.el9 387 kB/s | 19 kB 00:00 2024-05-20T19:48:57.169 INFO:teuthology.orchestra.run.smithi154.stdout:(128/159): python3-jaraco-text-3.2.0-6.el9.noar 391 kB/s | 20 kB 00:00 2024-05-20T19:48:57.213 INFO:teuthology.orchestra.run.smithi033.stdout:(28/159): rbd-nbd-19.0.0-3830.g4ebd1d53.el9.x86 1.3 MB/s | 171 kB 00:00 2024-05-20T19:48:57.219 INFO:teuthology.orchestra.run.smithi154.stdout:(129/159): python3-logutils-0.3.5-21.el9.noarch 926 kB/s | 46 kB 00:00 2024-05-20T19:48:57.238 INFO:teuthology.orchestra.run.smithi033.stdout:(29/159): ceph-grafana-dashboards-19.0.0-3830.g 315 kB/s | 34 kB 00:00 2024-05-20T19:48:57.253 INFO:teuthology.orchestra.run.smithi154.stdout:(130/159): python3-kubernetes-26.1.0-2.el9.noar 9.4 MB/s | 1.0 MB 00:00 2024-05-20T19:48:57.278 INFO:teuthology.orchestra.run.smithi154.stdout:(131/159): python3-more-itertools-8.12.0-2.el9. 1.3 MB/s | 79 kB 00:00 2024-05-20T19:48:57.303 INFO:teuthology.orchestra.run.smithi154.stdout:(132/159): python3-natsort-7.1.1-5.el9.noarch.r 1.1 MB/s | 58 kB 00:00 2024-05-20T19:48:57.321 INFO:teuthology.orchestra.run.smithi033.stdout:(30/159): ceph-mgr-cephadm-19.0.0-3830.g4ebd1d5 1.3 MB/s | 143 kB 00:00 2024-05-20T19:48:57.346 INFO:teuthology.orchestra.run.smithi154.stdout:(133/159): python3-portend-3.1.0-2.el9.noarch.r 392 kB/s | 16 kB 00:00 2024-05-20T19:48:57.371 INFO:teuthology.orchestra.run.smithi154.stdout:(134/159): python3-pecan-1.4.2-3.el9.noarch.rpm 2.9 MB/s | 272 kB 00:00 2024-05-20T19:48:57.397 INFO:teuthology.orchestra.run.smithi154.stdout:(135/159): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.7 MB/s | 90 kB 00:00 2024-05-20T19:48:57.422 INFO:teuthology.orchestra.run.smithi154.stdout:(136/159): python3-pyperclip-1.8.2-2.el9.noarch 563 kB/s | 28 kB 00:00 2024-05-20T19:48:57.447 INFO:teuthology.orchestra.run.smithi154.stdout:(137/159): python3-repoze-lru-0.7-16.el9.noarch 612 kB/s | 31 kB 00:00 2024-05-20T19:48:57.472 INFO:teuthology.orchestra.run.smithi154.stdout:(138/159): python3-routes-2.5.1-5.el9.noarch.rp 3.7 MB/s | 188 kB 00:00 2024-05-20T19:48:57.498 INFO:teuthology.orchestra.run.smithi154.stdout:(139/159): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-05-20T19:48:57.523 INFO:teuthology.orchestra.run.smithi154.stdout:(140/159): python3-tempora-5.0.0-2.el9.noarch.r 709 kB/s | 36 kB 00:00 2024-05-20T19:48:57.548 INFO:teuthology.orchestra.run.smithi154.stdout:(141/159): python3-typing-extensions-4.4.0-2.el 1.0 MB/s | 51 kB 00:00 2024-05-20T19:48:57.582 INFO:teuthology.orchestra.run.smithi154.stdout:(142/159): python3-webob-1.8.7-6.el9.noarch.rpm 3.8 MB/s | 230 kB 00:00 2024-05-20T19:48:57.616 INFO:teuthology.orchestra.run.smithi154.stdout:(143/159): python3-websocket-client-1.2.3-2.el9 1.3 MB/s | 90 kB 00:00 2024-05-20T19:48:57.649 INFO:teuthology.orchestra.run.smithi154.stdout:(144/159): python3-werkzeug-2.0.3-3.el9.1.noarc 6.2 MB/s | 427 kB 00:00 2024-05-20T19:48:57.675 INFO:teuthology.orchestra.run.smithi154.stdout:(145/159): python3-zc-lockfile-2.0-10.el9.noarc 339 kB/s | 20 kB 00:00 2024-05-20T19:48:57.700 INFO:teuthology.orchestra.run.smithi154.stdout:(146/159): re2-20211101-3.el9.x86_64.rpm 3.7 MB/s | 192 kB 00:00 2024-05-20T19:48:57.743 INFO:teuthology.orchestra.run.smithi154.stdout:(147/159): thrift-0.15.0-2.el9.x86_64.rpm 24 MB/s | 1.6 MB 00:00 2024-05-20T19:48:57.942 INFO:teuthology.orchestra.run.smithi028.stdout:(31/159): ceph-mgr-dashboard-19.0.0-3830.g4ebd1 3.8 MB/s | 3.5 MB 00:00 2024-05-20T19:48:58.065 INFO:teuthology.orchestra.run.smithi028.stdout:(32/159): ceph-mgr-modules-core-19.0.0-3830.g4e 2.2 MB/s | 274 kB 00:00 2024-05-20T19:48:58.174 INFO:teuthology.orchestra.run.smithi028.stdout:(33/159): ceph-mgr-rook-19.0.0-3830.g4ebd1d53.e 458 kB/s | 49 kB 00:00 2024-05-20T19:48:58.274 INFO:teuthology.orchestra.run.smithi028.stdout:(34/159): ceph-prometheus-alerts-19.0.0-3830.g4 166 kB/s | 16 kB 00:00 2024-05-20T19:48:58.399 INFO:teuthology.orchestra.run.smithi028.stdout:(35/159): ceph-volume-19.0.0-3830.g4ebd1d53.el9 2.2 MB/s | 273 kB 00:00 2024-05-20T19:48:58.455 INFO:teuthology.orchestra.run.smithi033.stdout:(31/159): ceph-mgr-dashboard-19.0.0-3830.g4ebd1 2.9 MB/s | 3.5 MB 00:01 2024-05-20T19:48:58.680 INFO:teuthology.orchestra.run.smithi033.stdout:(32/159): ceph-mgr-modules-core-19.0.0-3830.g4e 1.2 MB/s | 274 kB 00:00 2024-05-20T19:48:58.683 INFO:teuthology.orchestra.run.smithi028.stdout:(36/159): cephadm-19.0.0-3830.g4ebd1d53.el9.noa 2.6 MB/s | 764 kB 00:00 2024-05-20T19:48:58.789 INFO:teuthology.orchestra.run.smithi033.stdout:(33/159): ceph-mgr-rook-19.0.0-3830.g4ebd1d53.e 455 kB/s | 49 kB 00:00 2024-05-20T19:48:58.791 INFO:teuthology.orchestra.run.smithi028.stdout:(37/159): cephfs-top-19.0.0-3830.g4ebd1d53.el9. 170 kB/s | 18 kB 00:00 2024-05-20T19:48:58.900 INFO:teuthology.orchestra.run.smithi028.stdout:(38/159): fuse-2.9.9-15.el9.x86_64.rpm 749 kB/s | 80 kB 00:00 2024-05-20T19:48:58.906 INFO:teuthology.orchestra.run.smithi033.stdout:(34/159): ceph-prometheus-alerts-19.0.0-3830.g4 141 kB/s | 16 kB 00:00 2024-05-20T19:48:58.992 INFO:teuthology.orchestra.run.smithi154.stdout:(148/159): librbd1-19.0.0-3830.g4ebd1d53.el9.x8 2.5 MB/s | 3.1 MB 00:01 2024-05-20T19:48:59.050 INFO:teuthology.orchestra.run.smithi028.stdout:(39/159): ceph-mgr-diskprediction-local-19.0.0- 3.8 MB/s | 7.4 MB 00:01 2024-05-20T19:48:59.064 INFO:teuthology.orchestra.run.smithi033.stdout:(35/159): ceph-volume-19.0.0-3830.g4ebd1d53.el9 1.7 MB/s | 273 kB 00:00 2024-05-20T19:48:59.163 INFO:teuthology.orchestra.run.smithi028.stdout:(40/159): libgfortran-11.4.1-3.el9.x86_64.rpm 3.0 MB/s | 803 kB 00:00 2024-05-20T19:48:59.176 INFO:teuthology.orchestra.run.smithi154.stdout:(149/159): NetworkManager-1.47.90-1.el9.x86_64. 13 MB/s | 2.3 MB 00:00 2024-05-20T19:48:59.218 INFO:teuthology.orchestra.run.smithi028.stdout:(41/159): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.7 MB/s | 253 kB 00:00 2024-05-20T19:48:59.234 INFO:teuthology.orchestra.run.smithi154.stdout:(150/159): NetworkManager-libnm-1.47.90-1.el9.x 31 MB/s | 1.8 MB 00:00 2024-05-20T19:48:59.243 INFO:teuthology.orchestra.run.smithi028.stdout:(42/159): libquadmath-11.4.1-3.el9.x86_64.rpm 1.0 MB/s | 195 kB 00:00 2024-05-20T19:48:59.301 INFO:teuthology.orchestra.run.smithi154.stdout:(151/159): librados2-19.0.0-3830.g4ebd1d53.el9. 2.1 MB/s | 3.4 MB 00:01 2024-05-20T19:48:59.318 INFO:teuthology.orchestra.run.smithi028.stdout:(43/159): python3-cryptography-36.0.1-4.el9.x86 12 MB/s | 1.2 MB 00:00 2024-05-20T19:48:59.326 INFO:teuthology.orchestra.run.smithi154.stdout:(152/159): NetworkManager-team-1.47.90-1.el9.x8 451 kB/s | 41 kB 00:00 2024-05-20T19:48:59.335 INFO:teuthology.orchestra.run.smithi028.stdout:(44/159): python3-ply-3.11-14.el9.noarch.rpm 1.1 MB/s | 106 kB 00:00 2024-05-20T19:48:59.343 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:48:59.348 INFO:teuthology.orchestra.run.smithi033.stdout:(36/159): cephadm-19.0.0-3830.g4ebd1d53.el9.noa 2.6 MB/s | 764 kB 00:00 2024-05-20T19:48:59.361 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:59.369 INFO:teuthology.orchestra.run.smithi028.stdout:(45/159): python3-pycparser-2.20-6.el9.noarch.r 2.7 MB/s | 135 kB 00:00 2024-05-20T19:48:59.403 INFO:teuthology.orchestra.run.smithi028.stdout:(46/159): python3-requests-2.25.1-8.el9.noarch. 1.8 MB/s | 125 kB 00:00 2024-05-20T19:48:59.416 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:59.436 INFO:teuthology.orchestra.run.smithi028.stdout:(47/159): python3-urllib3-1.26.5-5.el9.noarch.r 3.2 MB/s | 215 kB 00:00 2024-05-20T19:48:59.457 INFO:teuthology.orchestra.run.smithi033.stdout:(37/159): cephfs-top-19.0.0-3830.g4ebd1d53.el9. 168 kB/s | 18 kB 00:00 2024-05-20T19:48:59.460 INFO:teuthology.orchestra.run.smithi154.stdout:(153/159): NetworkManager-tui-1.47.90-1.el9.x86 1.5 MB/s | 248 kB 00:00 2024-05-20T19:48:59.477 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:48:59.494 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:59.496 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:59.548 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:59.577 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:59.611 INFO:teuthology.orchestra.run.smithi154.stdout:(154/159): elfutils-debuginfod-client-0.191-4.e 129 kB/s | 37 kB 00:00 2024-05-20T19:48:59.627 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:59.628 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:59.644 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:48:59.698 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:48:59.706 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:59.752 INFO:teuthology.orchestra.run.smithi154.stdout:(155/159): elfutils-libelf-0.191-4.el9.x86_64.r 708 kB/s | 207 kB 00:00 2024-05-20T19:48:59.811 INFO:teuthology.orchestra.run.smithi154.stdout:(156/159): openssl-3.2.1-1.el9.x86_64.rpm 23 MB/s | 1.3 MB 00:00 2024-05-20T19:48:59.812 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:59.874 INFO:teuthology.orchestra.run.smithi033.stdout:(38/159): fuse-2.9.9-15.el9.x86_64.rpm 193 kB/s | 80 kB 00:00 2024-05-20T19:48:59.878 INFO:teuthology.orchestra.run.smithi154.stdout:(157/159): openssl-libs-3.2.1-1.el9.x86_64.rpm 36 MB/s | 2.4 MB 00:00 2024-05-20T19:48:59.892 INFO:teuthology.orchestra.run.smithi154.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:48:59.945 INFO:teuthology.orchestra.run.smithi154.stdout:(158/159): elfutils-libs-0.191-4.el9.x86_64.rpm 777 kB/s | 260 kB 00:00 2024-05-20T19:49:00.153 INFO:teuthology.orchestra.run.smithi028.stdout:(48/159): automake-1.16.2-8.el9.noarch.rpm 919 kB/s | 689 kB 00:00 2024-05-20T19:49:00.262 INFO:teuthology.orchestra.run.smithi028.stdout:(49/159): boost-program-options-1.75.0-8.el9.x8 995 kB/s | 107 kB 00:00 2024-05-20T19:49:00.304 INFO:teuthology.orchestra.run.smithi028.stdout:(50/159): bison-3.7.4-5.el9.x86_64.rpm 1.1 MB/s | 943 kB 00:00 2024-05-20T19:49:00.307 INFO:teuthology.orchestra.run.smithi033.stdout:(39/159): ceph-mgr-diskprediction-local-19.0.0- 2.5 MB/s | 7.4 MB 00:02 2024-05-20T19:49:00.342 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:00.429 INFO:teuthology.orchestra.run.smithi028.stdout:(51/159): flex-2.6.4-9.el9.x86_64.rpm 2.5 MB/s | 313 kB 00:00 2024-05-20T19:49:00.441 INFO:teuthology.orchestra.run.smithi033.stdout:(40/159): libgfortran-11.4.1-3.el9.x86_64.rpm 1.4 MB/s | 803 kB 00:00 2024-05-20T19:49:00.550 INFO:teuthology.orchestra.run.smithi033.stdout:(41/159): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.3 MB/s | 253 kB 00:00 2024-05-20T19:49:00.613 INFO:teuthology.orchestra.run.smithi028.stdout:(52/159): flexiblas-3.0.4-8.el9.x86_64.rpm 299 kB/s | 33 kB 00:00 2024-05-20T19:49:00.781 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:00.801 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:00.825 INFO:teuthology.orchestra.run.smithi033.stdout:(42/159): python3-cryptography-36.0.1-4.el9.x86 4.5 MB/s | 1.2 MB 00:00 2024-05-20T19:49:00.867 INFO:teuthology.orchestra.run.smithi033.stdout:(43/159): libquadmath-11.4.1-3.el9.x86_64.rpm 349 kB/s | 195 kB 00:00 2024-05-20T19:49:00.907 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:00.934 INFO:teuthology.orchestra.run.smithi033.stdout:(44/159): python3-ply-3.11-14.el9.noarch.rpm 981 kB/s | 106 kB 00:00 2024-05-20T19:49:00.965 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:00.976 INFO:teuthology.orchestra.run.smithi033.stdout:(45/159): python3-pycparser-2.20-6.el9.noarch.r 1.2 MB/s | 135 kB 00:00 2024-05-20T19:49:01.034 INFO:teuthology.orchestra.run.smithi033.stdout:(46/159): python3-requests-2.25.1-8.el9.noarch. 1.2 MB/s | 125 kB 00:00 2024-05-20T19:49:01.105 INFO:teuthology.orchestra.run.smithi028.stdout:(53/159): elfutils-libelf-devel-0.191-4.el9.x86 26 kB/s | 22 kB 00:00 2024-05-20T19:49:01.160 INFO:teuthology.orchestra.run.smithi033.stdout:(47/159): python3-urllib3-1.26.5-5.el9.noarch.r 1.1 MB/s | 215 kB 00:00 2024-05-20T19:49:01.193 INFO:teuthology.orchestra.run.smithi033.stdout:(48/159): automake-1.16.2-8.el9.noarch.rpm 4.2 MB/s | 689 kB 00:00 2024-05-20T19:49:01.197 INFO:teuthology.orchestra.run.smithi028.stdout:(54/159): flexiblas-netlib-3.0.4-8.el9.x86_64.r 5.2 MB/s | 3.0 MB 00:00 2024-05-20T19:49:01.222 INFO:teuthology.orchestra.run.smithi028.stdout:(55/159): flexiblas-openblas-openmp-3.0.4-8.el9 157 kB/s | 18 kB 00:00 2024-05-20T19:49:01.235 INFO:teuthology.orchestra.run.smithi033.stdout:(49/159): boost-program-options-1.75.0-8.el9.x8 2.5 MB/s | 107 kB 00:00 2024-05-20T19:49:01.237 INFO:teuthology.orchestra.run.smithi154.stdout:(159/159): ceph-test-19.0.0-3830.g4ebd1d53.el9. 3.3 MB/s | 49 MB 00:15 2024-05-20T19:49:01.243 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-05-20T19:49:01.243 INFO:teuthology.orchestra.run.smithi154.stdout:Total 12 MB/s | 227 MB 00:18 2024-05-20T19:49:01.252 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:01.306 INFO:teuthology.orchestra.run.smithi028.stdout:(56/159): libaio-devel-0.3.111-13.el9.x86_64.rp 103 kB/s | 11 kB 00:00 2024-05-20T19:49:01.340 INFO:teuthology.orchestra.run.smithi028.stdout:(57/159): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.3 MB/s | 160 kB 00:00 2024-05-20T19:49:01.344 INFO:teuthology.orchestra.run.smithi033.stdout:(50/159): bison-3.7.4-5.el9.x86_64.rpm 5.0 MB/s | 943 kB 00:00 2024-05-20T19:49:01.345 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:01.394 INFO:teuthology.orchestra.run.smithi033.stdout:(51/159): flex-2.6.4-9.el9.x86_64.rpm 6.1 MB/s | 313 kB 00:00 2024-05-20T19:49:01.415 INFO:teuthology.orchestra.run.smithi028.stdout:(58/159): librabbitmq-0.11.0-7.el9.x86_64.rpm 421 kB/s | 45 kB 00:00 2024-05-20T19:49:01.436 INFO:teuthology.orchestra.run.smithi033.stdout:(52/159): flexiblas-3.0.4-8.el9.x86_64.rpm 789 kB/s | 33 kB 00:00 2024-05-20T19:49:01.548 INFO:teuthology.orchestra.run.smithi028.stdout:(59/159): libtool-2.4.6-45.el9.x86_64.rpm 4.3 MB/s | 582 kB 00:00 2024-05-20T19:49:01.570 INFO:teuthology.orchestra.run.smithi033.stdout:(53/159): flexiblas-netlib-3.0.4-8.el9.x86_64.r 28 MB/s | 3.0 MB 00:00 2024-05-20T19:49:01.603 INFO:teuthology.orchestra.run.smithi033.stdout:(54/159): flexiblas-openblas-openmp-3.0.4-8.el9 574 kB/s | 18 kB 00:00 2024-05-20T19:49:01.632 INFO:teuthology.orchestra.run.smithi028.stdout:(60/159): librdkafka-1.6.1-102.el9.x86_64.rpm 2.2 MB/s | 662 kB 00:00 2024-05-20T19:49:01.637 INFO:teuthology.orchestra.run.smithi033.stdout:(55/159): libaio-devel-0.3.111-13.el9.x86_64.rp 334 kB/s | 11 kB 00:00 2024-05-20T19:49:01.657 INFO:teuthology.orchestra.run.smithi028.stdout:(61/159): libuuid-devel-2.37.4-18.el9.x86_64.rp 236 kB/s | 25 kB 00:00 2024-05-20T19:49:01.679 INFO:teuthology.orchestra.run.smithi033.stdout:(56/159): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.8 MB/s | 160 kB 00:00 2024-05-20T19:49:01.713 INFO:teuthology.orchestra.run.smithi033.stdout:(57/159): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.3 MB/s | 45 kB 00:00 2024-05-20T19:49:01.729 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:01.741 INFO:teuthology.orchestra.run.smithi028.stdout:(62/159): libzstd-devel-1.5.1-2.el9.x86_64.rpm 439 kB/s | 47 kB 00:00 2024-05-20T19:49:01.771 INFO:teuthology.orchestra.run.smithi033.stdout:(58/159): librdkafka-1.6.1-102.el9.x86_64.rpm 11 MB/s | 662 kB 00:00 2024-05-20T19:49:01.774 INFO:teuthology.orchestra.run.smithi028.stdout:(63/159): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.5 MB/s | 292 kB 00:00 2024-05-20T19:49:01.780 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:01.847 INFO:teuthology.orchestra.run.smithi033.stdout:(59/159): libtool-2.4.6-45.el9.x86_64.rpm 7.6 MB/s | 582 kB 00:00 2024-05-20T19:49:01.850 INFO:teuthology.orchestra.run.smithi028.stdout:(64/159): lua-5.4.4-4.el9.x86_64.rpm 1.7 MB/s | 188 kB 00:00 2024-05-20T19:49:01.856 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-devel-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/elfutils-libelf-devel-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:01.883 INFO:teuthology.orchestra.run.smithi028.stdout:(65/159): openblas-0.3.21-2.el9.x86_64.rpm 326 kB/s | 35 kB 00:00 2024-05-20T19:49:01.888 INFO:teuthology.orchestra.run.smithi033.stdout:(60/159): libuuid-devel-2.37.4-18.el9.x86_64.rp 628 kB/s | 25 kB 00:00 2024-05-20T19:49:01.930 INFO:teuthology.orchestra.run.smithi033.stdout:(61/159): libzstd-devel-1.5.1-2.el9.x86_64.rpm 1.1 MB/s | 47 kB 00:00 2024-05-20T19:49:01.956 INFO:teuthology.orchestra.run.smithi033.stdout:(62/159): elfutils-libelf-devel-0.191-4.el9.x86 30 kB/s | 22 kB 00:00 2024-05-20T19:49:01.989 INFO:teuthology.orchestra.run.smithi033.stdout:(63/159): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.9 MB/s | 292 kB 00:00 2024-05-20T19:49:02.015 INFO:teuthology.orchestra.run.smithi033.stdout:(64/159): lua-5.4.4-4.el9.x86_64.rpm 3.1 MB/s | 188 kB 00:00 2024-05-20T19:49:02.040 INFO:teuthology.orchestra.run.smithi033.stdout:(65/159): openblas-0.3.21-2.el9.x86_64.rpm 701 kB/s | 35 kB 00:00 2024-05-20T19:49:02.148 INFO:teuthology.orchestra.run.smithi033.stdout:(66/159): openssl-devel-3.2.1-1.el9.x86_64.rpm 41 MB/s | 4.4 MB 00:00 2024-05-20T19:49:02.232 INFO:teuthology.orchestra.run.smithi033.stdout:(67/159): openblas-openmp-0.3.21-2.el9.x86_64.r 22 MB/s | 4.7 MB 00:00 2024-05-20T19:49:02.291 INFO:teuthology.orchestra.run.smithi033.stdout:(68/159): postgresql-13.14-1.el9.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2024-05-20T19:49:02.325 INFO:teuthology.orchestra.run.smithi033.stdout:(69/159): postgresql-contrib-13.14-1.el9.x86_64 9.7 MB/s | 911 kB 00:00 2024-05-20T19:49:02.325 INFO:teuthology.orchestra.run.smithi028.stdout:(66/159): openssl-devel-3.2.1-1.el9.x86_64.rpm 10 MB/s | 4.4 MB 00:00 2024-05-20T19:49:02.350 INFO:teuthology.orchestra.run.smithi033.stdout:(70/159): postgresql-private-libs-13.14-1.el9.x 2.2 MB/s | 133 kB 00:00 2024-05-20T19:49:02.400 INFO:teuthology.orchestra.run.smithi033.stdout:(71/159): protobuf-3.14.0-13.el9.x86_64.rpm 20 MB/s | 1.0 MB 00:00 2024-05-20T19:49:02.475 INFO:teuthology.orchestra.run.smithi033.stdout:(72/159): postgresql-server-13.14-1.el9.x86_64. 38 MB/s | 5.8 MB 00:00 2024-05-20T19:49:02.486 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-05-20T19:49:02.501 INFO:teuthology.orchestra.run.smithi033.stdout:(73/159): python3-attrs-20.3.0-7.el9.noarch.rpm 873 kB/s | 87 kB 00:00 2024-05-20T19:49:02.518 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 199.223.246.113) 2024-05-20T19:49:02.559 INFO:teuthology.orchestra.run.smithi028.stdout:(67/159): postgresql-13.14-1.el9.x86_64.rpm 6.8 MB/s | 1.6 MB 00:00 2024-05-20T19:49:02.618 INFO:teuthology.orchestra.run.smithi033.stdout:(74/159): python3-babel-2.9.1-2.el9.noarch.rpm 42 MB/s | 6.0 MB 00:00 2024-05-20T19:49:02.618 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:02.648 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-05-20T19:49:02.648 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-05-20T19:49:02.659 INFO:teuthology.orchestra.run.smithi033.stdout:(75/159): python3-devel-3.9.18-3.el9.x86_64.rpm 5.7 MB/s | 244 kB 00:00 2024-05-20T19:49:02.676 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 199.223.246.113) 2024-05-20T19:49:02.705 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:02.722 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:02.763 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:02.781 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:02.860 INFO:teuthology.orchestra.run.smithi033.stdout:(76/159): python3-colorama-0.4.6-3.el9.noarch.r 171 kB/s | 61 kB 00:00 2024-05-20T19:49:02.860 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:02.876 INFO:teuthology.orchestra.run.smithi028.stdout:(68/159): postgresql-contrib-13.14-1.el9.x86_64 2.8 MB/s | 911 kB 00:00 2024-05-20T19:49:02.902 INFO:teuthology.orchestra.run.smithi033.stdout:(77/159): python3-jsonpatch-1.21-16.el9.noarch. 618 kB/s | 26 kB 00:00 2024-05-20T19:49:02.902 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:02.968 INFO:teuthology.orchestra.run.smithi033.stdout:(78/159): python3-jsonpointer-2.0-4.el9.noarch. 288 kB/s | 19 kB 00:00 2024-05-20T19:49:02.979 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:03.010 INFO:teuthology.orchestra.run.smithi028.stdout:(69/159): postgresql-private-libs-13.14-1.el9.x 1.0 MB/s | 133 kB 00:00 2024-05-20T19:49:03.010 INFO:teuthology.orchestra.run.smithi033.stdout:(79/159): python3-mako-1.1.4-6.el9.noarch.rpm 4.1 MB/s | 172 kB 00:00 2024-05-20T19:49:03.056 INFO:teuthology.orchestra.run.smithi033.stdout:(80/159): python3-markupsafe-1.1.1-12.el9.x86_6 832 kB/s | 35 kB 00:00 2024-05-20T19:49:03.078 INFO:teuthology.orchestra.run.smithi033.stdout:(81/159): python3-jinja2-2.11.3-6.el9.noarch.rp 594 kB/s | 248 kB 00:00 2024-05-20T19:49:03.120 INFO:teuthology.orchestra.run.smithi033.stdout:(82/159): python3-numpy-f2py-1.20.1-5.el9.x86_6 5.9 MB/s | 250 kB 00:00 2024-05-20T19:49:03.187 INFO:teuthology.orchestra.run.smithi033.stdout:(83/159): python3-numpy-1.20.1-5.el9.x86_64.rpm 38 MB/s | 5.1 MB 00:00 2024-05-20T19:49:03.203 INFO:teuthology.orchestra.run.smithi033.stdout:(84/159): python3-oauthlib-3.1.1-5.el9.noarch.r 2.6 MB/s | 222 kB 00:00 2024-05-20T19:49:03.229 INFO:teuthology.orchestra.run.smithi033.stdout:(85/159): python3-packaging-20.9-5.el9.noarch.r 1.8 MB/s | 77 kB 00:00 2024-05-20T19:49:03.254 INFO:teuthology.orchestra.run.smithi033.stdout:(86/159): python3-prettytable-0.7.2-27.el9.noar 839 kB/s | 42 kB 00:00 2024-05-20T19:49:03.288 INFO:teuthology.orchestra.run.smithi033.stdout:(87/159): python3-protobuf-3.14.0-13.el9.noarch 4.5 MB/s | 269 kB 00:00 2024-05-20T19:49:03.313 INFO:teuthology.orchestra.run.smithi033.stdout:(88/159): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.6 MB/s | 159 kB 00:00 2024-05-20T19:49:03.338 INFO:teuthology.orchestra.run.smithi033.stdout:(89/159): python3-pyasn1-modules-0.4.8-6.el9.no 5.4 MB/s | 279 kB 00:00 2024-05-20T19:49:03.355 INFO:teuthology.orchestra.run.smithi033.stdout:(90/159): python3-pytz-2021.1-5.el9.noarch.rpm 1.2 MB/s | 51 kB 00:00 2024-05-20T19:49:03.372 INFO:teuthology.orchestra.run.smithi033.stdout:(91/159): python3-requests-oauthlib-1.3.0-12.el 1.6 MB/s | 54 kB 00:00 2024-05-20T19:49:03.423 INFO:teuthology.orchestra.run.smithi033.stdout:(92/159): python3-toml-0.10.2-6.el9.noarch.rpm 834 kB/s | 42 kB 00:00 2024-05-20T19:49:03.465 INFO:teuthology.orchestra.run.smithi033.stdout:(93/159): python3-wcwidth-0.2.5-8.el9.noarch.rp 1.0 MB/s | 44 kB 00:00 2024-05-20T19:49:03.506 INFO:teuthology.orchestra.run.smithi033.stdout:(94/159): qatlib-24.02.0-1.el9.x86_64.rpm 5.2 MB/s | 221 kB 00:00 2024-05-20T19:49:03.540 INFO:teuthology.orchestra.run.smithi033.stdout:(95/159): qatlib-service-24.02.0-1.el9.x86_64.r 1.1 MB/s | 36 kB 00:00 2024-05-20T19:49:03.708 INFO:teuthology.orchestra.run.smithi033.stdout:(96/159): python3-scipy-1.6.2-8.el9.x86_64.rpm 46 MB/s | 16 MB 00:00 2024-05-20T19:49:03.733 INFO:teuthology.orchestra.run.smithi033.stdout:(97/159): qatzip-libs-1.2.0-1.el9.x86_64.rpm 247 kB/s | 47 kB 00:00 2024-05-20T19:49:03.758 INFO:teuthology.orchestra.run.smithi033.stdout:(98/159): socat-1.7.4.1-5.el9.x86_64.rpm 5.9 MB/s | 305 kB 00:00 2024-05-20T19:49:03.783 INFO:teuthology.orchestra.run.smithi033.stdout:(99/159): uuid-1.6.2-55.el9.x86_64.rpm 1.1 MB/s | 58 kB 00:00 2024-05-20T19:49:03.808 INFO:teuthology.orchestra.run.smithi033.stdout:(100/159): xfsprogs-devel-6.3.0-1.el9.x86_64.rp 1.8 MB/s | 94 kB 00:00 2024-05-20T19:49:03.833 INFO:teuthology.orchestra.run.smithi033.stdout:(101/159): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-05-20T19:49:03.859 INFO:teuthology.orchestra.run.smithi033.stdout:(102/159): zlib-devel-1.2.11-41.el9.x86_64.rpm 893 kB/s | 45 kB 00:00 2024-05-20T19:49:03.901 INFO:teuthology.orchestra.run.smithi033.stdout:(103/159): lua-devel-5.4.4-4.el9.x86_64.rpm 333 kB/s | 22 kB 00:00 2024-05-20T19:49:04.337 INFO:teuthology.orchestra.run.smithi033.stdout:(104/159): ceph-test-19.0.0-3830.g4ebd1d53.el9. 3.6 MB/s | 49 MB 00:13 2024-05-20T19:49:04.376 INFO:teuthology.orchestra.run.smithi028.stdout:(70/159): openblas-openmp-0.3.21-2.el9.x86_64.r 1.9 MB/s | 4.7 MB 00:02 2024-05-20T19:49:04.410 INFO:teuthology.orchestra.run.smithi033.stdout:(105/159): protobuf-compiler-3.14.0-13.el9.x86_ 1.5 MB/s | 863 kB 00:00 2024-05-20T19:49:04.484 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-05-20T19:49:04.484 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-05-20T19:49:04.727 INFO:teuthology.orchestra.run.smithi033.stdout:(106/159): grpc-data-1.46.7-10.el9.noarch.rpm 75 kB/s | 19 kB 00:00 2024-05-20T19:49:04.844 INFO:teuthology.orchestra.run.smithi033.stdout:(107/159): abseil-cpp-20211102.0-3.el9.x86_64.r 582 kB/s | 548 kB 00:00 2024-05-20T19:49:04.869 INFO:teuthology.orchestra.run.smithi033.stdout:(108/159): gperftools-libs-2.9.1-2.el9.x86_64.r 581 kB/s | 309 kB 00:00 2024-05-20T19:49:04.885 INFO:teuthology.orchestra.run.smithi028.stdout:(71/159): ceph-test-19.0.0-3830.g4ebd1d53.el9.x 4.7 MB/s | 49 MB 00:10 2024-05-20T19:49:04.919 INFO:teuthology.orchestra.run.smithi033.stdout:(109/159): libarrow-doc-9.0.0-11.el9.noarch.rpm 347 kB/s | 26 kB 00:00 2024-05-20T19:49:04.944 INFO:teuthology.orchestra.run.smithi033.stdout:(110/159): liboath-2.6.7-2.el9.x86_64.rpm 646 kB/s | 49 kB 00:00 2024-05-20T19:49:04.995 INFO:teuthology.orchestra.run.smithi033.stdout:(111/159): libunwind-1.6.2-1.el9.x86_64.rpm 895 kB/s | 67 kB 00:00 2024-05-20T19:49:05.037 INFO:teuthology.orchestra.run.smithi033.stdout:(112/159): luarocks-3.9.2-5.el9.noarch.rpm 1.6 MB/s | 151 kB 00:00 2024-05-20T19:49:05.137 INFO:teuthology.orchestra.run.smithi033.stdout:(113/159): parquet-libs-9.0.0-11.el9.x86_64.rpm 5.8 MB/s | 839 kB 00:00 2024-05-20T19:49:05.179 INFO:teuthology.orchestra.run.smithi033.stdout:(114/159): python3-asyncssh-2.13.2-5.el9.noarch 3.8 MB/s | 548 kB 00:00 2024-05-20T19:49:05.246 INFO:teuthology.orchestra.run.smithi033.stdout:(115/159): libarrow-9.0.0-11.el9.x86_64.rpm 8.5 MB/s | 4.4 MB 00:00 2024-05-20T19:49:05.271 INFO:teuthology.orchestra.run.smithi033.stdout:(116/159): python3-bcrypt-3.2.2-1.el9.x86_64.rp 324 kB/s | 43 kB 00:00 2024-05-20T19:49:05.296 INFO:teuthology.orchestra.run.smithi033.stdout:(117/159): python3-cachetools-4.2.4-1.el9.noarc 275 kB/s | 32 kB 00:00 2024-05-20T19:49:05.310 INFO:teuthology.orchestra.run.smithi028.stdout:(72/159): python3-attrs-20.3.0-7.el9.noarch.rpm 206 kB/s | 87 kB 00:00 2024-05-20T19:49:05.321 INFO:teuthology.orchestra.run.smithi033.stdout:(118/159): python3-certifi-2021.10.8-2.el9.noar 199 kB/s | 15 kB 00:00 2024-05-20T19:49:05.405 INFO:teuthology.orchestra.run.smithi033.stdout:(119/159): python3-cheroot-8.6.0-4.el9.noarch.r 1.3 MB/s | 172 kB 00:00 2024-05-20T19:49:05.439 INFO:teuthology.orchestra.run.smithi033.stdout:(120/159): python3-cmd2-2.4.2-2.el9.noarch.rpm 2.5 MB/s | 303 kB 00:00 2024-05-20T19:49:05.464 INFO:teuthology.orchestra.run.smithi033.stdout:(121/159): python3-cherrypy-18.6.1-2.el9.noarch 2.1 MB/s | 358 kB 00:00 2024-05-20T19:49:05.489 INFO:teuthology.orchestra.run.smithi033.stdout:(122/159): python3-google-auth-2.29.0-1.el9.noa 2.4 MB/s | 206 kB 00:00 2024-05-20T19:49:05.556 INFO:teuthology.orchestra.run.smithi033.stdout:(123/159): python3-grpcio-1.46.7-10.el9.x86_64. 17 MB/s | 2.0 MB 00:00 2024-05-20T19:49:05.582 INFO:teuthology.orchestra.run.smithi033.stdout:(124/159): python3-jaraco-8.2.1-3.el9.noarch.rp 116 kB/s | 11 kB 00:00 2024-05-20T19:49:05.616 INFO:teuthology.orchestra.run.smithi033.stdout:(125/159): python3-grpcio-tools-1.46.7-10.el9.x 956 kB/s | 144 kB 00:00 2024-05-20T19:49:05.641 INFO:teuthology.orchestra.run.smithi033.stdout:(126/159): python3-jaraco-classes-3.2.1-5.el9.n 210 kB/s | 18 kB 00:00 2024-05-20T19:49:05.666 INFO:teuthology.orchestra.run.smithi033.stdout:(127/159): python3-jaraco-collections-3.0.0-8.e 275 kB/s | 23 kB 00:00 2024-05-20T19:49:05.691 INFO:teuthology.orchestra.run.smithi033.stdout:(128/159): python3-jaraco-functools-3.5.0-2.el9 258 kB/s | 19 kB 00:00 2024-05-20T19:49:05.717 INFO:teuthology.orchestra.run.smithi033.stdout:(129/159): python3-jaraco-text-3.2.0-6.el9.noar 260 kB/s | 20 kB 00:00 2024-05-20T19:49:05.750 INFO:teuthology.orchestra.run.smithi033.stdout:(130/159): python3-kubernetes-26.1.0-2.el9.noar 12 MB/s | 1.0 MB 00:00 2024-05-20T19:49:05.767 INFO:teuthology.orchestra.run.smithi033.stdout:(131/159): python3-logutils-0.3.5-21.el9.noarch 612 kB/s | 46 kB 00:00 2024-05-20T19:49:05.792 INFO:teuthology.orchestra.run.smithi033.stdout:(132/159): python3-more-itertools-8.12.0-2.el9. 1.0 MB/s | 79 kB 00:00 2024-05-20T19:49:05.818 INFO:teuthology.orchestra.run.smithi033.stdout:(133/159): python3-natsort-7.1.1-5.el9.noarch.r 859 kB/s | 58 kB 00:00 2024-05-20T19:49:05.874 INFO:teuthology.orchestra.run.smithi033.stdout:(134/159): python3-pecan-1.4.2-3.el9.noarch.rpm 2.5 MB/s | 272 kB 00:00 2024-05-20T19:49:05.894 INFO:teuthology.orchestra.run.smithi033.stdout:(135/159): python3-portend-3.1.0-2.el9.noarch.r 163 kB/s | 16 kB 00:00 2024-05-20T19:49:05.918 INFO:teuthology.orchestra.run.smithi033.stdout:(136/159): python3-pyOpenSSL-21.0.0-1.el9.noarc 897 kB/s | 90 kB 00:00 2024-05-20T19:49:05.936 INFO:teuthology.orchestra.run.smithi033.stdout:(137/159): python3-pyperclip-1.8.2-2.el9.noarch 462 kB/s | 28 kB 00:00 2024-05-20T19:49:05.952 INFO:teuthology.orchestra.run.smithi028.stdout:(73/159): protobuf-3.14.0-13.el9.x86_64.rpm 656 kB/s | 1.0 MB 00:01 2024-05-20T19:49:05.961 INFO:teuthology.orchestra.run.smithi033.stdout:(138/159): python3-repoze-lru-0.7-16.el9.noarch 458 kB/s | 31 kB 00:00 2024-05-20T19:49:05.986 INFO:teuthology.orchestra.run.smithi033.stdout:(139/159): python3-routes-2.5.1-5.el9.noarch.rp 2.7 MB/s | 188 kB 00:00 2024-05-20T19:49:06.011 INFO:teuthology.orchestra.run.smithi033.stdout:(140/159): python3-rsa-4.9-2.el9.noarch.rpm 783 kB/s | 59 kB 00:00 2024-05-20T19:49:06.036 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:06.037 INFO:teuthology.orchestra.run.smithi033.stdout:(141/159): python3-tempora-5.0.0-2.el9.noarch.r 473 kB/s | 36 kB 00:00 2024-05-20T19:49:06.062 INFO:teuthology.orchestra.run.smithi033.stdout:(142/159): python3-typing-extensions-4.4.0-2.el 675 kB/s | 51 kB 00:00 2024-05-20T19:49:06.096 INFO:teuthology.orchestra.run.smithi033.stdout:(143/159): python3-webob-1.8.7-6.el9.noarch.rpm 2.7 MB/s | 230 kB 00:00 2024-05-20T19:49:06.121 INFO:teuthology.orchestra.run.smithi033.stdout:(144/159): python3-websocket-client-1.2.3-2.el9 1.0 MB/s | 90 kB 00:00 2024-05-20T19:49:06.146 INFO:teuthology.orchestra.run.smithi033.stdout:(145/159): python3-werkzeug-2.0.3-3.el9.1.noarc 5.0 MB/s | 427 kB 00:00 2024-05-20T19:49:06.171 INFO:teuthology.orchestra.run.smithi033.stdout:(146/159): python3-zc-lockfile-2.0-10.el9.noarc 265 kB/s | 20 kB 00:00 2024-05-20T19:49:06.197 INFO:teuthology.orchestra.run.smithi033.stdout:(147/159): re2-20211101-3.el9.x86_64.rpm 2.5 MB/s | 192 kB 00:00 2024-05-20T19:49:06.356 INFO:teuthology.orchestra.run.smithi033.stdout:(148/159): thrift-0.15.0-2.el9.x86_64.rpm 7.6 MB/s | 1.6 MB 00:00 2024-05-20T19:49:06.431 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:06.451 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:06.556 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:06.573 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-colorama-0.4.6-3.el9.noarch.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-colorama-0.4.6-3.el9.noarch.rpm (IP: 199.223.246.113) 2024-05-20T19:49:06.653 INFO:teuthology.orchestra.run.smithi028.stdout:(74/159): python3-colorama-0.4.6-3.el9.noarch.r 88 kB/s | 61 kB 00:00 2024-05-20T19:49:07.136 INFO:teuthology.orchestra.run.smithi028.stdout:(75/159): postgresql-server-13.14-1.el9.x86_64. 1.4 MB/s | 5.8 MB 00:04 2024-05-20T19:49:07.217 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:07.278 INFO:teuthology.orchestra.run.smithi028.stdout:(76/159): python3-devel-3.9.18-3.el9.x86_64.rpm 391 kB/s | 244 kB 00:00 2024-05-20T19:49:07.280 INFO:teuthology.orchestra.run.smithi033.stdout:(149/159): NetworkManager-1.47.90-1.el9.x86_64. 2.5 MB/s | 2.3 MB 00:00 2024-05-20T19:49:07.414 INFO:teuthology.orchestra.run.smithi033.stdout:(150/159): NetworkManager-libnm-1.47.90-1.el9.x 14 MB/s | 1.8 MB 00:00 2024-05-20T19:49:07.428 INFO:teuthology.orchestra.run.smithi028.stdout:(77/159): python3-jsonpatch-1.21-16.el9.noarch. 171 kB/s | 26 kB 00:00 2024-05-20T19:49:07.514 INFO:teuthology.orchestra.run.smithi033.stdout:(151/159): librbd1-19.0.0-3830.g4ebd1d53.el9.x8 2.4 MB/s | 3.1 MB 00:01 2024-05-20T19:49:07.537 INFO:teuthology.orchestra.run.smithi028.stdout:(78/159): python3-babel-2.9.1-2.el9.noarch.rpm 2.7 MB/s | 6.0 MB 00:02 2024-05-20T19:49:07.571 INFO:teuthology.orchestra.run.smithi028.stdout:(79/159): python3-jsonpointer-2.0-4.el9.noarch. 135 kB/s | 19 kB 00:00 2024-05-20T19:49:07.573 INFO:teuthology.orchestra.run.smithi033.stdout:(152/159): librados2-19.0.0-3830.g4ebd1d53.el9. 2.4 MB/s | 3.4 MB 00:01 2024-05-20T19:49:07.598 INFO:teuthology.orchestra.run.smithi033.stdout:(153/159): NetworkManager-team-1.47.90-1.el9.x8 226 kB/s | 41 kB 00:00 2024-05-20T19:49:07.679 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:07.680 INFO:teuthology.orchestra.run.smithi028.stdout:(80/159): python3-markupsafe-1.1.1-12.el9.x86_6 322 kB/s | 35 kB 00:00 2024-05-20T19:49:07.730 INFO:teuthology.orchestra.run.smithi028.stdout:(81/159): python3-mako-1.1.4-6.el9.noarch.rpm 917 kB/s | 172 kB 00:00 2024-05-20T19:49:07.731 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 140.211.166.134) 2024-05-20T19:49:07.750 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:07.759 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:07.863 INFO:teuthology.orchestra.run.smithi028.stdout:(82/159): python3-numpy-f2py-1.20.1-5.el9.x86_6 1.8 MB/s | 250 kB 00:00 2024-05-20T19:49:07.876 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 72.4.120.219) 2024-05-20T19:49:07.895 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] python3-jinja2-2.11.3-6.el9.noarch.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/AppStream/x86_64/os/Packages/python3-jinja2-2.11.3-6.el9.noarch.rpm (IP: 199.223.246.113) 2024-05-20T19:49:08.014 INFO:teuthology.orchestra.run.smithi028.stdout:(83/159): python3-oauthlib-3.1.1-5.el9.noarch.r 1.5 MB/s | 222 kB 00:00 2024-05-20T19:49:08.072 INFO:teuthology.orchestra.run.smithi028.stdout:(84/159): python3-jinja2-2.11.3-6.el9.noarch.rp 265 kB/s | 248 kB 00:00 2024-05-20T19:49:08.073 INFO:teuthology.orchestra.run.smithi033.stdout:(154/159): NetworkManager-tui-1.47.90-1.el9.x86 444 kB/s | 248 kB 00:00 2024-05-20T19:49:08.073 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:08.091 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:08.123 INFO:teuthology.orchestra.run.smithi028.stdout:(85/159): python3-packaging-20.9-5.el9.noarch.r 716 kB/s | 77 kB 00:00 2024-05-20T19:49:08.148 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:08.154 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:08.156 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:08.164 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:08.171 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:08.181 INFO:teuthology.orchestra.run.smithi028.stdout:(86/159): python3-prettytable-0.7.2-27.el9.noar 391 kB/s | 42 kB 00:00 2024-05-20T19:49:08.231 INFO:teuthology.orchestra.run.smithi028.stdout:(87/159): python3-protobuf-3.14.0-13.el9.noarch 2.4 MB/s | 269 kB 00:00 2024-05-20T19:49:08.236 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:08.253 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:08.253 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:08.290 INFO:teuthology.orchestra.run.smithi028.stdout:(88/159): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.4 MB/s | 159 kB 00:00 2024-05-20T19:49:08.315 INFO:teuthology.orchestra.run.smithi033.stdout:(155/159): elfutils-debuginfod-client-0.191-4.e 51 kB/s | 37 kB 00:00 2024-05-20T19:49:08.315 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:08.325 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:08.340 INFO:teuthology.orchestra.run.smithi028.stdout:(89/159): python3-pyasn1-modules-0.4.8-6.el9.no 2.5 MB/s | 279 kB 00:00 2024-05-20T19:49:08.342 INFO:teuthology.orchestra.run.smithi033.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:08.390 INFO:teuthology.orchestra.run.smithi033.stdout:(156/159): elfutils-libelf-0.191-4.el9.x86_64.r 261 kB/s | 207 kB 00:00 2024-05-20T19:49:08.399 INFO:teuthology.orchestra.run.smithi028.stdout:(90/159): python3-pytz-2021.1-5.el9.noarch.rpm 473 kB/s | 51 kB 00:00 2024-05-20T19:49:08.416 INFO:teuthology.orchestra.run.smithi033.stdout:(157/159): elfutils-libs-0.191-4.el9.x86_64.rpm 758 kB/s | 260 kB 00:00 2024-05-20T19:49:08.449 INFO:teuthology.orchestra.run.smithi033.stdout:(158/159): openssl-3.2.1-1.el9.x86_64.rpm 10 MB/s | 1.3 MB 00:00 2024-05-20T19:49:08.450 INFO:teuthology.orchestra.run.smithi028.stdout:(91/159): python3-requests-oauthlib-1.3.0-12.el 496 kB/s | 54 kB 00:00 2024-05-20T19:49:08.558 INFO:teuthology.orchestra.run.smithi028.stdout:(92/159): python3-toml-0.10.2-6.el9.noarch.rpm 386 kB/s | 42 kB 00:00 2024-05-20T19:49:08.675 INFO:teuthology.orchestra.run.smithi028.stdout:(93/159): python3-wcwidth-0.2.5-8.el9.noarch.rp 381 kB/s | 44 kB 00:00 2024-05-20T19:49:08.784 INFO:teuthology.orchestra.run.smithi028.stdout:(94/159): qatlib-24.02.0-1.el9.x86_64.rpm 2.0 MB/s | 221 kB 00:00 2024-05-20T19:49:08.892 INFO:teuthology.orchestra.run.smithi028.stdout:(95/159): qatlib-service-24.02.0-1.el9.x86_64.r 333 kB/s | 36 kB 00:00 2024-05-20T19:49:09.001 INFO:teuthology.orchestra.run.smithi028.stdout:(96/159): qatzip-libs-1.2.0-1.el9.x86_64.rpm 440 kB/s | 47 kB 00:00 2024-05-20T19:49:09.016 INFO:teuthology.orchestra.run.smithi033.stdout:(159/159): openssl-libs-3.2.1-1.el9.x86_64.rpm 3.8 MB/s | 2.4 MB 00:00 2024-05-20T19:49:09.021 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-05-20T19:49:09.021 INFO:teuthology.orchestra.run.smithi033.stdout:Total 9.5 MB/s | 227 MB 00:23 2024-05-20T19:49:09.118 INFO:teuthology.orchestra.run.smithi028.stdout:(97/159): socat-1.7.4.1-5.el9.x86_64.rpm 2.6 MB/s | 305 kB 00:00 2024-05-20T19:49:09.227 INFO:teuthology.orchestra.run.smithi028.stdout:(98/159): uuid-1.6.2-55.el9.x86_64.rpm 536 kB/s | 58 kB 00:00 2024-05-20T19:49:09.335 INFO:teuthology.orchestra.run.smithi028.stdout:(99/159): xfsprogs-devel-6.3.0-1.el9.x86_64.rpm 871 kB/s | 94 kB 00:00 2024-05-20T19:49:09.444 INFO:teuthology.orchestra.run.smithi028.stdout:(100/159): xmlstarlet-1.6.1-20.el9.x86_64.rpm 592 kB/s | 64 kB 00:00 2024-05-20T19:49:09.561 INFO:teuthology.orchestra.run.smithi028.stdout:(101/159): zlib-devel-1.2.11-41.el9.x86_64.rpm 386 kB/s | 45 kB 00:00 2024-05-20T19:49:09.678 INFO:teuthology.orchestra.run.smithi028.stdout:(102/159): lua-devel-5.4.4-4.el9.x86_64.rpm 192 kB/s | 22 kB 00:00 2024-05-20T19:49:09.811 INFO:teuthology.orchestra.run.smithi028.stdout:(103/159): protobuf-compiler-3.14.0-13.el9.x86_ 6.4 MB/s | 863 kB 00:00 2024-05-20T19:49:10.285 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-05-20T19:49:10.445 INFO:teuthology.orchestra.run.smithi028.stdout:(104/159): python3-numpy-1.20.1-5.el9.x86_64.rp 1.8 MB/s | 5.1 MB 00:02 2024-05-20T19:49:10.465 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-05-20T19:49:10.466 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-05-20T19:49:10.537 INFO:teuthology.orchestra.run.smithi028.stdout:(105/159): abseil-cpp-20211102.0-3.el9.x86_64.r 756 kB/s | 548 kB 00:00 2024-05-20T19:49:10.571 INFO:teuthology.orchestra.run.smithi028.stdout:(106/159): gperftools-libs-2.9.1-2.el9.x86_64.r 2.5 MB/s | 309 kB 00:00 2024-05-20T19:49:10.629 INFO:teuthology.orchestra.run.smithi028.stdout:(107/159): grpc-data-1.46.7-10.el9.noarch.rpm 220 kB/s | 19 kB 00:00 2024-05-20T19:49:10.721 INFO:teuthology.orchestra.run.smithi028.stdout:(108/159): libarrow-doc-9.0.0-11.el9.noarch.rpm 286 kB/s | 26 kB 00:00 2024-05-20T19:49:10.768 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-05-20T19:49:10.883 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : openssl-libs-1:3.2.1-1.el9.x86_64 1/170 2024-05-20T19:49:10.905 INFO:teuthology.orchestra.run.smithi028.stdout:(109/159): libarrow-9.0.0-11.el9.x86_64.rpm 13 MB/s | 4.4 MB 00:00 2024-05-20T19:49:10.947 INFO:teuthology.orchestra.run.smithi028.stdout:(110/159): liboath-2.6.7-2.el9.x86_64.rpm 219 kB/s | 49 kB 00:00 2024-05-20T19:49:10.989 INFO:teuthology.orchestra.run.smithi028.stdout:(111/159): libunwind-1.6.2-1.el9.x86_64.rpm 820 kB/s | 67 kB 00:00 2024-05-20T19:49:11.073 INFO:teuthology.orchestra.run.smithi028.stdout:(112/159): luarocks-3.9.2-5.el9.noarch.rpm 1.2 MB/s | 151 kB 00:00 2024-05-20T19:49:11.132 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/170 2024-05-20T19:49:11.195 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/170 2024-05-20T19:49:11.235 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/170 2024-05-20T19:49:11.259 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/170 2024-05-20T19:49:11.273 INFO:teuthology.orchestra.run.smithi028.stdout:(113/159): parquet-libs-9.0.0-11.el9.x86_64.rpm 2.9 MB/s | 839 kB 00:00 2024-05-20T19:49:11.304 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/170 2024-05-20T19:49:11.315 INFO:teuthology.orchestra.run.smithi028.stdout:(114/159): python3-asyncssh-2.13.2-5.el9.noarch 2.2 MB/s | 548 kB 00:00 2024-05-20T19:49:11.390 INFO:teuthology.orchestra.run.smithi028.stdout:(115/159): python3-bcrypt-3.2.2-1.el9.x86_64.rp 378 kB/s | 43 kB 00:00 2024-05-20T19:49:11.415 INFO:teuthology.orchestra.run.smithi028.stdout:(116/159): python3-cachetools-4.2.4-1.el9.noarc 323 kB/s | 32 kB 00:00 2024-05-20T19:49:11.474 INFO:teuthology.orchestra.run.smithi028.stdout:(117/159): python3-certifi-2021.10.8-2.el9.noar 181 kB/s | 15 kB 00:00 2024-05-20T19:49:11.516 INFO:teuthology.orchestra.run.smithi028.stdout:(118/159): python3-cheroot-8.6.0-4.el9.noarch.r 1.7 MB/s | 172 kB 00:00 2024-05-20T19:49:11.600 INFO:teuthology.orchestra.run.smithi028.stdout:(119/159): python3-cherrypy-18.6.1-2.el9.noarch 2.8 MB/s | 358 kB 00:00 2024-05-20T19:49:11.633 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/170 2024-05-20T19:49:11.634 INFO:teuthology.orchestra.run.smithi028.stdout:(120/159): python3-cmd2-2.4.2-2.el9.noarch.rpm 2.5 MB/s | 303 kB 00:00 2024-05-20T19:49:11.717 INFO:teuthology.orchestra.run.smithi028.stdout:(121/159): python3-google-auth-2.29.0-1.el9.noa 1.8 MB/s | 206 kB 00:00 2024-05-20T19:49:11.735 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:49:11.826 INFO:teuthology.orchestra.run.smithi028.stdout:(122/159): python3-grpcio-tools-1.46.7-10.el9.x 1.3 MB/s | 144 kB 00:00 2024-05-20T19:49:11.857 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:49:11.884 INFO:teuthology.orchestra.run.smithi028.stdout:(123/159): python3-grpcio-1.46.7-10.el9.x86_64. 8.2 MB/s | 2.0 MB 00:00 2024-05-20T19:49:11.886 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:49:11.910 INFO:teuthology.orchestra.run.smithi028.stdout:(124/159): python3-jaraco-8.2.1-3.el9.noarch.rp 130 kB/s | 11 kB 00:00 2024-05-20T19:49:11.961 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:49:11.976 INFO:teuthology.orchestra.run.smithi028.stdout:(125/159): python3-jaraco-classes-3.2.1-5.el9.n 194 kB/s | 18 kB 00:00 2024-05-20T19:49:11.991 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 10/170 2024-05-20T19:49:12.002 INFO:teuthology.orchestra.run.smithi028.stdout:(126/159): python3-jaraco-collections-3.0.0-8.e 253 kB/s | 23 kB 00:00 2024-05-20T19:49:12.024 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/170 2024-05-20T19:49:12.052 INFO:teuthology.orchestra.run.smithi028.stdout:(127/159): python3-jaraco-functools-3.5.0-2.el9 262 kB/s | 19 kB 00:00 2024-05-20T19:49:12.077 INFO:teuthology.orchestra.run.smithi028.stdout:(128/159): python3-jaraco-text-3.2.0-6.el9.noar 263 kB/s | 20 kB 00:00 2024-05-20T19:49:12.199 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : elfutils-libelf-0.191-4.el9.x86_64 12/170 2024-05-20T19:49:12.211 INFO:teuthology.orchestra.run.smithi028.stdout:(129/159): python3-kubernetes-26.1.0-2.el9.noar 6.5 MB/s | 1.0 MB 00:00 2024-05-20T19:49:12.228 INFO:teuthology.orchestra.run.smithi028.stdout:(130/159): python3-logutils-0.3.5-21.el9.noarch 309 kB/s | 46 kB 00:00 2024-05-20T19:49:12.248 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : postgresql-13.14-1.el9.x86_64 13/170 2024-05-20T19:49:12.276 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:49:12.303 INFO:teuthology.orchestra.run.smithi028.stdout:(131/159): python3-more-itertools-8.12.0-2.el9. 869 kB/s | 79 kB 00:00 2024-05-20T19:49:12.320 INFO:teuthology.orchestra.run.smithi028.stdout:(132/159): python3-natsort-7.1.1-5.el9.noarch.r 632 kB/s | 58 kB 00:00 2024-05-20T19:49:12.387 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-05-20T19:49:12.388 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-05-20T19:49:12.388 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:49:12.418 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 15/170 2024-05-20T19:49:12.420 INFO:teuthology.orchestra.run.smithi028.stdout:(133/159): python3-pecan-1.4.2-3.el9.noarch.rpm 2.3 MB/s | 272 kB 00:00 2024-05-20T19:49:12.446 INFO:teuthology.orchestra.run.smithi028.stdout:(134/159): python3-portend-3.1.0-2.el9.noarch.r 131 kB/s | 16 kB 00:00 2024-05-20T19:49:12.481 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 15/170 2024-05-20T19:49:12.503 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 16/170 2024-05-20T19:49:12.513 INFO:teuthology.orchestra.run.smithi028.stdout:(135/159): python3-pyOpenSSL-21.0.0-1.el9.noarc 987 kB/s | 90 kB 00:00 2024-05-20T19:49:12.528 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 16/170 2024-05-20T19:49:12.530 INFO:teuthology.orchestra.run.smithi028.stdout:(136/159): python3-pyperclip-1.8.2-2.el9.noarch 340 kB/s | 28 kB 00:00 2024-05-20T19:49:12.605 INFO:teuthology.orchestra.run.smithi028.stdout:(137/159): python3-repoze-lru-0.7-16.el9.noarch 346 kB/s | 31 kB 00:00 2024-05-20T19:49:12.638 INFO:teuthology.orchestra.run.smithi028.stdout:(138/159): python3-routes-2.5.1-5.el9.noarch.rp 1.7 MB/s | 188 kB 00:00 2024-05-20T19:49:12.681 INFO:teuthology.orchestra.run.smithi028.stdout:(139/159): python3-rsa-4.9-2.el9.noarch.rpm 810 kB/s | 59 kB 00:00 2024-05-20T19:49:12.721 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 17/170 2024-05-20T19:49:12.739 INFO:teuthology.orchestra.run.smithi028.stdout:(140/159): python3-tempora-5.0.0-2.el9.noarch.r 358 kB/s | 36 kB 00:00 2024-05-20T19:49:12.764 INFO:teuthology.orchestra.run.smithi028.stdout:(141/159): python3-typing-extensions-4.4.0-2.el 613 kB/s | 51 kB 00:00 2024-05-20T19:49:12.848 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 17/170 2024-05-20T19:49:12.856 INFO:teuthology.orchestra.run.smithi028.stdout:(142/159): python3-webob-1.8.7-6.el9.noarch.rpm 1.9 MB/s | 230 kB 00:00 2024-05-20T19:49:12.882 INFO:teuthology.orchestra.run.smithi028.stdout:(143/159): python3-websocket-client-1.2.3-2.el9 771 kB/s | 90 kB 00:00 2024-05-20T19:49:12.990 INFO:teuthology.orchestra.run.smithi028.stdout:(144/159): python3-werkzeug-2.0.3-3.el9.1.noarc 3.2 MB/s | 427 kB 00:00 2024-05-20T19:49:13.007 INFO:teuthology.orchestra.run.smithi028.stdout:(145/159): python3-zc-lockfile-2.0-10.el9.noarc 160 kB/s | 20 kB 00:00 2024-05-20T19:49:13.042 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : NetworkManager-libnm-1:1.47.90-1.el9.x86_64 18/170 2024-05-20T19:49:13.086 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.90-1.el9.x86_64 18/170 2024-05-20T19:49:13.116 INFO:teuthology.orchestra.run.smithi028.stdout:(146/159): re2-20211101-3.el9.x86_64.rpm 1.5 MB/s | 192 kB 00:00 2024-05-20T19:49:13.180 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:13.216 INFO:teuthology.orchestra.run.smithi028.stdout:(147/159): thrift-0.15.0-2.el9.x86_64.rpm 7.6 MB/s | 1.6 MB 00:00 2024-05-20T19:49:13.262 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:13.516 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:13.633 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : re2-1:20211101-3.el9.x86_64 20/170 2024-05-20T19:49:13.652 INFO:teuthology.orchestra.run.smithi028.stdout:(148/159): librbd1-19.0.0-3830.g4ebd1d53.el9.x8 7.3 MB/s | 3.1 MB 00:00 2024-05-20T19:49:13.667 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 21/170 2024-05-20T19:49:13.703 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : liboath-2.6.7-2.el9.x86_64 22/170 2024-05-20T19:49:13.743 INFO:teuthology.orchestra.run.smithi028.stdout:(149/159): librados2-19.0.0-3830.g4ebd1d53.el9. 5.4 MB/s | 3.4 MB 00:00 2024-05-20T19:49:13.778 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 23/170 2024-05-20T19:49:13.819 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 24/170 2024-05-20T19:49:13.863 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/170 2024-05-20T19:49:13.892 INFO:teuthology.orchestra.run.smithi028.stdout:(150/159): NetworkManager-1.47.90-1.el9.x86_64. 9.7 MB/s | 2.3 MB 00:00 2024-05-20T19:49:13.906 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 26/170 2024-05-20T19:49:13.931 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : lua-5.4.4-4.el9.x86_64 27/170 2024-05-20T19:49:13.934 INFO:teuthology.orchestra.run.smithi028.stdout:(151/159): NetworkManager-team-1.47.90-1.el9.x8 1.0 MB/s | 41 kB 00:00 2024-05-20T19:49:13.993 INFO:teuthology.orchestra.run.smithi028.stdout:(152/159): NetworkManager-libnm-1.47.90-1.el9.x 7.2 MB/s | 1.8 MB 00:00 2024-05-20T19:49:14.001 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 28/170 2024-05-20T19:49:14.026 INFO:teuthology.orchestra.run.smithi028.stdout:(153/159): NetworkManager-tui-1.47.90-1.el9.x86 2.7 MB/s | 248 kB 00:00 2024-05-20T19:49:14.027 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:14.044 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:14.046 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:14.057 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 29/170 2024-05-20T19:49:14.090 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 30/170 2024-05-20T19:49:14.130 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:14.131 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 31/170 2024-05-20T19:49:14.203 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:14.209 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 32/170 2024-05-20T19:49:14.210 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:14.243 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9. 33/170 2024-05-20T19:49:14.278 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el 34/170 2024-05-20T19:49:14.282 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:14.292 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:14.307 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_6 35/170 2024-05-20T19:49:14.377 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 36/170 2024-05-20T19:49:14.423 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : luarocks-3.9.2-5.el9.noarch 37/170 2024-05-20T19:49:14.498 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 38/170 2024-05-20T19:49:14.631 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 39/170 2024-05-20T19:49:14.683 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 40/170 2024-05-20T19:49:14.689 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-debuginfod-client-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:14.783 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libelf-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libelf-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:14.885 INFO:teuthology.orchestra.run.smithi028.stdout:(154/159): elfutils-debuginfod-client-0.191-4.e 41 kB/s | 37 kB 00:00 2024-05-20T19:49:14.905 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:15.040 INFO:teuthology.orchestra.run.smithi028.stdout:(155/159): elfutils-libelf-0.191-4.el9.x86_64.r 204 kB/s | 207 kB 00:01 2024-05-20T19:49:15.121 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for https://mirror.rackspace.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 72.4.120.219) 2024-05-20T19:49:15.138 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T19:49:15.219 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rsa-4.9-2.el9.noarch 41/170 2024-05-20T19:49:15.252 INFO:teuthology.orchestra.run.smithi028.stdout:(156/159): openssl-3.2.1-1.el9.x86_64.rpm 6.5 MB/s | 1.3 MB 00:00 2024-05-20T19:49:15.253 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for http://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:15.314 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 42/170 2024-05-20T19:49:15.334 INFO:teuthology.orchestra.run.smithi028.stdout:[MIRROR] elfutils-libs-0.191-4.el9.x86_64.rpm: Status code: 404 for https://ftp-osl.osuosl.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/elfutils-libs-0.191-4.el9.x86_64.rpm (IP: 140.211.166.134) 2024-05-20T19:49:15.341 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 43/170 2024-05-20T19:49:15.377 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 44/170 2024-05-20T19:49:15.419 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : elfutils-libs-0.191-4.el9.x86_64 45/170 2024-05-20T19:49:15.419 INFO:teuthology.orchestra.run.smithi028.stdout:(157/159): elfutils-libs-0.191-4.el9.x86_64.rpm 489 kB/s | 260 kB 00:00 2024-05-20T19:49:15.456 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_6 46/170 2024-05-20T19:49:15.498 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 47/170 2024-05-20T19:49:15.528 INFO:teuthology.orchestra.run.smithi028.stdout:(158/159): openssl-libs-3.2.1-1.el9.x86_64.rpm 8.7 MB/s | 2.4 MB 00:00 2024-05-20T19:49:15.659 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 48/170 2024-05-20T19:49:15.709 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : openssl-1:3.2.1-1.el9.x86_64 49/170 2024-05-20T19:49:15.801 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 50/170 2024-05-20T19:49:15.865 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 51/170 2024-05-20T19:49:15.938 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 52/170 2024-05-20T19:49:15.969 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 53/170 2024-05-20T19:49:16.006 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 54/170 2024-05-20T19:49:16.053 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 55/170 2024-05-20T19:49:16.089 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 56/170 2024-05-20T19:49:16.134 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 57/170 2024-05-20T19:49:16.174 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 58/170 2024-05-20T19:49:16.312 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 59/170 2024-05-20T19:49:16.351 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 60/170 2024-05-20T19:49:16.393 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 61/170 2024-05-20T19:49:16.488 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 62/170 2024-05-20T19:49:16.517 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 63/170 2024-05-20T19:49:16.557 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 64/170 2024-05-20T19:49:16.583 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 65/170 2024-05-20T19:49:16.608 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 66/170 2024-05-20T19:49:16.684 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 67/170 2024-05-20T19:49:16.810 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 68/170 2024-05-20T19:49:16.848 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 69/170 2024-05-20T19:49:16.876 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 70/170 2024-05-20T19:49:16.911 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 71/170 2024-05-20T19:49:16.952 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : uuid-1.6.2-55.el9.x86_64 72/170 2024-05-20T19:49:16.983 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 73/170 2024-05-20T19:49:17.158 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 74/170 2024-05-20T19:49:17.263 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 75/170 2024-05-20T19:49:17.401 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 76/170 2024-05-20T19:49:17.454 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 77/170 2024-05-20T19:49:17.489 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 78/170 2024-05-20T19:49:17.602 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 79/170 2024-05-20T19:49:17.645 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 80/170 2024-05-20T19:49:17.684 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 81/170 2024-05-20T19:49:17.731 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 82/170 2024-05-20T19:49:17.830 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-packaging-20.9-5.el9.noarch 83/170 2024-05-20T19:49:17.878 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 84/170 2024-05-20T19:49:18.607 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/170 2024-05-20T19:49:18.658 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 86/170 2024-05-20T19:49:18.692 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 87/170 2024-05-20T19:49:18.793 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 88/170 2024-05-20T19:49:18.814 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-05-20T19:49:18.847 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 89/170 2024-05-20T19:49:18.892 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-colorama-0.4.6-3.el9.noarch 90/170 2024-05-20T19:49:18.920 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : openssl-libs-1:3.2.1-1.el9.x86_64 1/170 2024-05-20T19:49:18.980 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 91/170 2024-05-20T19:49:19.011 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 92/170 2024-05-20T19:49:19.128 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/170 2024-05-20T19:49:19.192 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : openblas-0.3.21-2.el9.x86_64 93/170 2024-05-20T19:49:19.217 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 94/170 2024-05-20T19:49:19.225 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/170 2024-05-20T19:49:19.287 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 95/170 2024-05-20T19:49:19.340 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/170 2024-05-20T19:49:19.372 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/170 2024-05-20T19:49:19.418 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/170 2024-05-20T19:49:19.770 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/170 2024-05-20T19:49:19.881 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:49:19.943 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 96/170 2024-05-20T19:49:20.005 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:49:20.032 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:49:20.058 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 97/170 2024-05-20T19:49:20.124 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:49:20.153 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 10/170 2024-05-20T19:49:20.186 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/170 2024-05-20T19:49:20.377 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : elfutils-libelf-0.191-4.el9.x86_64 12/170 2024-05-20T19:49:20.435 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : postgresql-13.14-1.el9.x86_64 13/170 2024-05-20T19:49:20.462 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:49:20.568 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:49:20.597 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 15/170 2024-05-20T19:49:20.659 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 15/170 2024-05-20T19:49:20.681 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 16/170 2024-05-20T19:49:20.708 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 16/170 2024-05-20T19:49:20.900 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 17/170 2024-05-20T19:49:21.061 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 17/170 2024-05-20T19:49:21.228 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : NetworkManager-libnm-1:1.47.90-1.el9.x86_64 18/170 2024-05-20T19:49:21.288 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.90-1.el9.x86_64 18/170 2024-05-20T19:49:21.309 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 98/170 2024-05-20T19:49:21.376 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:21.435 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 99/170 2024-05-20T19:49:21.466 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 100/170 2024-05-20T19:49:21.475 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:21.500 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 101/170 2024-05-20T19:49:21.678 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:21.792 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : re2-1:20211101-3.el9.x86_64 20/170 2024-05-20T19:49:21.813 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 102/170 2024-05-20T19:49:21.821 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 21/170 2024-05-20T19:49:21.857 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : liboath-2.6.7-2.el9.x86_64 22/170 2024-05-20T19:49:21.862 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 103/170 2024-05-20T19:49:21.940 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 23/170 2024-05-20T19:49:21.962 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 103/170 2024-05-20T19:49:21.981 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 24/170 2024-05-20T19:49:21.992 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 104/170 2024-05-20T19:49:22.025 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/170 2024-05-20T19:49:22.029 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 105/170 2024-05-20T19:49:22.057 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 26/170 2024-05-20T19:49:22.076 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : lua-5.4.4-4.el9.x86_64 27/170 2024-05-20T19:49:22.148 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 28/170 2024-05-20T19:49:22.200 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 29/170 2024-05-20T19:49:22.229 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 30/170 2024-05-20T19:49:22.265 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 31/170 2024-05-20T19:49:22.344 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 32/170 2024-05-20T19:49:22.376 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9. 33/170 2024-05-20T19:49:22.419 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el 34/170 2024-05-20T19:49:22.448 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_6 35/170 2024-05-20T19:49:22.513 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 106/170 2024-05-20T19:49:22.520 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 36/170 2024-05-20T19:49:22.549 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 107/170 2024-05-20T19:49:22.563 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : luarocks-3.9.2-5.el9.noarch 37/170 2024-05-20T19:49:22.640 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 38/170 2024-05-20T19:49:22.658 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 107/170 2024-05-20T19:49:22.777 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 39/170 2024-05-20T19:49:22.833 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 40/170 2024-05-20T19:49:22.860 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 108/170 2024-05-20T19:49:23.389 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rsa-4.9-2.el9.noarch 41/170 2024-05-20T19:49:23.472 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 42/170 2024-05-20T19:49:23.507 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 43/170 2024-05-20T19:49:23.543 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 44/170 2024-05-20T19:49:23.593 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : elfutils-libs-0.191-4.el9.x86_64 45/170 2024-05-20T19:49:23.631 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_6 46/170 2024-05-20T19:49:23.673 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 47/170 2024-05-20T19:49:23.833 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 48/170 2024-05-20T19:49:23.892 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : openssl-1:3.2.1-1.el9.x86_64 49/170 2024-05-20T19:49:23.989 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 50/170 2024-05-20T19:49:24.050 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 51/170 2024-05-20T19:49:24.124 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 52/170 2024-05-20T19:49:24.160 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 53/170 2024-05-20T19:49:24.197 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 54/170 2024-05-20T19:49:24.237 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 55/170 2024-05-20T19:49:24.271 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 56/170 2024-05-20T19:49:24.348 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 57/170 2024-05-20T19:49:24.390 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 58/170 2024-05-20T19:49:24.535 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 59/170 2024-05-20T19:49:24.575 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 60/170 2024-05-20T19:49:24.617 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 61/170 2024-05-20T19:49:24.716 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 62/170 2024-05-20T19:49:24.750 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 63/170 2024-05-20T19:49:24.790 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 64/170 2024-05-20T19:49:24.816 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 65/170 2024-05-20T19:49:24.840 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 66/170 2024-05-20T19:49:24.920 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 67/170 2024-05-20T19:49:25.048 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 68/170 2024-05-20T19:49:25.082 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 69/170 2024-05-20T19:49:25.130 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 70/170 2024-05-20T19:49:25.161 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 71/170 2024-05-20T19:49:25.194 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:49:25.200 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : uuid-1.6.2-55.el9.x86_64 72/170 2024-05-20T19:49:25.222 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:49:25.240 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 73/170 2024-05-20T19:49:25.269 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 74/170 2024-05-20T19:49:25.304 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 75/170 2024-05-20T19:49:25.337 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 76/170 2024-05-20T19:49:25.379 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 77/170 2024-05-20T19:49:25.413 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 78/170 2024-05-20T19:49:25.513 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 79/170 2024-05-20T19:49:25.562 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 80/170 2024-05-20T19:49:25.592 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 81/170 2024-05-20T19:49:25.631 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 82/170 2024-05-20T19:49:25.746 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-packaging-20.9-5.el9.noarch 83/170 2024-05-20T19:49:25.767 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:49:25.795 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 84/170 2024-05-20T19:49:25.801 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 110/170 2024-05-20T19:49:25.893 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 110/170 2024-05-20T19:49:25.893 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-05-20T19:49:25.893 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-05-20T19:49:25.893 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:25.927 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 111/170 2024-05-20T19:49:26.561 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/170 2024-05-20T19:49:26.607 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 86/170 2024-05-20T19:49:26.633 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 87/170 2024-05-20T19:49:26.736 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 88/170 2024-05-20T19:49:26.789 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 89/170 2024-05-20T19:49:26.835 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-colorama-0.4.6-3.el9.noarch 90/170 2024-05-20T19:49:26.926 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 91/170 2024-05-20T19:49:26.961 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 92/170 2024-05-20T19:49:27.144 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : openblas-0.3.21-2.el9.x86_64 93/170 2024-05-20T19:49:27.145 INFO:teuthology.orchestra.run.smithi028.stdout:(159/159): python3-scipy-1.6.2-8.el9.x86_64.rpm 888 kB/s | 16 MB 00:18 2024-05-20T19:49:27.159 INFO:teuthology.orchestra.run.smithi028.stdout:-------------------------------------------------------------------------------- 2024-05-20T19:49:27.159 INFO:teuthology.orchestra.run.smithi028.stdout:Total 5.6 MB/s | 227 MB 00:40 2024-05-20T19:49:27.166 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 94/170 2024-05-20T19:49:27.246 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 95/170 2024-05-20T19:49:27.939 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 96/170 2024-05-20T19:49:28.061 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 97/170 2024-05-20T19:49:29.374 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 98/170 2024-05-20T19:49:29.455 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 99/170 2024-05-20T19:49:29.483 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 100/170 2024-05-20T19:49:29.516 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 101/170 2024-05-20T19:49:29.845 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 102/170 2024-05-20T19:49:29.887 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 103/170 2024-05-20T19:49:29.979 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 103/170 2024-05-20T19:49:30.009 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 104/170 2024-05-20T19:49:30.045 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 105/170 2024-05-20T19:49:30.465 INFO:teuthology.orchestra.run.smithi028.stdout:Running transaction check 2024-05-20T19:49:30.542 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 106/170 2024-05-20T19:49:30.573 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 107/170 2024-05-20T19:49:30.628 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 107/170 2024-05-20T19:49:30.659 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 108/170 2024-05-20T19:49:30.738 INFO:teuthology.orchestra.run.smithi028.stdout:Transaction check succeeded. 2024-05-20T19:49:30.738 INFO:teuthology.orchestra.run.smithi028.stdout:Running transaction test 2024-05-20T19:49:33.074 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:49:33.105 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:49:33.551 INFO:teuthology.orchestra.run.smithi028.stdout:Transaction test succeeded. 2024-05-20T19:49:33.552 INFO:teuthology.orchestra.run.smithi028.stdout:Running transaction 2024-05-20T19:49:33.676 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:49:33.717 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 110/170 2024-05-20T19:49:33.811 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 110/170 2024-05-20T19:49:33.812 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-05-20T19:49:33.812 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-05-20T19:49:33.812 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:33.853 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 111/170 2024-05-20T19:49:37.719 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 111/170 2024-05-20T19:49:37.719 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /sys 2024-05-20T19:49:37.719 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /proc 2024-05-20T19:49:37.719 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /mnt 2024-05-20T19:49:37.719 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /var/tmp 2024-05-20T19:49:37.719 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /home 2024-05-20T19:49:37.720 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /root 2024-05-20T19:49:37.720 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /tmp 2024-05-20T19:49:37.720 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:37.900 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 112/170 2024-05-20T19:49:37.917 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 112/170 2024-05-20T19:49:37.917 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:37.917 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-05-20T19:49:37.917 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:39.386 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 113/170 2024-05-20T19:49:39.413 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 113/170 2024-05-20T19:49:39.413 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:39.413 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-05-20T19:49:39.413 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-05-20T19:49:39.413 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-05-20T19:49:39.413 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:39.637 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 114/170 2024-05-20T19:49:39.660 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 114/170 2024-05-20T19:49:39.660 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:39.660 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-05-20T19:49:39.661 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-05-20T19:49:39.661 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-05-20T19:49:39.661 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:40.104 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 115/170 2024-05-20T19:49:40.127 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 115/170 2024-05-20T19:49:40.127 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:40.127 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-05-20T19:49:40.127 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-05-20T19:49:40.127 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-05-20T19:49:40.127 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:40.225 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : automake-1.16.2-8.el9.noarch 116/170 2024-05-20T19:49:40.283 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-ply-3.11-14.el9.noarch 117/170 2024-05-20T19:49:40.322 INFO:teuthology.orchestra.run.smithi028.stdout: Preparing : 1/1 2024-05-20T19:49:40.338 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 118/170 2024-05-20T19:49:40.428 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : openssl-libs-1:3.2.1-1.el9.x86_64 1/170 2024-05-20T19:49:40.497 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 2/170 2024-05-20T19:49:40.527 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 119/170 2024-05-20T19:49:40.572 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 120/170 2024-05-20T19:49:40.604 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 3/170 2024-05-20T19:49:40.650 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 121/170 2024-05-20T19:49:40.663 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/170 2024-05-20T19:49:40.694 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 5/170 2024-05-20T19:49:40.747 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 122/170 2024-05-20T19:49:40.806 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 6/170 2024-05-20T19:49:40.887 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 123/170 2024-05-20T19:49:40.946 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 124/170 2024-05-20T19:49:40.978 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 125/170 2024-05-20T19:49:41.006 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : fuse-2.9.9-15.el9.x86_64 126/170 2024-05-20T19:49:41.381 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 7/170 2024-05-20T19:49:41.467 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 127/170 2024-05-20T19:49:41.487 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 127/170 2024-05-20T19:49:41.519 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.e 128/170 2024-05-20T19:49:41.550 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:49:41.582 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53. 129/170 2024-05-20T19:49:41.610 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 130/170 2024-05-20T19:49:41.723 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:49:41.751 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:49:41.882 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:49:42.043 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 10/170 2024-05-20T19:49:42.129 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : postgresql-private-libs-13.14-1.el9.x86_64 11/170 2024-05-20T19:49:42.571 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : elfutils-libelf-0.191-4.el9.x86_64 12/170 2024-05-20T19:49:42.641 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : postgresql-13.14-1.el9.x86_64 13/170 2024-05-20T19:49:42.662 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 130/170 2024-05-20T19:49:42.665 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:49:42.704 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 131/170 2024-05-20T19:49:42.853 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:49:42.883 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 15/170 2024-05-20T19:49:42.953 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 15/170 2024-05-20T19:49:42.976 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 16/170 2024-05-20T19:49:43.053 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 16/170 2024-05-20T19:49:43.373 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 17/170 2024-05-20T19:49:43.670 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 131/170 2024-05-20T19:49:43.696 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 132/170 2024-05-20T19:49:43.711 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 17/170 2024-05-20T19:49:43.810 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 132/170 2024-05-20T19:49:43.932 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : NetworkManager-libnm-1:1.47.90-1.el9.x86_64 18/170 2024-05-20T19:49:43.975 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el 133/170 2024-05-20T19:49:43.998 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 134/170 2024-05-20T19:49:44.020 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.90-1.el9.x86_64 18/170 2024-05-20T19:49:44.022 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 134/170 2024-05-20T19:49:44.022 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:44.022 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-05-20T19:49:44.022 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-05-20T19:49:44.022 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-05-20T19:49:44.022 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:44.082 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 135/170 2024-05-20T19:49:44.094 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 135/170 2024-05-20T19:49:44.184 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 136/170 2024-05-20T19:49:44.191 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:44.239 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 137/170 2024-05-20T19:49:44.286 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:44.647 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: NetworkManager-1:1.47.90-1.el9.x86_64 19/170 2024-05-20T19:49:44.837 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : re2-1:20211101-3.el9.x86_64 20/170 2024-05-20T19:49:44.875 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 21/170 2024-05-20T19:49:44.935 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : liboath-2.6.7-2.el9.x86_64 22/170 2024-05-20T19:49:45.054 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 23/170 2024-05-20T19:49:45.115 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 24/170 2024-05-20T19:49:45.173 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libtool-2.4.6-45.el9.x86_64 138/170 2024-05-20T19:49:45.182 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/170 2024-05-20T19:49:45.218 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 139/170 2024-05-20T19:49:45.241 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 139/170 2024-05-20T19:49:45.241 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:45.241 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-05-20T19:49:45.241 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-05-20T19:49:45.241 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-05-20T19:49:45.241 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:45.274 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 26/170 2024-05-20T19:49:45.285 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 140/170 2024-05-20T19:49:45.307 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 140/170 2024-05-20T19:49:45.307 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:45.308 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-05-20T19:49:45.308 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:45.318 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : lua-5.4.4-4.el9.x86_64 27/170 2024-05-20T19:49:45.352 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 141/170 2024-05-20T19:49:45.375 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 141/170 2024-05-20T19:49:45.375 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:45.375 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-05-20T19:49:45.375 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-05-20T19:49:45.375 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-05-20T19:49:45.376 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:45.451 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 28/170 2024-05-20T19:49:45.529 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 29/170 2024-05-20T19:49:45.568 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 30/170 2024-05-20T19:49:45.628 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 31/170 2024-05-20T19:49:45.686 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 142/170 2024-05-20T19:49:45.710 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 142/170 2024-05-20T19:49:45.710 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:45.710 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-05-20T19:49:45.710 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-05-20T19:49:45.710 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-05-20T19:49:45.710 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:49:45.748 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 32/170 2024-05-20T19:49:45.789 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9. 33/170 2024-05-20T19:49:45.838 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el 34/170 2024-05-20T19:49:45.903 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_6 35/170 2024-05-20T19:49:46.034 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 36/170 2024-05-20T19:49:46.074 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 111/170 2024-05-20T19:49:46.074 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /sys 2024-05-20T19:49:46.075 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /proc 2024-05-20T19:49:46.075 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /mnt 2024-05-20T19:49:46.075 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /var/tmp 2024-05-20T19:49:46.075 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /home 2024-05-20T19:49:46.075 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /root 2024-05-20T19:49:46.075 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /tmp 2024-05-20T19:49:46.075 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:46.099 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : luarocks-3.9.2-5.el9.noarch 37/170 2024-05-20T19:49:46.241 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 38/170 2024-05-20T19:49:46.259 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 112/170 2024-05-20T19:49:46.275 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 112/170 2024-05-20T19:49:46.275 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:46.275 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-05-20T19:49:46.276 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:46.428 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 39/170 2024-05-20T19:49:46.511 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 40/170 2024-05-20T19:49:47.259 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-rsa-4.9-2.el9.noarch 41/170 2024-05-20T19:49:47.371 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 42/170 2024-05-20T19:49:47.409 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 43/170 2024-05-20T19:49:47.456 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 44/170 2024-05-20T19:49:47.522 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : elfutils-libs-0.191-4.el9.x86_64 45/170 2024-05-20T19:49:47.579 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_6 46/170 2024-05-20T19:49:47.648 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 47/170 2024-05-20T19:49:47.795 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 113/170 2024-05-20T19:49:47.823 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 113/170 2024-05-20T19:49:47.823 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:47.823 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-05-20T19:49:47.823 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-05-20T19:49:47.823 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-05-20T19:49:47.824 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:47.887 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 48/170 2024-05-20T19:49:47.954 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : openssl-1:3.2.1-1.el9.x86_64 49/170 2024-05-20T19:49:48.062 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 114/170 2024-05-20T19:49:48.087 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 114/170 2024-05-20T19:49:48.087 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:48.087 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-05-20T19:49:48.087 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-05-20T19:49:48.087 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-05-20T19:49:48.087 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:48.087 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 50/170 2024-05-20T19:49:48.161 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 51/170 2024-05-20T19:49:48.256 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 52/170 2024-05-20T19:49:48.305 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 53/170 2024-05-20T19:49:48.343 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 54/170 2024-05-20T19:49:48.412 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 55/170 2024-05-20T19:49:48.469 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 56/170 2024-05-20T19:49:48.533 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-pyperclip-1.8.2-2.el9.noarch 57/170 2024-05-20T19:49:48.554 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 115/170 2024-05-20T19:49:48.578 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 115/170 2024-05-20T19:49:48.578 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:48.579 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-05-20T19:49:48.579 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-05-20T19:49:48.579 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-05-20T19:49:48.579 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:48.601 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 58/170 2024-05-20T19:49:48.694 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : automake-1.16.2-8.el9.noarch 116/170 2024-05-20T19:49:48.752 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ply-3.11-14.el9.noarch 117/170 2024-05-20T19:49:48.813 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 59/170 2024-05-20T19:49:48.816 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 118/170 2024-05-20T19:49:48.870 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 60/170 2024-05-20T19:49:48.925 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 61/170 2024-05-20T19:49:49.013 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 119/170 2024-05-20T19:49:49.058 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 62/170 2024-05-20T19:49:49.064 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 120/170 2024-05-20T19:49:49.094 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 63/170 2024-05-20T19:49:49.138 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 121/170 2024-05-20T19:49:49.141 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 64/170 2024-05-20T19:49:49.176 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 65/170 2024-05-20T19:49:49.201 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 66/170 2024-05-20T19:49:49.227 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 122/170 2024-05-20T19:49:49.371 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 123/170 2024-05-20T19:49:49.389 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 67/170 2024-05-20T19:49:49.431 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 124/170 2024-05-20T19:49:49.473 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 125/170 2024-05-20T19:49:49.499 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : fuse-2.9.9-15.el9.x86_64 126/170 2024-05-20T19:49:49.577 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 68/170 2024-05-20T19:49:49.616 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 69/170 2024-05-20T19:49:49.655 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : zlib-devel-1.2.11-41.el9.x86_64 70/170 2024-05-20T19:49:49.716 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 71/170 2024-05-20T19:49:49.779 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : uuid-1.6.2-55.el9.x86_64 72/170 2024-05-20T19:49:49.830 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-wcwidth-0.2.5-8.el9.noarch 73/170 2024-05-20T19:49:49.881 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 74/170 2024-05-20T19:49:49.920 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 127/170 2024-05-20T19:49:49.924 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 75/170 2024-05-20T19:49:49.947 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 127/170 2024-05-20T19:49:49.988 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.e 128/170 2024-05-20T19:49:49.993 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 76/170 2024-05-20T19:49:50.053 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53. 129/170 2024-05-20T19:49:50.081 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 77/170 2024-05-20T19:49:50.087 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 130/170 2024-05-20T19:49:50.133 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 78/170 2024-05-20T19:49:50.285 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 79/170 2024-05-20T19:49:50.315 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 143/170 2024-05-20T19:49:50.348 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 80/170 2024-05-20T19:49:50.356 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 144/170 2024-05-20T19:49:50.388 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 81/170 2024-05-20T19:49:50.397 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 145/170 2024-05-20T19:49:50.429 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 146/170 2024-05-20T19:49:50.439 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 82/170 2024-05-20T19:49:50.456 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : elfutils-libelf-devel-0.191-4.el9.x86_64 147/170 2024-05-20T19:49:50.590 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-packaging-20.9-5.el9.noarch 83/170 2024-05-20T19:49:50.643 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 148/170 2024-05-20T19:49:50.669 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 84/170 2024-05-20T19:49:50.693 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 149/170 2024-05-20T19:49:50.726 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_ 150/170 2024-05-20T19:49:50.767 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : elfutils-debuginfod-client-0.191-4.el9.x86_64 151/170 2024-05-20T19:49:50.797 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : NetworkManager-team-1:1.47.90-1.el9.x86_64 152/170 2024-05-20T19:49:50.843 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : NetworkManager-tui-1:1.47.90-1.el9.x86_64 153/170 2024-05-20T19:49:51.187 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 130/170 2024-05-20T19:49:51.230 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 131/170 2024-05-20T19:49:51.441 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:49:51.482 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:49:51.502 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:49:51.694 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 85/170 2024-05-20T19:49:51.775 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 86/170 2024-05-20T19:49:51.818 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 87/170 2024-05-20T19:49:51.973 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 88/170 2024-05-20T19:49:52.043 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 89/170 2024-05-20T19:49:52.099 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-colorama-0.4.6-3.el9.noarch 90/170 2024-05-20T19:49:52.215 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 91/170 2024-05-20T19:49:52.244 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 131/170 2024-05-20T19:49:52.261 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-cmd2-2.4.2-2.el9.noarch 92/170 2024-05-20T19:49:52.272 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 132/170 2024-05-20T19:49:52.348 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 155/170 2024-05-20T19:49:52.389 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 132/170 2024-05-20T19:49:52.436 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : openssl-devel-1:3.2.1-1.el9.x86_64 156/170 2024-05-20T19:49:52.477 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 157/170 2024-05-20T19:49:52.509 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : openblas-0.3.21-2.el9.x86_64 93/170 2024-05-20T19:49:52.533 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 94/170 2024-05-20T19:49:52.557 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el 133/170 2024-05-20T19:49:52.559 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : flex-2.6.4-9.el9.x86_64 158/170 2024-05-20T19:49:52.591 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 134/170 2024-05-20T19:49:52.597 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : bison-3.7.4-5.el9.x86_64 159/170 2024-05-20T19:49:52.615 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 134/170 2024-05-20T19:49:52.616 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:52.616 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-05-20T19:49:52.616 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-05-20T19:49:52.616 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-05-20T19:49:52.616 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:52.639 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/170 2024-05-20T19:49:52.649 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 95/170 2024-05-20T19:49:52.665 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : elfutils-debuginfod-client-0.190-2.el9.x86_64 161/170 2024-05-20T19:49:52.667 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 135/170 2024-05-20T19:49:52.678 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 135/170 2024-05-20T19:49:52.692 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : elfutils-libs-0.190-2.el9.x86_64 162/170 2024-05-20T19:49:52.692 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 163/170 2024-05-20T19:49:52.759 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 163/170 2024-05-20T19:49:52.779 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 136/170 2024-05-20T19:49:52.792 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 164/170 2024-05-20T19:49:52.792 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 165/170 2024-05-20T19:49:52.833 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 137/170 2024-05-20T19:49:52.885 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 165/170 2024-05-20T19:49:52.910 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : NetworkManager-team-1:1.47.4-1.el9.x86_64 166/170 2024-05-20T19:49:52.915 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:49:52.915 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:49:53.103 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:49:53.104 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 168/170 2024-05-20T19:49:53.196 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.4-1.el9.x86_64 168/170 2024-05-20T19:49:53.221 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 169/170 2024-05-20T19:49:53.284 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : elfutils-libelf-0.190-2.el9.x86_64 170/170 2024-05-20T19:49:53.550 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 96/170 2024-05-20T19:49:53.722 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 97/170 2024-05-20T19:49:53.816 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libtool-2.4.6-45.el9.x86_64 138/170 2024-05-20T19:49:53.853 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 139/170 2024-05-20T19:49:53.877 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 139/170 2024-05-20T19:49:53.877 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:53.877 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-05-20T19:49:53.877 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-05-20T19:49:53.877 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-05-20T19:49:53.877 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:53.920 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 140/170 2024-05-20T19:49:53.944 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 140/170 2024-05-20T19:49:53.944 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:53.944 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-05-20T19:49:53.944 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:53.996 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 141/170 2024-05-20T19:49:54.019 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 141/170 2024-05-20T19:49:54.020 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:54.020 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-05-20T19:49:54.020 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-05-20T19:49:54.020 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-05-20T19:49:54.020 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:54.338 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 142/170 2024-05-20T19:49:54.362 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 142/170 2024-05-20T19:49:54.362 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:49:54.362 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-05-20T19:49:54.362 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-05-20T19:49:54.362 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-05-20T19:49:54.362 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:49:55.474 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 98/170 2024-05-20T19:49:55.573 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 99/170 2024-05-20T19:49:55.609 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libzstd-devel-1.5.1-2.el9.x86_64 100/170 2024-05-20T19:49:55.645 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libuuid-devel-2.37.4-18.el9.x86_64 101/170 2024-05-20T19:49:56.037 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 102/170 2024-05-20T19:49:56.077 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 103/170 2024-05-20T19:49:56.199 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 103/170 2024-05-20T19:49:56.233 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 104/170 2024-05-20T19:49:56.297 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 105/170 2024-05-20T19:49:56.903 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 106/170 2024-05-20T19:49:56.938 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 107/170 2024-05-20T19:49:57.034 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 107/170 2024-05-20T19:49:57.056 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 108/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: elfutils-libelf-0.190-2.el9.x86_64 170/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 1/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 3/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 4/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 5/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 6/170 2024-05-20T19:49:59.015 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 7/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 10/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 11/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 12/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 13/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_ 15/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 16/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 17/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_6 18/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 19/170 2024-05-20T19:49:59.016 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 20/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el 21/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9. 22/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_6 23/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 24/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 25/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 26/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 27/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 28/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 29/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53. 30/170 2024-05-20T19:49:59.017 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 31/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 32/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 33/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el 34/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 35/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.e 36/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 37/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 38/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 39/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 40/170 2024-05-20T19:49:59.030 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 48/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : elfutils-libelf-devel-0.191-4.el9.x86_64 52/170 2024-05-20T19:49:59.031 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 61/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/170 2024-05-20T19:49:59.032 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openssl-devel-1:3.2.1-1.el9.x86_64 68/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-colorama-0.4.6-3.el9.noarch 76/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 77/170 2024-05-20T19:49:59.033 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 78/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 79/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 80/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 81/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 82/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 83/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 84/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 85/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 86/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 87/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 88/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 89/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 90/170 2024-05-20T19:49:59.034 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 91/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 92/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 93/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 94/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 95/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 96/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 97/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 98/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 99/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 100/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 101/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 102/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 103/170 2024-05-20T19:49:59.035 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 104/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 105/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 106/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 107/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 108/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 109/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 110/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 111/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 112/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 113/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 114/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 115/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 116/170 2024-05-20T19:49:59.036 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 117/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 118/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 119/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 120/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 121/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 122/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/170 2024-05-20T19:49:59.037 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 142/170 2024-05-20T19:49:59.038 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 149/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 151/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-1:1.47.90-1.el9.x86_64 153/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-1:1.47.4-1.el9.x86_64 154/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-libnm-1:1.47.90-1.el9.x86_64 155/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 156/170 2024-05-20T19:49:59.039 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-team-1:1.47.90-1.el9.x86_64 157/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-team-1:1.47.4-1.el9.x86_64 158/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-tui-1:1.47.90-1.el9.x86_64 159/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : elfutils-debuginfod-client-0.191-4.el9.x86_64 161/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 162/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : elfutils-libelf-0.191-4.el9.x86_64 163/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 164/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : elfutils-libs-0.191-4.el9.x86_64 165/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 166/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openssl-1:3.2.1-1.el9.x86_64 167/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 168/170 2024-05-20T19:49:59.040 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openssl-libs-1:3.2.1-1.el9.x86_64 169/170 2024-05-20T19:49:59.191 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 143/170 2024-05-20T19:49:59.233 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 144/170 2024-05-20T19:49:59.274 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 145/170 2024-05-20T19:49:59.306 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 146/170 2024-05-20T19:49:59.341 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : elfutils-libelf-devel-0.191-4.el9.x86_64 147/170 2024-05-20T19:49:59.542 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 148/170 2024-05-20T19:49:59.587 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 149/170 2024-05-20T19:49:59.620 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_ 150/170 2024-05-20T19:49:59.660 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : elfutils-debuginfod-client-0.191-4.el9.x86_64 151/170 2024-05-20T19:49:59.690 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : NetworkManager-team-1:1.47.90-1.el9.x86_64 152/170 2024-05-20T19:49:59.736 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : NetworkManager-tui-1:1.47.90-1.el9.x86_64 153/170 2024-05-20T19:49:59.863 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:49:59.912 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:50:00.262 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:50:00.300 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:50:00.333 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:50:00.591 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 109/170 2024-05-20T19:50:00.640 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 110/170 2024-05-20T19:50:00.821 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 110/170 2024-05-20T19:50:00.821 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-05-20T19:50:00.821 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-05-20T19:50:00.821 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:00.858 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 111/170 2024-05-20T19:50:01.234 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 155/170 2024-05-20T19:50:01.330 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : openssl-devel-1:3.2.1-1.el9.x86_64 156/170 2024-05-20T19:50:01.372 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 157/170 2024-05-20T19:50:01.455 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flex-2.6.4-9.el9.x86_64 158/170 2024-05-20T19:50:01.483 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : bison-3.7.4-5.el9.x86_64 159/170 2024-05-20T19:50:01.508 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/170 2024-05-20T19:50:01.533 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : elfutils-debuginfod-client-0.190-2.el9.x86_64 161/170 2024-05-20T19:50:01.560 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : elfutils-libs-0.190-2.el9.x86_64 162/170 2024-05-20T19:50:01.560 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 163/170 2024-05-20T19:50:01.644 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 163/170 2024-05-20T19:50:01.685 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 164/170 2024-05-20T19:50:01.686 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 165/170 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 170/170 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout:Upgraded: 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager-libnm-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager-team-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: NetworkManager-tui-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-debuginfod-client-0.191-4.el9.x86_64 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-libelf-0.191-4.el9.x86_64 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-libs-0.191-4.el9.x86_64 2024-05-20T19:50:01.740 INFO:teuthology.orchestra.run.smithi154.stdout: librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: openssl-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: openssl-libs-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: automake-1.16.2-8.el9.noarch 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: bison-3.7.4-5.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.741 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-diskprediction-local-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.742 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: elfutils-libelf-devel-0.191-4.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: flex-2.6.4-9.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-05-20T19:50:01.743 INFO:teuthology.orchestra.run.smithi154.stdout: fuse-2.9.9-15.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: grpc-data-1.46.7-10.el9.noarch 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libarrow-9.0.0-11.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: liboath-2.6.7-2.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-05-20T19:50:01.744 INFO:teuthology.orchestra.run.smithi154.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: libtool-2.4.6-45.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: libunwind-1.6.2-1.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: lua-5.4.4-4.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: luarocks-3.9.2-5.el9.noarch 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: openblas-0.3.21-2.el9.x86_64 2024-05-20T19:50:01.745 INFO:teuthology.orchestra.run.smithi154.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: openssl-devel-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql-13.14-1.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: postgresql-server-13.14-1.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: protobuf-3.14.0-13.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: python3-babel-2.9.1-2.el9.noarch 2024-05-20T19:50:01.746 INFO:teuthology.orchestra.run.smithi154.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-colorama-0.4.6-3.el9.noarch 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-05-20T19:50:01.747 INFO:teuthology.orchestra.run.smithi154.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-mako-1.1.4-6.el9.noarch 2024-05-20T19:50:01.748 INFO:teuthology.orchestra.run.smithi154.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-packaging-20.9-5.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ply-3.11-14.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-portend-3.1.0-2.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-05-20T19:50:01.749 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pycparser-2.20-6.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pytz-2021.1-5.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests-2.25.1-8.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-routes-2.5.1-5.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rsa-4.9-2.el9.noarch 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-05-20T19:50:01.750 INFO:teuthology.orchestra.run.smithi154.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-toml-0.10.2-6.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-webob-1.8.7-6.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: qatlib-24.02.0-1.el9.x86_64 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.751 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.752 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:01.752 INFO:teuthology.orchestra.run.smithi154.stdout: re2-1:20211101-3.el9.x86_64 2024-05-20T19:50:01.752 INFO:teuthology.orchestra.run.smithi154.stdout: socat-1.7.4.1-5.el9.x86_64 2024-05-20T19:50:01.752 INFO:teuthology.orchestra.run.smithi154.stdout: thrift-0.15.0-2.el9.x86_64 2024-05-20T19:50:01.752 INFO:teuthology.orchestra.run.smithi154.stdout: uuid-1.6.2-55.el9.x86_64 2024-05-20T19:50:01.752 INFO:teuthology.orchestra.run.smithi154.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-05-20T19:50:01.752 INFO:teuthology.orchestra.run.smithi154.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-05-20T19:50:01.753 INFO:teuthology.orchestra.run.smithi154.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-05-20T19:50:01.753 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:50:01.753 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-05-20T19:50:01.770 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 165/170 2024-05-20T19:50:01.795 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : NetworkManager-team-1:1.47.4-1.el9.x86_64 166/170 2024-05-20T19:50:01.801 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:50:01.801 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:50:01.989 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:50:01.989 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 168/170 2024-05-20T19:50:02.073 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.4-1.el9.x86_64 168/170 2024-05-20T19:50:02.098 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 169/170 2024-05-20T19:50:02.164 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : elfutils-libelf-0.190-2.el9.x86_64 170/170 2024-05-20T19:50:02.184 DEBUG:teuthology.parallel:result is None 2024-05-20T19:50:07.792 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: elfutils-libelf-0.190-2.el9.x86_64 170/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 1/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 3/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 4/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 5/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 6/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 7/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 10/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 11/170 2024-05-20T19:50:07.793 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 12/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 13/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_ 15/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 16/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 17/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_6 18/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 19/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 20/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el 21/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9. 22/170 2024-05-20T19:50:07.794 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_6 23/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 24/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 25/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 26/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 27/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 28/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 29/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53. 30/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 31/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 32/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 33/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el 34/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 35/170 2024-05-20T19:50:07.795 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.e 36/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 37/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 38/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 39/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 40/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/170 2024-05-20T19:50:07.796 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 48/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : elfutils-libelf-devel-0.191-4.el9.x86_64 52/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/170 2024-05-20T19:50:07.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 61/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openssl-devel-1:3.2.1-1.el9.x86_64 68/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/170 2024-05-20T19:50:07.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-colorama-0.4.6-3.el9.noarch 76/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 77/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 78/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 79/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 80/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 81/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 82/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 83/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 84/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 85/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 86/170 2024-05-20T19:50:07.799 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 87/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 88/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 89/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 90/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 91/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 92/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 93/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 94/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 95/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 96/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 97/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 98/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 99/170 2024-05-20T19:50:07.800 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 100/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 101/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 102/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 103/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 104/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 105/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 106/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 107/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 108/170 2024-05-20T19:50:07.801 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 109/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 110/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 111/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 112/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 113/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 114/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 115/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 116/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 117/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 118/170 2024-05-20T19:50:07.802 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 119/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 120/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 121/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 122/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/170 2024-05-20T19:50:07.803 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/170 2024-05-20T19:50:07.804 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 142/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 149/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 151/170 2024-05-20T19:50:07.805 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-1:1.47.90-1.el9.x86_64 153/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-1:1.47.4-1.el9.x86_64 154/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-libnm-1:1.47.90-1.el9.x86_64 155/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 156/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-team-1:1.47.90-1.el9.x86_64 157/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-team-1:1.47.4-1.el9.x86_64 158/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-tui-1:1.47.90-1.el9.x86_64 159/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : elfutils-debuginfod-client-0.191-4.el9.x86_64 161/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 162/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : elfutils-libelf-0.191-4.el9.x86_64 163/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 164/170 2024-05-20T19:50:07.806 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : elfutils-libs-0.191-4.el9.x86_64 165/170 2024-05-20T19:50:07.807 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 166/170 2024-05-20T19:50:07.807 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openssl-1:3.2.1-1.el9.x86_64 167/170 2024-05-20T19:50:07.807 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 168/170 2024-05-20T19:50:07.807 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openssl-libs-1:3.2.1-1.el9.x86_64 169/170 2024-05-20T19:50:10.536 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 170/170 2024-05-20T19:50:10.536 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:50:10.536 INFO:teuthology.orchestra.run.smithi033.stdout:Upgraded: 2024-05-20T19:50:10.536 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:10.536 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager-libnm-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager-team-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: NetworkManager-tui-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-debuginfod-client-0.191-4.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-libelf-0.191-4.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-libs-0.191-4.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: openssl-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: openssl-libs-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: automake-1.16.2-8.el9.noarch 2024-05-20T19:50:10.537 INFO:teuthology.orchestra.run.smithi033.stdout: bison-3.7.4-5.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.538 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-diskprediction-local-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.539 INFO:teuthology.orchestra.run.smithi033.stdout: cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: elfutils-libelf-devel-0.191-4.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: flex-2.6.4-9.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: fuse-2.9.9-15.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: grpc-data-1.46.7-10.el9.noarch 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: libarrow-9.0.0-11.el9.x86_64 2024-05-20T19:50:10.540 INFO:teuthology.orchestra.run.smithi033.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: liboath-2.6.7-2.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libtool-2.4.6-45.el9.x86_64 2024-05-20T19:50:10.541 INFO:teuthology.orchestra.run.smithi033.stdout: libunwind-1.6.2-1.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: lua-5.4.4-4.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: luarocks-3.9.2-5.el9.noarch 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-0.3.21-2.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: openssl-devel-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql-13.14-1.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-05-20T19:50:10.542 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: postgresql-server-13.14-1.el9.x86_64 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: protobuf-3.14.0-13.el9.x86_64 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: python3-babel-2.9.1-2.el9.noarch 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.543 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-colorama-0.4.6-3.el9.noarch 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-05-20T19:50:10.544 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-mako-1.1.4-6.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-05-20T19:50:10.545 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-packaging-20.9-5.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ply-3.11-14.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-portend-3.1.0-2.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-05-20T19:50:10.546 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pycparser-2.20-6.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytz-2021.1-5.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-2.25.1-8.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-routes-2.5.1-5.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rsa-4.9-2.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-05-20T19:50:10.547 INFO:teuthology.orchestra.run.smithi033.stdout: python3-toml-0.10.2-6.el9.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: python3-webob-1.8.7-6.el9.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: qatlib-24.02.0-1.el9.x86_64 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.548 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: re2-1:20211101-3.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: socat-1.7.4.1-5.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: thrift-0.15.0-2.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: uuid-1.6.2-55.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-05-20T19:50:10.549 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-05-20T19:50:10.550 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-05-20T19:50:10.999 DEBUG:teuthology.parallel:result is None 2024-05-20T19:50:14.461 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 111/170 2024-05-20T19:50:14.461 INFO:teuthology.orchestra.run.smithi028.stdout:skipping the directory /sys 2024-05-20T19:50:14.461 INFO:teuthology.orchestra.run.smithi028.stdout:skipping the directory /proc 2024-05-20T19:50:14.461 INFO:teuthology.orchestra.run.smithi028.stdout:skipping the directory /mnt 2024-05-20T19:50:14.461 INFO:teuthology.orchestra.run.smithi028.stdout:skipping the directory /var/tmp 2024-05-20T19:50:14.461 INFO:teuthology.orchestra.run.smithi028.stdout:skipping the directory /home 2024-05-20T19:50:14.462 INFO:teuthology.orchestra.run.smithi028.stdout:skipping the directory /root 2024-05-20T19:50:14.462 INFO:teuthology.orchestra.run.smithi028.stdout:skipping the directory /tmp 2024-05-20T19:50:14.462 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:14.704 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 112/170 2024-05-20T19:50:14.757 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 112/170 2024-05-20T19:50:14.757 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:14.757 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-05-20T19:50:14.757 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:16.473 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 113/170 2024-05-20T19:50:16.582 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 113/170 2024-05-20T19:50:16.582 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:16.583 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-05-20T19:50:16.583 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-05-20T19:50:16.583 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-05-20T19:50:16.583 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:16.858 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 114/170 2024-05-20T19:50:16.936 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 114/170 2024-05-20T19:50:16.936 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:16.936 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-05-20T19:50:16.936 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-05-20T19:50:16.936 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-05-20T19:50:16.936 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:17.466 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 115/170 2024-05-20T19:50:17.547 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 115/170 2024-05-20T19:50:17.547 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:17.548 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-05-20T19:50:17.548 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-05-20T19:50:17.548 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-05-20T19:50:17.548 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:17.684 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : automake-1.16.2-8.el9.noarch 116/170 2024-05-20T19:50:17.747 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-ply-3.11-14.el9.noarch 117/170 2024-05-20T19:50:17.824 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 118/170 2024-05-20T19:50:18.086 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 119/170 2024-05-20T19:50:18.146 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 120/170 2024-05-20T19:50:18.235 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 121/170 2024-05-20T19:50:18.354 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 122/170 2024-05-20T19:50:18.533 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 123/170 2024-05-20T19:50:18.586 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 124/170 2024-05-20T19:50:18.635 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 125/170 2024-05-20T19:50:18.668 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : fuse-2.9.9-15.el9.x86_64 126/170 2024-05-20T19:50:19.313 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 127/170 2024-05-20T19:50:19.359 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 127/170 2024-05-20T19:50:19.418 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.e 128/170 2024-05-20T19:50:19.530 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53. 129/170 2024-05-20T19:50:19.561 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 130/170 2024-05-20T19:50:20.902 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 130/170 2024-05-20T19:50:20.946 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 131/170 2024-05-20T19:50:22.112 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 131/170 2024-05-20T19:50:22.146 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 132/170 2024-05-20T19:50:22.312 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 132/170 2024-05-20T19:50:22.493 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el 133/170 2024-05-20T19:50:22.523 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 134/170 2024-05-20T19:50:22.606 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 134/170 2024-05-20T19:50:22.607 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:22.607 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-05-20T19:50:22.607 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-05-20T19:50:22.607 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-05-20T19:50:22.607 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:22.674 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 135/170 2024-05-20T19:50:22.712 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 135/170 2024-05-20T19:50:22.828 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 136/170 2024-05-20T19:50:22.893 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 137/170 2024-05-20T19:50:23.957 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libtool-2.4.6-45.el9.x86_64 138/170 2024-05-20T19:50:23.993 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 139/170 2024-05-20T19:50:24.073 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 139/170 2024-05-20T19:50:24.073 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:24.073 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-05-20T19:50:24.073 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-05-20T19:50:24.073 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-05-20T19:50:24.073 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:24.118 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 140/170 2024-05-20T19:50:24.204 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 140/170 2024-05-20T19:50:24.204 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:24.204 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-05-20T19:50:24.204 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:24.277 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 141/170 2024-05-20T19:50:24.357 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 141/170 2024-05-20T19:50:24.357 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:24.357 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "cephfs-mirror@*.service" escaped as "cephfs-mirror@\x2a.service". 2024-05-20T19:50:24.357 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-05-20T19:50:24.357 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/cephfs-mirror.target → /usr/lib/systemd/system/cephfs-mirror.target. 2024-05-20T19:50:24.357 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:24.745 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 142/170 2024-05-20T19:50:24.817 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 142/170 2024-05-20T19:50:24.817 INFO:teuthology.orchestra.run.smithi028.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-05-20T19:50:24.817 INFO:teuthology.orchestra.run.smithi028.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-05-20T19:50:24.817 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-05-20T19:50:24.817 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-05-20T19:50:24.818 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:30.118 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 143/170 2024-05-20T19:50:30.170 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 144/170 2024-05-20T19:50:30.227 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 145/170 2024-05-20T19:50:30.266 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : xfsprogs-devel-6.3.0-1.el9.x86_64 146/170 2024-05-20T19:50:30.303 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : elfutils-libelf-devel-0.191-4.el9.x86_64 147/170 2024-05-20T19:50:30.604 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 148/170 2024-05-20T19:50:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : postgresql-contrib-13.14-1.el9.x86_64 149/170 2024-05-20T19:50:30.696 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_ 150/170 2024-05-20T19:50:30.764 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : elfutils-debuginfod-client-0.191-4.el9.x86_64 151/170 2024-05-20T19:50:30.820 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : NetworkManager-team-1:1.47.90-1.el9.x86_64 152/170 2024-05-20T19:50:30.870 INFO:teuthology.orchestra.run.smithi028.stdout: Upgrading : NetworkManager-tui-1:1.47.90-1.el9.x86_64 153/170 2024-05-20T19:50:31.875 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:50:31.932 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:50:31.991 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: postgresql-server-13.14-1.el9.x86_64 154/170 2024-05-20T19:50:33.443 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 155/170 2024-05-20T19:50:33.546 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : openssl-devel-1:3.2.1-1.el9.x86_64 156/170 2024-05-20T19:50:33.600 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : libaio-devel-0.3.111-13.el9.x86_64 157/170 2024-05-20T19:50:33.734 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : flex-2.6.4-9.el9.x86_64 158/170 2024-05-20T19:50:33.772 INFO:teuthology.orchestra.run.smithi028.stdout: Installing : bison-3.7.4-5.el9.x86_64 159/170 2024-05-20T19:50:33.805 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/170 2024-05-20T19:50:33.840 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : elfutils-debuginfod-client-0.190-2.el9.x86_64 161/170 2024-05-20T19:50:33.887 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : elfutils-libs-0.190-2.el9.x86_64 162/170 2024-05-20T19:50:33.888 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 163/170 2024-05-20T19:50:34.023 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 163/170 2024-05-20T19:50:34.068 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : openssl-1:3.0.7-27.el9.x86_64 164/170 2024-05-20T19:50:34.069 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 165/170 2024-05-20T19:50:34.205 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 165/170 2024-05-20T19:50:34.244 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : NetworkManager-team-1:1.47.4-1.el9.x86_64 166/170 2024-05-20T19:50:34.277 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:50:34.278 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:50:34.597 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: NetworkManager-1:1.47.4-1.el9.x86_64 167/170 2024-05-20T19:50:34.597 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 168/170 2024-05-20T19:50:34.744 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: NetworkManager-libnm-1:1.47.4-1.el9.x86_64 168/170 2024-05-20T19:50:34.797 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : openssl-libs-1:3.0.7-27.el9.x86_64 169/170 2024-05-20T19:50:34.921 INFO:teuthology.orchestra.run.smithi028.stdout: Cleanup : elfutils-libelf-0.190-2.el9.x86_64 170/170 2024-05-20T19:50:35.199 INFO:teuthology.orchestra.run.smithi028.stdout: Running scriptlet: elfutils-libelf-0.190-2.el9.x86_64 170/170 2024-05-20T19:50:35.199 INFO:teuthology.orchestra.run.smithi028.stdout:Couldn't write '0' to 'kernel/yama/ptrace_scope', ignoring: No such file or directory 2024-05-20T19:50:35.200 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:40.801 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 1/170 2024-05-20T19:50:40.801 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2/170 2024-05-20T19:50:40.801 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 3/170 2024-05-20T19:50:40.801 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 4/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1 5/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 6/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 7/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 8/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 9/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 10/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 11/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 12/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 13/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 14/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_ 15/170 2024-05-20T19:50:40.802 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 16/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 17/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_6 18/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86 19/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 20/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el 21/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9. 22/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_6 23/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 24/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 25/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 26/170 2024-05-20T19:50:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 27/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 28/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 29/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53. 30/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noa 31/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.n 32/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.0.0-3830.g4eb 33/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el 34/170 2024-05-20T19:50:40.818 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 35/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.e 36/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 37/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 38/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 39/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 40/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 41/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 42/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 43/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 44/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-ply-3.11-14.el9.noarch 45/170 2024-05-20T19:50:40.819 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 46/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 47/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 48/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : automake-1.16.2-8.el9.noarch 49/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : bison-3.7.4-5.el9.x86_64 50/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 51/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : elfutils-libelf-devel-0.191-4.el9.x86_64 52/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : flex-2.6.4-9.el9.x86_64 53/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 54/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 55/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 56/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libaio-devel-0.3.111-13.el9.x86_64 57/170 2024-05-20T19:50:40.820 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 58/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 59/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 60/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libtool-2.4.6-45.el9.x86_64 61/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libuuid-devel-2.37.4-18.el9.x86_64 62/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libzstd-devel-1.5.1-2.el9.x86_64 63/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 64/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : lua-5.4.4-4.el9.x86_64 65/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 66/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 67/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : openssl-devel-1:3.2.1-1.el9.x86_64 68/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : postgresql-13.14-1.el9.x86_64 69/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : postgresql-contrib-13.14-1.el9.x86_64 70/170 2024-05-20T19:50:40.821 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : postgresql-private-libs-13.14-1.el9.x86_64 71/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : postgresql-server-13.14-1.el9.x86_64 72/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 73/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 74/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 75/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-colorama-0.4.6-3.el9.noarch 76/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 77/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 78/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 79/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 80/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 81/170 2024-05-20T19:50:40.822 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 82/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 83/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 84/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 85/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 86/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 87/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 88/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 89/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 90/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 91/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 92/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 93/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 94/170 2024-05-20T19:50:40.823 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-wcwidth-0.2.5-8.el9.noarch 95/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 96/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 97/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 98/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 99/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : uuid-1.6.2-55.el9.x86_64 100/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : xfsprogs-devel-6.3.0-1.el9.x86_64 101/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 102/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : zlib-devel-1.2.11-41.el9.x86_64 103/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 104/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 105/170 2024-05-20T19:50:40.824 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 106/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 107/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 108/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 109/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 110/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 111/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 112/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 113/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 114/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 115/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 116/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 117/170 2024-05-20T19:50:40.825 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 118/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 119/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 120/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-cmd2-2.4.2-2.el9.noarch 121/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 122/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 123/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 124/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 125/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 126/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 127/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 128/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 129/170 2024-05-20T19:50:40.826 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 130/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 131/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 132/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 133/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 134/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 135/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 136/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-pyperclip-1.8.2-2.el9.noarch 137/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 138/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 139/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 140/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 141/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 142/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 143/170 2024-05-20T19:50:40.827 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 144/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 145/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 146/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : re2-1:20211101-3.el9.x86_64 147/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 148/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 149/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 150/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 151/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 152/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-1:1.47.90-1.el9.x86_64 153/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-1:1.47.4-1.el9.x86_64 154/170 2024-05-20T19:50:40.828 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-libnm-1:1.47.90-1.el9.x86_64 155/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-libnm-1:1.47.4-1.el9.x86_64 156/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-team-1:1.47.90-1.el9.x86_64 157/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-team-1:1.47.4-1.el9.x86_64 158/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-tui-1:1.47.90-1.el9.x86_64 159/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : NetworkManager-tui-1:1.47.4-1.el9.x86_64 160/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : elfutils-debuginfod-client-0.191-4.el9.x86_64 161/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : elfutils-debuginfod-client-0.190-2.el9.x86_64 162/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : elfutils-libelf-0.191-4.el9.x86_64 163/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : elfutils-libelf-0.190-2.el9.x86_64 164/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : elfutils-libs-0.191-4.el9.x86_64 165/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : elfutils-libs-0.190-2.el9.x86_64 166/170 2024-05-20T19:50:40.829 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : openssl-1:3.2.1-1.el9.x86_64 167/170 2024-05-20T19:50:40.830 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : openssl-1:3.0.7-27.el9.x86_64 168/170 2024-05-20T19:50:40.830 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : openssl-libs-1:3.2.1-1.el9.x86_64 169/170 2024-05-20T19:50:43.173 INFO:teuthology.orchestra.run.smithi028.stdout: Verifying : openssl-libs-1:3.0.7-27.el9.x86_64 170/170 2024-05-20T19:50:43.173 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:43.174 INFO:teuthology.orchestra.run.smithi028.stdout:Upgraded: 2024-05-20T19:50:43.174 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:43.174 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager-libnm-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:43.174 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager-team-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:43.174 INFO:teuthology.orchestra.run.smithi028.stdout: NetworkManager-tui-1:1.47.90-1.el9.x86_64 2024-05-20T19:50:43.175 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-debuginfod-client-0.191-4.el9.x86_64 2024-05-20T19:50:43.175 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-libelf-0.191-4.el9.x86_64 2024-05-20T19:50:43.175 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-libs-0.191-4.el9.x86_64 2024-05-20T19:50:43.175 INFO:teuthology.orchestra.run.smithi028.stdout: librados2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.176 INFO:teuthology.orchestra.run.smithi028.stdout: librbd1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.176 INFO:teuthology.orchestra.run.smithi028.stdout: openssl-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:43.176 INFO:teuthology.orchestra.run.smithi028.stdout: openssl-libs-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:43.176 INFO:teuthology.orchestra.run.smithi028.stdout:Installed: 2024-05-20T19:50:43.176 INFO:teuthology.orchestra.run.smithi028.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-05-20T19:50:43.176 INFO:teuthology.orchestra.run.smithi028.stdout: automake-1.16.2-8.el9.noarch 2024-05-20T19:50:43.176 INFO:teuthology.orchestra.run.smithi028.stdout: bison-3.7.4-5.el9.x86_64 2024-05-20T19:50:43.177 INFO:teuthology.orchestra.run.smithi028.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-05-20T19:50:43.177 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.177 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-base-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.177 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.177 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.177 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-grafana-dashboards-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.178 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-immutable-object-cache-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.178 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mds-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.178 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.178 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.178 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-dashboard-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.178 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-diskprediction-local-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.178 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-modules-core-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.179 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-rook-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.179 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mon-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.179 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-osd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.179 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-prometheus-alerts-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.179 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-radosgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.179 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-selinux-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.180 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-test-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.180 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-volume-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.180 INFO:teuthology.orchestra.run.smithi028.stdout: cephadm-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.180 INFO:teuthology.orchestra.run.smithi028.stdout: cephfs-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.180 INFO:teuthology.orchestra.run.smithi028.stdout: cephfs-shell-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.180 INFO:teuthology.orchestra.run.smithi028.stdout: cephfs-top-2:19.0.0-3830.g4ebd1d53.el9.noarch 2024-05-20T19:50:43.180 INFO:teuthology.orchestra.run.smithi028.stdout: elfutils-libelf-devel-0.191-4.el9.x86_64 2024-05-20T19:50:43.181 INFO:teuthology.orchestra.run.smithi028.stdout: flex-2.6.4-9.el9.x86_64 2024-05-20T19:50:43.181 INFO:teuthology.orchestra.run.smithi028.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-05-20T19:50:43.181 INFO:teuthology.orchestra.run.smithi028.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-05-20T19:50:43.181 INFO:teuthology.orchestra.run.smithi028.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-05-20T19:50:43.181 INFO:teuthology.orchestra.run.smithi028.stdout: fuse-2.9.9-15.el9.x86_64 2024-05-20T19:50:43.181 INFO:teuthology.orchestra.run.smithi028.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-05-20T19:50:43.181 INFO:teuthology.orchestra.run.smithi028.stdout: grpc-data-1.46.7-10.el9.noarch 2024-05-20T19:50:43.182 INFO:teuthology.orchestra.run.smithi028.stdout: libaio-devel-0.3.111-13.el9.x86_64 2024-05-20T19:50:43.182 INFO:teuthology.orchestra.run.smithi028.stdout: libarrow-9.0.0-11.el9.x86_64 2024-05-20T19:50:43.182 INFO:teuthology.orchestra.run.smithi028.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-05-20T19:50:43.182 INFO:teuthology.orchestra.run.smithi028.stdout: libcephfs-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.182 INFO:teuthology.orchestra.run.smithi028.stdout: libcephfs2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.182 INFO:teuthology.orchestra.run.smithi028.stdout: libcephsqlite-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.182 INFO:teuthology.orchestra.run.smithi028.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: liboath-2.6.7-2.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: librados-devel-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: libradosstriper1-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: librgw2-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: libtool-2.4.6-45.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: libunwind-1.6.2-1.el9.x86_64 2024-05-20T19:50:43.183 INFO:teuthology.orchestra.run.smithi028.stdout: libuuid-devel-2.37.4-18.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: libzstd-devel-1.5.1-2.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: lua-5.4.4-4.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: luarocks-3.9.2-5.el9.noarch 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: openblas-0.3.21-2.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: openssl-devel-1:3.2.1-1.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql-13.14-1.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql-contrib-13.14-1.el9.x86_64 2024-05-20T19:50:43.184 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql-private-libs-13.14-1.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: postgresql-server-13.14-1.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: protobuf-3.14.0-13.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-babel-2.9.1-2.el9.noarch 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ceph-argparse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ceph-common-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cephfs-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.185 INFO:teuthology.orchestra.run.smithi028.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cmd2-2.4.2-2.el9.noarch 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-colorama-0.4.6-3.el9.noarch 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-05-20T19:50:43.186 INFO:teuthology.orchestra.run.smithi028.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-05-20T19:50:43.187 INFO:teuthology.orchestra.run.smithi028.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-05-20T19:50:43.187 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-05-20T19:50:43.187 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-05-20T19:50:43.187 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-05-20T19:50:43.187 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-05-20T19:50:43.188 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-05-20T19:50:43.188 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-05-20T19:50:43.188 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-05-20T19:50:43.188 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-05-20T19:50:43.188 INFO:teuthology.orchestra.run.smithi028.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-05-20T19:50:43.188 INFO:teuthology.orchestra.run.smithi028.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-05-20T19:50:43.189 INFO:teuthology.orchestra.run.smithi028.stdout: python3-mako-1.1.4-6.el9.noarch 2024-05-20T19:50:43.189 INFO:teuthology.orchestra.run.smithi028.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-05-20T19:50:43.189 INFO:teuthology.orchestra.run.smithi028.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-05-20T19:50:43.189 INFO:teuthology.orchestra.run.smithi028.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-05-20T19:50:43.189 INFO:teuthology.orchestra.run.smithi028.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-05-20T19:50:43.189 INFO:teuthology.orchestra.run.smithi028.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-05-20T19:50:43.189 INFO:teuthology.orchestra.run.smithi028.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-05-20T19:50:43.190 INFO:teuthology.orchestra.run.smithi028.stdout: python3-packaging-20.9-5.el9.noarch 2024-05-20T19:50:43.190 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-05-20T19:50:43.190 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ply-3.11-14.el9.noarch 2024-05-20T19:50:43.190 INFO:teuthology.orchestra.run.smithi028.stdout: python3-portend-3.1.0-2.el9.noarch 2024-05-20T19:50:43.190 INFO:teuthology.orchestra.run.smithi028.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-05-20T19:50:43.190 INFO:teuthology.orchestra.run.smithi028.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pycparser-2.20-6.el9.noarch 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pyperclip-1.8.2-2.el9.noarch 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pytz-2021.1-5.el9.noarch 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rados-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.191 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.192 INFO:teuthology.orchestra.run.smithi028.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-05-20T19:50:43.192 INFO:teuthology.orchestra.run.smithi028.stdout: python3-requests-2.25.1-8.el9.noarch 2024-05-20T19:50:43.192 INFO:teuthology.orchestra.run.smithi028.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-05-20T19:50:43.192 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rgw-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.192 INFO:teuthology.orchestra.run.smithi028.stdout: python3-routes-2.5.1-5.el9.noarch 2024-05-20T19:50:43.192 INFO:teuthology.orchestra.run.smithi028.stdout: python3-rsa-4.9-2.el9.noarch 2024-05-20T19:50:43.193 INFO:teuthology.orchestra.run.smithi028.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-05-20T19:50:43.193 INFO:teuthology.orchestra.run.smithi028.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-05-20T19:50:43.193 INFO:teuthology.orchestra.run.smithi028.stdout: python3-toml-0.10.2-6.el9.noarch 2024-05-20T19:50:43.193 INFO:teuthology.orchestra.run.smithi028.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-05-20T19:50:43.193 INFO:teuthology.orchestra.run.smithi028.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-05-20T19:50:43.193 INFO:teuthology.orchestra.run.smithi028.stdout: python3-wcwidth-0.2.5-8.el9.noarch 2024-05-20T19:50:43.194 INFO:teuthology.orchestra.run.smithi028.stdout: python3-webob-1.8.7-6.el9.noarch 2024-05-20T19:50:43.194 INFO:teuthology.orchestra.run.smithi028.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-05-20T19:50:43.194 INFO:teuthology.orchestra.run.smithi028.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-05-20T19:50:43.194 INFO:teuthology.orchestra.run.smithi028.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-05-20T19:50:43.194 INFO:teuthology.orchestra.run.smithi028.stdout: qatlib-24.02.0-1.el9.x86_64 2024-05-20T19:50:43.194 INFO:teuthology.orchestra.run.smithi028.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: rbd-fuse-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: rbd-mirror-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: rbd-nbd-2:19.0.0-3830.g4ebd1d53.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: re2-1:20211101-3.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: socat-1.7.4.1-5.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: thrift-0.15.0-2.el9.x86_64 2024-05-20T19:50:43.195 INFO:teuthology.orchestra.run.smithi028.stdout: uuid-1.6.2-55.el9.x86_64 2024-05-20T19:50:43.196 INFO:teuthology.orchestra.run.smithi028.stdout: xfsprogs-devel-6.3.0-1.el9.x86_64 2024-05-20T19:50:43.196 INFO:teuthology.orchestra.run.smithi028.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-05-20T19:50:43.196 INFO:teuthology.orchestra.run.smithi028.stdout: zlib-devel-1.2.11-41.el9.x86_64 2024-05-20T19:50:43.196 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:50:43.196 INFO:teuthology.orchestra.run.smithi028.stdout:Complete! 2024-05-20T19:50:43.965 DEBUG:teuthology.parallel:result is None 2024-05-20T19:50:43.966 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:50:44.184 DEBUG:teuthology.orchestra.run.smithi028:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-05-20T19:50:44.244 INFO:teuthology.orchestra.run.smithi028.stdout:19.0.0-3830.g4ebd1d53.el9 2024-05-20T19:50:44.245 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-3830.g4ebd1d53.el9 2024-05-20T19:50:44.245 INFO:teuthology.task.install:The correct ceph version 19.0.0-3830.g4ebd1d53 is installed. 2024-05-20T19:50:44.247 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:50:44.385 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-05-20T19:50:44.411 INFO:teuthology.orchestra.run.smithi033.stdout:19.0.0-3830.g4ebd1d53.el9 2024-05-20T19:50:44.412 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-3830.g4ebd1d53.el9 2024-05-20T19:50:44.412 INFO:teuthology.task.install:The correct ceph version 19.0.0-3830.g4ebd1d53 is installed. 2024-05-20T19:50:44.415 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:50:44.629 DEBUG:teuthology.orchestra.run.smithi154:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-05-20T19:50:44.655 INFO:teuthology.orchestra.run.smithi154.stdout:19.0.0-3830.g4ebd1d53.el9 2024-05-20T19:50:44.655 INFO:teuthology.packaging:The installed version of ceph is 19.0.0-3830.g4ebd1d53.el9 2024-05-20T19:50:44.656 INFO:teuthology.task.install:The correct ceph version 19.0.0-3830.g4ebd1d53 is installed. 2024-05-20T19:50:44.657 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-05-20T19:50:44.658 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:50:44.658 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-05-20T19:50:44.734 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:50:44.734 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-05-20T19:50:44.766 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:50:44.766 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-05-20T19:50:44.798 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-05-20T19:50:44.798 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:50:44.798 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/usr/bin/daemon-helper 2024-05-20T19:50:44.865 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-05-20T19:50:44.974 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:50:44.974 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/daemon-helper 2024-05-20T19:50:45.004 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-05-20T19:50:45.074 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:50:45.074 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/usr/bin/daemon-helper 2024-05-20T19:50:45.104 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-05-20T19:50:45.170 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-05-20T19:50:45.170 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:50:45.170 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/usr/bin/adjust-ulimits 2024-05-20T19:50:45.240 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-05-20T19:50:45.347 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:50:45.347 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/adjust-ulimits 2024-05-20T19:50:45.373 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-05-20T19:50:45.438 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:50:45.438 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/usr/bin/adjust-ulimits 2024-05-20T19:50:45.463 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-05-20T19:50:45.528 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-05-20T19:50:45.528 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:50:45.528 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/usr/bin/stdin-killer 2024-05-20T19:50:45.603 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-05-20T19:50:45.678 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:50:45.678 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/stdin-killer 2024-05-20T19:50:45.703 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-05-20T19:50:45.768 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:50:45.768 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/usr/bin/stdin-killer 2024-05-20T19:50:45.793 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-05-20T19:50:45.858 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-20T19:50:45.965 INFO:tasks.cephadm:Config: {'roleless': False, 'cephfs': {'max_mds': 5, 'session_timeout': 300, 'standby_replay': True}, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds bal fragment size max': 10000, 'mds bal merge size': 5, 'mds bal split bits': 3, 'mds bal split size': 100, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'mds_export_ephemeral_random_max': 0.1, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'cephsqlite lock renewal timeout': 900000, 'debug cephsqlite': 20, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon op complaint time': 120}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': 20, 'debug bluestore': 20, 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': 10, 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180, 'osd shutdown pgref assert': True, 'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'slow metadata IO', 'SLOW_OPS', 'slow request'], 'sha1': '4ebd1d5361d457738957dc0455bba90102296634', 'subvols': {'create': 2, 'subvol_options': '--namespace-isolated --size 25000000000'}} 2024-05-20T19:50:45.965 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:50:45.966 INFO:tasks.cephadm:Cluster fsid is 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:50:45.966 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-20T19:50:45.966 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.28', 'mon.b': '172.21.15.33', 'mon.c': '172.21.15.154'} 2024-05-20T19:50:45.966 INFO:tasks.cephadm:First mon is mon.a on smithi028 2024-05-20T19:50:45.967 INFO:tasks.cephadm:First mgr is x 2024-05-20T19:50:45.967 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-20T19:50:45.967 DEBUG:teuthology.orchestra.run.smithi028:> sudo hostname $(hostname -s) 2024-05-20T19:50:46.041 DEBUG:teuthology.orchestra.run.smithi033:> sudo hostname $(hostname -s) 2024-05-20T19:50:46.066 DEBUG:teuthology.orchestra.run.smithi154:> sudo hostname $(hostname -s) 2024-05-20T19:50:46.089 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2024-05-20T19:50:46.090 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:50:46.307 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '4ebd1d5361d457738957dc0455bba90102296634', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/79554/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-3830-g4ebd1d53', 'node_name': '172.21.5.40+adami10', 'job_name': 'ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-3830.g4ebd1d53'}, 'url': 'https://3.chacra.ceph.com/r/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-05-20 18:01:42.330101', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-lusov-quiesce', 'chacra_url': 'https://3.chacra.ceph.com/repos/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2024-05-20T19:50:46.524 INFO:tasks.util.chacra:got chacra host 3.chacra.ceph.com, ref wip-lusov-quiesce, sha1 4ebd1d5361d457738957dc0455bba90102296634 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&sha1=4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:50:46.526 INFO:tasks.cephadm:Discovered cachra url: https://3.chacra.ceph.com/binaries/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/x86_64/flavors/default/cephadm 2024-05-20T19:50:46.526 INFO:tasks.cephadm:Downloading cephadm from url: https://3.chacra.ceph.com/binaries/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/x86_64/flavors/default/cephadm 2024-05-20T19:50:46.526 DEBUG:teuthology.orchestra.run.smithi028:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:47.844 INFO:teuthology.orchestra.run.smithi028.stdout:-rw-r--r--. 1 ubuntu ubuntu 782820 May 20 19:50 /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:47.845 DEBUG:teuthology.orchestra.run.smithi033:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:48.941 INFO:teuthology.orchestra.run.smithi033.stdout:-rw-r--r--. 1 ubuntu ubuntu 782820 May 20 19:50 /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:48.941 DEBUG:teuthology.orchestra.run.smithi154:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/wip-lusov-quiesce/4ebd1d5361d457738957dc0455bba90102296634/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:50.293 INFO:teuthology.orchestra.run.smithi154.stdout:-rw-r--r--. 1 ubuntu ubuntu 782820 May 20 19:50 /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:50.294 DEBUG:teuthology.orchestra.run.smithi028:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:50.356 DEBUG:teuthology.orchestra.run.smithi033:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:50.376 DEBUG:teuthology.orchestra.run.smithi154:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-20T19:50:50.410 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 on all hosts... 2024-05-20T19:50:50.411 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 pull 2024-05-20T19:50:50.418 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 pull 2024-05-20T19:50:50.421 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 pull 2024-05-20T19:50:51.222 INFO:teuthology.orchestra.run.smithi033.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634... 2024-05-20T19:50:51.232 INFO:teuthology.orchestra.run.smithi154.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634... 2024-05-20T19:50:51.675 INFO:teuthology.orchestra.run.smithi028.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634... 2024-05-20T19:51:22.952 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-05-20T19:51:22.952 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_version": "ceph version 19.0.0-3830-g4ebd1d53 (4ebd1d5361d457738957dc0455bba90102296634) squid (dev)", 2024-05-20T19:51:22.952 INFO:teuthology.orchestra.run.smithi154.stdout: "image_id": "559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489", 2024-05-20T19:51:22.952 INFO:teuthology.orchestra.run.smithi154.stdout: "repo_digests": [ 2024-05-20T19:51:22.953 INFO:teuthology.orchestra.run.smithi154.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3" 2024-05-20T19:51:22.953 INFO:teuthology.orchestra.run.smithi154.stdout: ] 2024-05-20T19:51:22.953 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-05-20T19:51:26.757 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-05-20T19:51:26.757 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_version": "ceph version 19.0.0-3830-g4ebd1d53 (4ebd1d5361d457738957dc0455bba90102296634) squid (dev)", 2024-05-20T19:51:26.757 INFO:teuthology.orchestra.run.smithi033.stdout: "image_id": "559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489", 2024-05-20T19:51:26.757 INFO:teuthology.orchestra.run.smithi033.stdout: "repo_digests": [ 2024-05-20T19:51:26.757 INFO:teuthology.orchestra.run.smithi033.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3" 2024-05-20T19:51:26.757 INFO:teuthology.orchestra.run.smithi033.stdout: ] 2024-05-20T19:51:26.758 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-05-20T19:51:30.576 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T19:51:30.576 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_version": "ceph version 19.0.0-3830-g4ebd1d53 (4ebd1d5361d457738957dc0455bba90102296634) squid (dev)", 2024-05-20T19:51:30.576 INFO:teuthology.orchestra.run.smithi028.stdout: "image_id": "559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489", 2024-05-20T19:51:30.576 INFO:teuthology.orchestra.run.smithi028.stdout: "repo_digests": [ 2024-05-20T19:51:30.577 INFO:teuthology.orchestra.run.smithi028.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3" 2024-05-20T19:51:30.577 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T19:51:30.577 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T19:51:30.646 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /etc/ceph 2024-05-20T19:51:30.744 DEBUG:teuthology.orchestra.run.smithi033:> sudo mkdir -p /etc/ceph 2024-05-20T19:51:30.776 DEBUG:teuthology.orchestra.run.smithi154:> sudo mkdir -p /etc/ceph 2024-05-20T19:51:30.808 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 777 /etc/ceph 2024-05-20T19:51:30.905 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 777 /etc/ceph 2024-05-20T19:51:30.935 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 777 /etc/ceph 2024-05-20T19:51:30.963 INFO:tasks.cephadm:Writing seed config... 2024-05-20T19:51:30.965 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2024-05-20T19:51:30.965 INFO:tasks.cephadm: override: [client] debug client = 20 2024-05-20T19:51:30.965 INFO:tasks.cephadm: override: [client] debug ms = 1 2024-05-20T19:51:30.965 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2024-05-20T19:51:30.965 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2024-05-20T19:51:30.965 INFO:tasks.cephadm: override: [global] ms die on skipped message = False 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] debug mds = 20 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] debug mds balancer = 20 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] debug ms = 1 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds bal fragment size max = 10000 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds bal merge size = 5 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds bal split bits = 3 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds bal split size = 100 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2024-05-20T19:51:30.966 INFO:tasks.cephadm: override: [mds] mds_export_ephemeral_random_max = 0.1 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mgr] cephsqlite lock renewal timeout = 900000 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mgr] debug cephsqlite = 20 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mgr] debug client = 20 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mon] mon down mkfs grace = 300 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2024-05-20T19:51:30.967 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] bluestore compression mode = aggressive 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] debug bluefs = 20 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] debug bluestore = 20 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] debug rocksdb = 10 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2024-05-20T19:51:30.968 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2024-05-20T19:51:30.969 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-05-20T19:51:30.969 INFO:tasks.cephadm: override: [osd] osd_max_omap_entries_per_request = 10000 2024-05-20T19:51:30.970 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:51:30.970 DEBUG:teuthology.orchestra.run.smithi028:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-20T19:51:31.033 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 3fbec9b2-16e2-11ef-bc9a-c7b262605968 ms die on skipped message = False [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true bluestore block size = 96636764160 bluestore compression mode = aggressive bluestore fsck on mount = True debug bluefs = 20 debug bluestore = 20 debug ms = 1 debug osd = 20 debug rocksdb = 10 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd objectstore = bluestore osd op complaint time = 180 osd_max_omap_entries_per_request = 10000 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false cephsqlite lock renewal timeout = 900000 debug cephsqlite = 20 debug client = 20 debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon down mkfs grace = 300 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug mds balancer = 20 debug ms = 1 mds bal fragment size max = 10000 mds bal merge size = 5 mds bal split bits = 3 mds bal split size = 100 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True mds_export_ephemeral_random_max = 0.1 osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2024-05-20T19:51:31.034 DEBUG:teuthology.orchestra.run.smithi028:mon.a> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a.service 2024-05-20T19:51:31.081 DEBUG:teuthology.orchestra.run.smithi028:mgr.x> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.x.service 2024-05-20T19:51:31.128 INFO:tasks.cephadm:Bootstrapping... 2024-05-20T19:51:31.128 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 -v bootstrap --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.28 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-20T19:51:31.402 INFO:teuthology.orchestra.run.smithi028.stdout:-------------------------------------------------------------------------------- 2024-05-20T19:51:31.402 INFO:teuthology.orchestra.run.smithi028.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634', '-v', 'bootstrap', '--fsid', '3fbec9b2-16e2-11ef-bc9a-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.28', '--skip-admin-label'] 2024-05-20T19:51:31.402 INFO:teuthology.orchestra.run.smithi028.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-05-20T19:51:31.402 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying podman|docker is present... 2024-05-20T19:51:31.460 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stdout 5.0.2 2024-05-20T19:51:31.460 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying lvm2 is present... 2024-05-20T19:51:31.460 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying time synchronization is in place... 2024-05-20T19:51:31.478 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-05-20T19:51:31.479 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-05-20T19:51:31.494 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-05-20T19:51:31.495 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout inactive 2024-05-20T19:51:31.513 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout enabled 2024-05-20T19:51:31.532 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout active 2024-05-20T19:51:31.533 INFO:teuthology.orchestra.run.smithi028.stdout:Unit chronyd.service is enabled and running 2024-05-20T19:51:31.533 INFO:teuthology.orchestra.run.smithi028.stdout:Repeating the final host check... 2024-05-20T19:51:31.596 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stdout 5.0.2 2024-05-20T19:51:31.596 INFO:teuthology.orchestra.run.smithi028.stdout:podman (/bin/podman) version 5.0.2 is present 2024-05-20T19:51:31.596 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl is present 2024-05-20T19:51:31.596 INFO:teuthology.orchestra.run.smithi028.stdout:lvcreate is present 2024-05-20T19:51:31.611 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-05-20T19:51:31.611 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-05-20T19:51:31.625 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-05-20T19:51:31.626 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout inactive 2024-05-20T19:51:31.642 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout enabled 2024-05-20T19:51:31.660 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout active 2024-05-20T19:51:31.660 INFO:teuthology.orchestra.run.smithi028.stdout:Unit chronyd.service is enabled and running 2024-05-20T19:51:31.660 INFO:teuthology.orchestra.run.smithi028.stdout:Host looks OK 2024-05-20T19:51:31.660 INFO:teuthology.orchestra.run.smithi028.stdout:Cluster fsid: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:51:31.661 INFO:teuthology.orchestra.run.smithi028.stdout:Acquiring lock 140393149465360 on /run/cephadm/3fbec9b2-16e2-11ef-bc9a-c7b262605968.lock 2024-05-20T19:51:31.661 INFO:teuthology.orchestra.run.smithi028.stdout:Lock 140393149465360 acquired on /run/cephadm/3fbec9b2-16e2-11ef-bc9a-c7b262605968.lock 2024-05-20T19:51:31.661 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying IP 172.21.15.28 port 3300 ... 2024-05-20T19:51:31.662 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying IP 172.21.15.28 port 6789 ... 2024-05-20T19:51:31.662 INFO:teuthology.orchestra.run.smithi028.stdout:Base mon IP(s) is [172.21.15.28:3300, 172.21.15.28:6789], mon addrv is [v2:172.21.15.28:3300,v1:172.21.15.28:6789] 2024-05-20T19:51:31.670 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.28 metric 100 2024-05-20T19:51:31.670 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.28 metric 100 2024-05-20T19:51:31.676 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-05-20T19:51:31.677 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-05-20T19:51:31.677 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-05-20T19:51:31.683 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-20T19:51:31.683 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-05-20T19:51:31.683 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-05-20T19:51:31.683 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-05-20T19:51:31.683 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fea5:ca3c/64 scope link noprefixroute 2024-05-20T19:51:31.683 INFO:teuthology.orchestra.run.smithi028.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-05-20T19:51:31.684 INFO:teuthology.orchestra.run.smithi028.stdout:Mon IP `172.21.15.28` is in CIDR network `172.21.0.0/20` 2024-05-20T19:51:31.684 INFO:teuthology.orchestra.run.smithi028.stdout:Mon IP `172.21.15.28` is in CIDR network `172.21.0.0/20` 2024-05-20T19:51:31.684 INFO:teuthology.orchestra.run.smithi028.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-05-20T19:51:31.685 INFO:teuthology.orchestra.run.smithi028.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-20T19:51:31.686 INFO:teuthology.orchestra.run.smithi028.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634... 2024-05-20T19:51:32.361 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stdout 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 2024-05-20T19:51:32.361 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634... 2024-05-20T19:51:32.361 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stderr Getting image source signatures 2024-05-20T19:51:32.361 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stderr Copying blob sha256:9af6dedc48c4fca9f79c2db09a94f41f0f515ee6ded78101f148d319f313a254 2024-05-20T19:51:32.361 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stderr Copying blob sha256:d0e808cd455827ca3108db1b0125dcee9ffc148daa1d7dde963f550bc831d501 2024-05-20T19:51:32.362 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stderr Copying config sha256:559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 2024-05-20T19:51:32.362 INFO:teuthology.orchestra.run.smithi028.stdout:/bin/podman: stderr Writing manifest to image destination 2024-05-20T19:51:33.656 INFO:teuthology.orchestra.run.smithi028.stdout:ceph: stdout ceph version 19.0.0-3830-g4ebd1d53 (4ebd1d5361d457738957dc0455bba90102296634) squid (dev) 2024-05-20T19:51:33.656 INFO:teuthology.orchestra.run.smithi028.stdout:Ceph version: ceph version 19.0.0-3830-g4ebd1d53 (4ebd1d5361d457738957dc0455bba90102296634) squid (dev) 2024-05-20T19:51:33.656 INFO:teuthology.orchestra.run.smithi028.stdout:Extracting ceph user uid/gid from container image... 2024-05-20T19:51:35.097 INFO:teuthology.orchestra.run.smithi028.stdout:stat: stdout 167 167 2024-05-20T19:51:35.097 INFO:teuthology.orchestra.run.smithi028.stdout:Creating initial keys... 2024-05-20T19:51:36.191 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph-authtool: stdout AQDHqUtmkHQNIRAA8MDLRKAd1GQUu9mJh7usgg== 2024-05-20T19:51:37.402 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph-authtool: stdout AQDIqUtmJK7RJRAA+v7wEdeaxdutxv5VqOxz3w== 2024-05-20T19:51:38.518 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph-authtool: stdout AQDJqUtm0MP8MxAAfMbVyrYhwySQ7Z04y2tQwg== 2024-05-20T19:51:38.519 INFO:teuthology.orchestra.run.smithi028.stdout:Creating initial monmap... 2024-05-20T19:51:48.834 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-20T19:51:48.834 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = quincy 2024-05-20T19:51:48.834 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:51:48.834 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-20T19:51:48.834 INFO:teuthology.orchestra.run.smithi028.stdout:monmaptool for a [v2:172.21.15.28:3300,v1:172.21.15.28:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-20T19:51:48.834 INFO:teuthology.orchestra.run.smithi028.stdout:setting min_mon_release = quincy 2024-05-20T19:51:48.835 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/monmaptool: set fsid to 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:51:48.835 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-20T19:51:48.835 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:51:48.835 INFO:teuthology.orchestra.run.smithi028.stdout:Creating mon... 2024-05-20T19:51:50.294 INFO:teuthology.orchestra.run.smithi028.stdout:create mon.a on 2024-05-20T19:51:50.697 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-05-20T19:51:50.945 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-20T19:51:51.234 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968.target → /etc/systemd/system/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968.target. 2024-05-20T19:51:51.235 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968.target → /etc/systemd/system/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968.target. 2024-05-20T19:51:51.628 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a 2024-05-20T19:51:51.628 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Failed to reset failed state of unit ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a.service: Unit ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a.service not loaded. 2024-05-20T19:51:51.933 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968.target.wants/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a.service → /etc/systemd/system/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@.service. 2024-05-20T19:51:52.660 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:52 smithi028 podman[30797]: 2024-05-20 19:51:52.362211979 +0000 UTC m=+0.145771185 container create f68965f1aa54a74156fec57108f41200ba221521897b377d7b9a6f9c32008b5a (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-20T19:51:52.660 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:52 smithi028 podman[30797]: 2024-05-20 19:51:52.657804608 +0000 UTC m=+0.441363797 container init f68965f1aa54a74156fec57108f41200ba221521897b377d7b9a6f9c32008b5a (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-20T19:51:52.779 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-05-20T19:51:52.779 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout disabled 2024-05-20T19:51:52.795 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-05-20T19:51:52.795 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout inactive 2024-05-20T19:51:52.795 INFO:teuthology.orchestra.run.smithi028.stdout:firewalld.service is not enabled 2024-05-20T19:51:52.795 INFO:teuthology.orchestra.run.smithi028.stdout:Not possible to enable service . firewalld.service is not available 2024-05-20T19:51:52.795 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for mon to start... 2024-05-20T19:51:52.795 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for mon... 2024-05-20T19:51:52.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:52 smithi028 podman[30797]: 2024-05-20 19:51:52.668779739 +0000 UTC m=+0.452338951 container start f68965f1aa54a74156fec57108f41200ba221521897b377d7b9a6f9c32008b5a (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-20T19:51:52.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:52 smithi028 bash[30797]: f68965f1aa54a74156fec57108f41200ba221521897b377d7b9a6f9c32008b5a 2024-05-20T19:51:52.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:52 smithi028 ceph-mon[30815]: mkfs 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:51:52.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:52 smithi028 ceph-mon[30815]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-20T19:51:52.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:52 smithi028 systemd[1]: Started Ceph mon.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: monmap epoch 1 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: last_changed 2024-05-20T19:51:39.025045+0000 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: min_mon_release 19 (squid) 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: election_strategy: 1 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: fsmap 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: osdmap e1: 0 total, 0 up, 0 in 2024-05-20T19:51:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: mgrmap e1: no daemons active 2024-05-20T19:51:54.057 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:53 smithi028 ceph-mon[30815]: from='client.? 172.21.15.28:0/3686008564' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T19:51:54.337 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout cluster: 2024-05-20T19:51:54.337 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout id: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:51:54.337 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-05-20T19:51:54.337 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:51:54.338 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout services: 2024-05-20T19:51:54.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.997667s) 2024-05-20T19:51:54.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-05-20T19:51:54.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout data: 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout pgs: 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.649+0000 7f7bf7000700 1 Processor -- start 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.649+0000 7f7bf7000700 1 -- start start 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.650+0000 7f7bf7000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf0109460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.650+0000 7f7bf7000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7bf01099a0 con 0x7f7bf0109040 2024-05-20T19:51:54.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.651+0000 7f7befe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf0109460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.651+0000 7f7befe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf0109460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40872/0 (socket says 172.21.15.28:40872) 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.651+0000 7f7befe00700 1 -- 172.21.15.28:0/1002983020 learned_addr learned my addr 172.21.15.28:0/1002983020 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.652+0000 7f7befe00700 1 -- 172.21.15.28:0/1002983020 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7bf0109b20 con 0x7f7bf0109040 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.652+0000 7f7befe00700 1 --2- 172.21.15.28:0/1002983020 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf0109460 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f7be00098a0 tx=0x7f7be0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9db60f1c405e610e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.652+0000 7f7bef400700 1 -- 172.21.15.28:0/1002983020 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7be0004160 con 0x7f7bf0109040 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.652+0000 7f7bef400700 1 -- 172.21.15.28:0/1002983020 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f7be0004300 con 0x7f7bf0109040 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.652+0000 7f7bef400700 1 -- 172.21.15.28:0/1002983020 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7be00036a0 con 0x7f7bf0109040 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.653+0000 7f7bf7000700 1 -- 172.21.15.28:0/1002983020 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 msgr2=0x7f7bf0109460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:51:54.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.653+0000 7f7bf7000700 1 --2- 172.21.15.28:0/1002983020 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf0109460 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f7be00098a0 tx=0x7f7be0009bb0 comp rx=0 tx=0).stop 2024-05-20T19:51:54.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.653+0000 7f7bf7000700 1 -- 172.21.15.28:0/1002983020 shutdown_connections 2024-05-20T19:51:54.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.653+0000 7f7bf7000700 1 --2- 172.21.15.28:0/1002983020 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf0109460 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:51:54.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.653+0000 7f7bf7000700 1 -- 172.21.15.28:0/1002983020 >> 172.21.15.28:0/1002983020 conn(0x7f7bf0104680 msgr2=0x7f7bf0106ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:51:54.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.653+0000 7f7bf7000700 1 -- 172.21.15.28:0/1002983020 shutdown_connections 2024-05-20T19:51:54.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.654+0000 7f7bf7000700 1 -- 172.21.15.28:0/1002983020 wait complete. 2024-05-20T19:51:54.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.655+0000 7f7bf7000700 1 Processor -- start 2024-05-20T19:51:54.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.655+0000 7f7bf7000700 1 -- start start 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.655+0000 7f7bf7000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf019c690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.655+0000 7f7bf7000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7bf010a4f0 con 0x7f7bf0109040 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.655+0000 7f7befe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf019c690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.655+0000 7f7befe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf019c690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40878/0 (socket says 172.21.15.28:40878) 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.655+0000 7f7befe00700 1 -- 172.21.15.28:0/3686008564 learned_addr learned my addr 172.21.15.28:0/3686008564 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.656+0000 7f7befe00700 1 -- 172.21.15.28:0/3686008564 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7bf019cbd0 con 0x7f7bf0109040 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.656+0000 7f7befe00700 1 --2- 172.21.15.28:0/3686008564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf019c690 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f7be0000c00 tx=0x7f7be0004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.656+0000 7f7bed600700 1 -- 172.21.15.28:0/3686008564 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7be0003db0 con 0x7f7bf0109040 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.656+0000 7f7bed600700 1 -- 172.21.15.28:0/3686008564 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f7be0011630 con 0x7f7bf0109040 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.656+0000 7f7bed600700 1 -- 172.21.15.28:0/3686008564 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7be0011950 con 0x7f7bf0109040 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.656+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7bf019ce60 con 0x7f7bf0109040 2024-05-20T19:51:54.347 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.657+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7bf019fb50 con 0x7f7bf0109040 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.657+0000 7f7bed600700 1 -- 172.21.15.28:0/3686008564 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f7be0011af0 con 0x7f7bf0109040 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.657+0000 7f7bed600700 1 -- 172.21.15.28:0/3686008564 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f7be001c450 con 0x7f7bf0109040 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.658+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7bf0064ba0 con 0x7f7bf0109040 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.661+0000 7f7bed600700 1 -- 172.21.15.28:0/3686008564 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73389 (secure 0 0 0) 0x7f7be0004260 con 0x7f7bf0109040 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.742+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "status"} v 0) -- 0x7f7bf002e660 con 0x7f7bf0109040 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.742+0000 7f7bed600700 1 -- 172.21.15.28:0/3686008564 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) ==== 54+0+317 (secure 0 0 0) 0x7f7be001b450 con 0x7f7bf0109040 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.744+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 msgr2=0x7f7bf019c690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.744+0000 7f7bf7000700 1 --2- 172.21.15.28:0/3686008564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf019c690 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f7be0000c00 tx=0x7f7be0004750 comp rx=0 tx=0).stop 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.744+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 shutdown_connections 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.744+0000 7f7bf7000700 1 --2- 172.21.15.28:0/3686008564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bf0109040 0x7f7bf019c690 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.744+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 >> 172.21.15.28:0/3686008564 conn(0x7f7bf0104680 msgr2=0x7f7bf01932f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:51:54.348 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.744+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 shutdown_connections 2024-05-20T19:51:54.349 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:53.744+0000 7f7bf7000700 1 -- 172.21.15.28:0/3686008564 wait complete. 2024-05-20T19:51:54.349 INFO:teuthology.orchestra.run.smithi028.stdout:mon is available 2024-05-20T19:51:54.349 INFO:teuthology.orchestra.run.smithi028.stdout:Assimilating anything we can from ceph.conf... 2024-05-20T19:51:55.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:55 smithi028 ceph-mon[30815]: from='client.? 172.21.15.28:0/3690166549' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-20T19:51:55.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:55 smithi028 ceph-mon[30815]: from='client.? 172.21.15.28:0/3690166549' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-20T19:51:55.692 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:51:55.692 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout [global] 2024-05-20T19:51:55.692 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout fsid = 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:51:55.692 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-05-20T19:51:55.692 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.28:3300,v1:172.21.15.28:6789] 2024-05-20T19:51:55.692 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-05-20T19:51:55.692 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout [mgr] 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout [osd] 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-05-20T19:51:55.693 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.944+0000 7f0ec9a00700 1 Processor -- start 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.945+0000 7f0ec9a00700 1 -- start start 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.945+0000 7f0ec9a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec410b630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.945+0000 7f0ec9a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0ec410bc00 con 0x7f0ec410b210 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.946+0000 7f0ec2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec410b630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.946+0000 7f0ec2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec410b630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40882/0 (socket says 172.21.15.28:40882) 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.946+0000 7f0ec2a00700 1 -- 172.21.15.28:0/2787139879 learned_addr learned my addr 172.21.15.28:0/2787139879 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.946+0000 7f0ec2a00700 1 -- 172.21.15.28:0/2787139879 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0ec410c410 con 0x7f0ec410b210 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.947+0000 7f0ec2a00700 1 --2- 172.21.15.28:0/2787139879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec410b630 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f0eb40098a0 tx=0x7f0eb4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=19d7f6167b5b022a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.947+0000 7f0ec1600700 1 -- 172.21.15.28:0/2787139879 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0eb4004160 con 0x7f0ec410b210 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.947+0000 7f0ec1600700 1 -- 172.21.15.28:0/2787139879 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f0eb4004300 con 0x7f0ec410b210 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.947+0000 7f0ec1600700 1 -- 172.21.15.28:0/2787139879 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0eb40036a0 con 0x7f0ec410b210 2024-05-20T19:51:55.694 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.948+0000 7f0ec9a00700 1 -- 172.21.15.28:0/2787139879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 msgr2=0x7f0ec410b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.948+0000 7f0ec9a00700 1 --2- 172.21.15.28:0/2787139879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec410b630 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f0eb40098a0 tx=0x7f0eb4009bb0 comp rx=0 tx=0).stop 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.948+0000 7f0ec9a00700 1 -- 172.21.15.28:0/2787139879 shutdown_connections 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.948+0000 7f0ec9a00700 1 --2- 172.21.15.28:0/2787139879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec410b630 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.948+0000 7f0ec9a00700 1 -- 172.21.15.28:0/2787139879 >> 172.21.15.28:0/2787139879 conn(0x7f0ec407e650 msgr2=0x7f0ec407eaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.948+0000 7f0ec9a00700 1 -- 172.21.15.28:0/2787139879 shutdown_connections 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.948+0000 7f0ec9a00700 1 -- 172.21.15.28:0/2787139879 wait complete. 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.949+0000 7f0ec9a00700 1 Processor -- start 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.949+0000 7f0ec9a00700 1 -- start start 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.950+0000 7f0ec9a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec419e950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.950+0000 7f0ec9a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0ec410c940 con 0x7f0ec410b210 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.950+0000 7f0ec2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec419e950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.950+0000 7f0ec2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec419e950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40888/0 (socket says 172.21.15.28:40888) 2024-05-20T19:51:55.695 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.950+0000 7f0ec2a00700 1 -- 172.21.15.28:0/3690166549 learned_addr learned my addr 172.21.15.28:0/3690166549 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.950+0000 7f0ec2a00700 1 -- 172.21.15.28:0/3690166549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0ec419ee90 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ec2a00700 1 --2- 172.21.15.28:0/3690166549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec419e950 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f0eb4000c00 tx=0x7f0eb4004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0eb4003db0 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(0 keys) ==== 4+0+0 (secure 0 0 0) 0x7f0eb4011630 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0eb4011950 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0ec419f120 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0ec41a1e10 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f0eb401a460 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.951+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f0eb401b780 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.952+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0ec4064ba0 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:54.955+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73389 (secure 0 0 0) 0x7f0eb401b980 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.034+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) -- 0x7f0ec419f600 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.041+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) ==== 70+0+465 (secure 0 0 0) 0x7f0ec419f600 con 0x7f0ec410b210 2024-05-20T19:51:55.696 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.042+0000 7f0ebb400700 1 -- 172.21.15.28:0/3690166549 <== mon.0 v2:172.21.15.28:3300/0 8 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f0eb401a8e0 con 0x7f0ec410b210 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.042+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 msgr2=0x7f0ec419e950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.042+0000 7f0ec9a00700 1 --2- 172.21.15.28:0/3690166549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec419e950 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f0eb4000c00 tx=0x7f0eb4004750 comp rx=0 tx=0).stop 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.042+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 shutdown_connections 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.042+0000 7f0ec9a00700 1 --2- 172.21.15.28:0/3690166549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ec410b210 0x7f0ec419e950 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.042+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 >> 172.21.15.28:0/3690166549 conn(0x7f0ec407e650 msgr2=0x7f0ec41955c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.043+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 shutdown_connections 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:55.043+0000 7f0ec9a00700 1 -- 172.21.15.28:0/3690166549 wait complete. 2024-05-20T19:51:55.697 INFO:teuthology.orchestra.run.smithi028.stdout:Generating new minimal ceph.conf... 2024-05-20T19:51:56.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:56 smithi028 ceph-mon[30815]: from='client.? 172.21.15.28:0/1812241684' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:51:57.030 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.274+0000 7f081dc00700 1 Processor -- start 2024-05-20T19:51:57.030 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.274+0000 7f081dc00700 1 -- start start 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.275+0000 7f081dc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081810b630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.275+0000 7f081dc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f081810bc00 con 0x7f081810b210 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.275+0000 7f0816a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081810b630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.275+0000 7f0816a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081810b630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40904/0 (socket says 172.21.15.28:40904) 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.275+0000 7f0816a00700 1 -- 172.21.15.28:0/2062339599 learned_addr learned my addr 172.21.15.28:0/2062339599 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.276+0000 7f0816a00700 1 -- 172.21.15.28:0/2062339599 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f081810c410 con 0x7f081810b210 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.276+0000 7f0816a00700 1 --2- 172.21.15.28:0/2062339599 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081810b630 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f08080098a0 tx=0x7f0808009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9bca294ffb964c78 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.276+0000 7f0816000700 1 -- 172.21.15.28:0/2062339599 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0808004160 con 0x7f081810b210 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.276+0000 7f0816000700 1 -- 172.21.15.28:0/2062339599 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f0808004300 con 0x7f081810b210 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.276+0000 7f0816000700 1 -- 172.21.15.28:0/2062339599 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0808011af0 con 0x7f081810b210 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.277+0000 7f081dc00700 1 -- 172.21.15.28:0/2062339599 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 msgr2=0x7f081810b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:51:57.031 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.277+0000 7f081dc00700 1 --2- 172.21.15.28:0/2062339599 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081810b630 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f08080098a0 tx=0x7f0808009bb0 comp rx=0 tx=0).stop 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.277+0000 7f081dc00700 1 -- 172.21.15.28:0/2062339599 shutdown_connections 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.277+0000 7f081dc00700 1 --2- 172.21.15.28:0/2062339599 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081810b630 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.277+0000 7f081dc00700 1 -- 172.21.15.28:0/2062339599 >> 172.21.15.28:0/2062339599 conn(0x7f081807e650 msgr2=0x7f081807eaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.277+0000 7f081dc00700 1 -- 172.21.15.28:0/2062339599 shutdown_connections 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.277+0000 7f081dc00700 1 -- 172.21.15.28:0/2062339599 wait complete. 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.278+0000 7f081dc00700 1 Processor -- start 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.278+0000 7f081dc00700 1 -- start start 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.279+0000 7f081dc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081819f1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.279+0000 7f081dc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f081810f7c0 con 0x7f081810b210 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.279+0000 7f0816a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081819f1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.279+0000 7f0816a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081819f1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40914/0 (socket says 172.21.15.28:40914) 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.279+0000 7f0816a00700 1 -- 172.21.15.28:0/1812241684 learned_addr learned my addr 172.21.15.28:0/1812241684 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:51:57.032 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.279+0000 7f0816a00700 1 -- 172.21.15.28:0/1812241684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f081819f730 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.280+0000 7f0816a00700 1 --2- 172.21.15.28:0/1812241684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081819f1f0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f08080117b0 tx=0x7f0808011b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.280+0000 7f080fe00700 1 -- 172.21.15.28:0/1812241684 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0808003af0 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.280+0000 7f080fe00700 1 -- 172.21.15.28:0/1812241684 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f0808020070 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.280+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f081819f9c0 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.280+0000 7f080fe00700 1 -- 172.21.15.28:0/1812241684 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0808019630 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.280+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f08181a26b0 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.280+0000 7f080fe00700 1 -- 172.21.15.28:0/1812241684 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f0808019c30 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.281+0000 7f080fe00700 1 -- 172.21.15.28:0/1812241684 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f080800b720 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.282+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0818064ba0 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.285+0000 7f080fe00700 1 -- 172.21.15.28:0/1812241684 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73389 (secure 0 0 0) 0x7f08080042a0 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.362+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) -- 0x7f08181a2b50 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.363+0000 7f080fe00700 1 -- 172.21.15.28:0/1812241684 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) ==== 76+0+175 (secure 0 0 0) 0x7f080801a9c0 con 0x7f081810b210 2024-05-20T19:51:57.033 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.364+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 msgr2=0x7f081819f1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:51:57.034 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.364+0000 7f081dc00700 1 --2- 172.21.15.28:0/1812241684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081819f1f0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f08080117b0 tx=0x7f0808011b70 comp rx=0 tx=0).stop 2024-05-20T19:51:57.034 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.364+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 shutdown_connections 2024-05-20T19:51:57.034 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.364+0000 7f081dc00700 1 --2- 172.21.15.28:0/1812241684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f081810b210 0x7f081819f1f0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:51:57.034 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.364+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 >> 172.21.15.28:0/1812241684 conn(0x7f081807e650 msgr2=0x7f0818108e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:51:57.034 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.364+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 shutdown_connections 2024-05-20T19:51:57.034 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:51:56.365+0000 7f081dc00700 1 -- 172.21.15.28:0/1812241684 wait complete. 2024-05-20T19:51:57.034 INFO:teuthology.orchestra.run.smithi028.stdout:Restarting the monitor... 2024-05-20T19:51:57.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:57 smithi028 systemd[1]: Stopping Ceph mon.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T19:51:57.916 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a[30811]: 2024-05-20T19:51:57.591+0000 7fc4a7a00700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T19:51:57.917 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a[30811]: 2024-05-20T19:51:57.591+0000 7fc4a7a00700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-05-20T19:51:57.917 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:57 smithi028 podman[31056]: 2024-05-20 19:51:57.643976364 +0000 UTC m=+0.205895857 container died f68965f1aa54a74156fec57108f41200ba221521897b377d7b9a6f9c32008b5a (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True) 2024-05-20T19:51:58.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:58 smithi028 podman[31056]: 2024-05-20 19:51:58.088766015 +0000 UTC m=+0.650685562 container cleanup f68965f1aa54a74156fec57108f41200ba221521897b377d7b9a6f9c32008b5a (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T19:51:58.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:58 smithi028 bash[31056]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a 2024-05-20T19:51:59.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:58 smithi028 podman[31067]: 2024-05-20 19:51:58.743663053 +0000 UTC m=+1.079677841 container remove f68965f1aa54a74156fec57108f41200ba221521897b377d7b9a6f9c32008b5a (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-05-20T19:51:59.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a.service: Deactivated successfully. 2024-05-20T19:51:59.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 systemd[1]: Stopped Ceph mon.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:51:59.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a.service: Consumed 2.069s CPU time. 2024-05-20T19:51:59.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 systemd[1]: Starting Ceph mon.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T19:51:59.899 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 podman[31160]: 2024-05-20 19:51:59.644344062 +0000 UTC m=+0.131465307 container create ea4da3bc48153474d3c2669c839a61eb5ff45779cc6c5684da9482da426a7dea (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-20T19:51:59.899 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 podman[31160]: 2024-05-20 19:51:59.551132664 +0000 UTC m=+0.038253874 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:51:59.899 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 podman[31160]: 2024-05-20 19:51:59.898251488 +0000 UTC m=+0.385372739 container init ea4da3bc48153474d3c2669c839a61eb5ff45779cc6c5684da9482da426a7dea (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514) 2024-05-20T19:51:59.997 INFO:teuthology.orchestra.run.smithi028.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-05-20T19:52:00.281 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 podman[31160]: 2024-05-20 19:51:59.909519557 +0000 UTC m=+0.396640808 container start ea4da3bc48153474d3c2669c839a61eb5ff45779cc6c5684da9482da426a7dea (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514) 2024-05-20T19:52:00.281 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T19:52:00.281 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: ceph version 19.0.0-3830-g4ebd1d53 (4ebd1d5361d457738957dc0455bba90102296634) squid (dev), process ceph-mon, pid 2 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: pidfile_write: ignore empty --pid-file 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: load: jerasure load: lrc 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: RocksDB version: 7.9.2 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Git sha 0 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Compile date 2024-05-20 17:11:59 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: DB SUMMARY 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: DB Session ID: 2I6YT93WNY6KCRZD1C7J 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: CURRENT file: CURRENT 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 105130 ; 2024-05-20T19:52:00.282 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.error_if_exists: 0 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.create_if_missing: 0 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.paranoid_checks: 1 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.env: 0x564b7f947cc0 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.fs: PosixFileSystem 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.info_log: 0x564b80289de0 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.statistics: (nil) 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.use_fsync: 0 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_log_file_size: 0 2024-05-20T19:52:00.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T19:52:00.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T19:52:00.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T19:52:00.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T19:52:00.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.allow_fallocate: 1 2024-05-20T19:52:00.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.use_direct_reads: 0 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.db_log_dir: 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.wal_dir: 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T19:52:00.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.write_buffer_manager: 0x564b8025b0e0 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.unordered_write: 0 2024-05-20T19:52:00.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.row_cache: None 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.wal_filter: None 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.two_write_queues: 0 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.wal_compression: 0 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.atomic_flush: 0 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T19:52:00.287 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T19:52:00.288 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T19:52:00.288 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.log_readahead_size: 0 2024-05-20T19:52:00.288 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T19:52:00.288 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T19:52:00.288 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T19:52:00.288 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T19:52:00.288 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_background_jobs: 2 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_background_compactions: -1 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_subcompactions: 1 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T19:52:00.289 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_open_files: -1 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_background_flushes: -1 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Compression algorithms supported: 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kZSTD supported: 0 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kXpressCompression supported: 0 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T19:52:00.290 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kZlibCompression supported: 1 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kSnappyCompression supported: 1 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kLZ4Compression supported: 1 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: kBZip2Compression supported: 0 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.merge_operator: 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_filter: None 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T19:52:00.291 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564b80289fa0) 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: cache_index_and_filter_blocks: 1 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: pin_top_level_index_and_filter: 1 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: index_type: 0 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: data_block_index_type: 0 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: index_shortening: 1 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: checksum: 4 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: no_block_cache: 0 2024-05-20T19:52:00.292 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_cache: 0x564b8028d1f0 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_cache_name: BinnedLRUCache 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_cache_options: 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: capacity : 536870912 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: num_shard_bits : 4 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: strict_capacity_limit : 0 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: high_pri_pool_ratio: 0.000 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_cache_compressed: (nil) 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: persistent_cache: (nil) 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_size: 4096 2024-05-20T19:52:00.293 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_size_deviation: 10 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_restart_interval: 16 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: index_block_restart_interval: 1 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: metadata_block_size: 4096 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: partition_filters: 0 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: use_delta_encoding: 1 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: filter_policy: bloomfilter 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: whole_key_filtering: 1 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: verify_compression: 0 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: read_amp_bytes_per_bit: 0 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: format_version: 5 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: enable_index_compression: 1 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: block_align: 0 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: max_auto_readahead_size: 262144 2024-05-20T19:52:00.294 INFO:journalctl@ceph.mon.a.smithi028.stdout: prepopulate_block_cache: 0 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout: initial_auto_readahead_size: 8192 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout: num_file_reads_for_auto_readahead: 2 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression: NoCompression 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.num_levels: 7 2024-05-20T19:52:00.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T19:52:00.296 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T19:52:00.296 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T19:52:00.296 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T19:52:00.296 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T19:52:00.297 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T19:52:00.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.arena_block_size: 1048576 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T19:52:00.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.inplace_update_support: 0 2024-05-20T19:52:00.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.bloom_locality: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.max_successive_merges: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.ttl: 2592000 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-20T19:52:00.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.enable_blob_files: false 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.min_blob_size: 0 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.blob_file_starting_level: 0 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-05-20T19:52:00.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dc451b42-9152-4ab9-8158-2f18c4570812 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716234719969292, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716234719971290, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 100297, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 326, "table_properties": {"data_size": 98182, "index_size": 307, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 14169, "raw_average_key_size": 49, "raw_value_size": 90466, "raw_average_value_size": 317, "num_data_blocks": 12, "num_entries": 285, "num_filter_entries": 285, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716234719, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dc451b42-9152-4ab9-8158-2f18c4570812", "db_session_id": "2I6YT93WNY6KCRZD1C7J", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716234719971453, "job": 1, "event": "recovery_finished"} 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564b803aa000 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: DB pointer 0x564b80396000 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout: ** DB Stats ** 2024-05-20T19:52:00.303 INFO:journalctl@ceph.mon.a.smithi028.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: ** Compaction Stats [default] ** 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: L0 2/0 99.81 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Sum 2/0 99.81 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-20T19:52:00.304 INFO:journalctl@ceph.mon.a.smithi028.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: ** Compaction Stats [default] ** 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T19:52:00.305 INFO:journalctl@ceph.mon.a.smithi028.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: Cumulative compaction: 0.00 GB write, 14.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: Interval compaction: 0.00 GB write, 14.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: Block cache BinnedLRUCache@0x564b8028d1f0#2 capacity: 512.00 MB usage: 1.52 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,1.02 KB,0.000193715%) IndexBlock(2,0.50 KB,9.53674e-05%) Misc(1,0.00 KB,0%) 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: starting mon.a rank 0 at public addrs [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] at bind addrs [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???) e1 preinit fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:52:00.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???).mds e1 new map 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???).mds e1 print_map 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout: e1 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout: legacy client fscid: -1 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout: 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout: No filesystems configured 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 ceph-mon[31178]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-20T19:52:00.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 bash[31160]: ea4da3bc48153474d3c2669c839a61eb5ff45779cc6c5684da9482da426a7dea 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:51:59 smithi028 systemd[1]: Started Ceph mon.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: mon.a is new leader, mons a in quorum (ranks 0) 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: monmap epoch 1 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: last_changed 2024-05-20T19:51:39.025045+0000 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: min_mon_release 19 (squid) 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: election_strategy: 1 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: fsmap 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: osdmap e1: 0 total, 0 up, 0 in 2024-05-20T19:52:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:00 smithi028 ceph-mon[31178]: mgrmap e1: no daemons active 2024-05-20T19:52:01.450 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.653+0000 7fb1f8800700 1 Processor -- start 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.653+0000 7fb1f8800700 1 -- start start 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.654+0000 7fb1f8800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f007d2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.654+0000 7fb1f8800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb1f007d7f0 con 0x7fb1f007ed40 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.655+0000 7fb1f5e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f007d2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.655+0000 7fb1f5e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f007d2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:46386/0 (socket says 172.21.15.28:46386) 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.655+0000 7fb1f5e00700 1 -- 172.21.15.28:0/513752324 learned_addr learned my addr 172.21.15.28:0/513752324 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.655+0000 7fb1f5e00700 1 -- 172.21.15.28:0/513752324 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1f007d970 con 0x7fb1f007ed40 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.656+0000 7fb1f5e00700 1 --2- 172.21.15.28:0/513752324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f007d2b0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fb1e40098a0 tx=0x7fb1e4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c6a903dbe9ed151 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.656+0000 7fb1f4a00700 1 -- 172.21.15.28:0/513752324 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1e4004160 con 0x7fb1f007ed40 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.656+0000 7fb1f4a00700 1 -- 172.21.15.28:0/513752324 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fb1e4004300 con 0x7fb1f007ed40 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.656+0000 7fb1f4a00700 1 -- 172.21.15.28:0/513752324 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1e4011af0 con 0x7fb1f007ed40 2024-05-20T19:52:01.451 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.657+0000 7fb1f8800700 1 -- 172.21.15.28:0/513752324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 msgr2=0x7fb1f007d2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.657+0000 7fb1f8800700 1 --2- 172.21.15.28:0/513752324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f007d2b0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fb1e40098a0 tx=0x7fb1e4009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.657+0000 7fb1f8800700 1 -- 172.21.15.28:0/513752324 shutdown_connections 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.657+0000 7fb1f8800700 1 --2- 172.21.15.28:0/513752324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f007d2b0 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.657+0000 7fb1f8800700 1 -- 172.21.15.28:0/513752324 >> 172.21.15.28:0/513752324 conn(0x7fb1f0104680 msgr2=0x7fb1f0106ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.657+0000 7fb1f8800700 1 -- 172.21.15.28:0/513752324 shutdown_connections 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.657+0000 7fb1f8800700 1 -- 172.21.15.28:0/513752324 wait complete. 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.658+0000 7fb1f8800700 1 Processor -- start 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.658+0000 7fb1f8800700 1 -- start start 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.659+0000 7fb1f8800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f019f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.659+0000 7fb1f8800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb1f010f460 con 0x7fb1f007ed40 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.659+0000 7fb1f5e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f019f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.659+0000 7fb1f5e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f019f140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:46390/0 (socket says 172.21.15.28:46390) 2024-05-20T19:52:01.452 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.659+0000 7fb1f5e00700 1 -- 172.21.15.28:0/1655355922 learned_addr learned my addr 172.21.15.28:0/1655355922 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.660+0000 7fb1f5e00700 1 -- 172.21.15.28:0/1655355922 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1f019f680 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.660+0000 7fb1f5e00700 1 --2- 172.21.15.28:0/1655355922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f019f140 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fb1e4011ac0 tx=0x7fb1e4011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.660+0000 7fb1eaa00700 1 -- 172.21.15.28:0/1655355922 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1e4003b20 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.660+0000 7fb1eaa00700 1 -- 172.21.15.28:0/1655355922 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fb1e401a8c0 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.660+0000 7fb1eaa00700 1 -- 172.21.15.28:0/1655355922 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1e40248d0 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.660+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb1f019f910 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.660+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb1f01a2600 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.661+0000 7fb1eaa00700 1 -- 172.21.15.28:0/1655355922 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fb1e4022020 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.661+0000 7fb1eaa00700 1 -- 172.21.15.28:0/1655355922 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fb1e40193c0 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.662+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb1f0064ba0 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.665+0000 7fb1eaa00700 1 -- 172.21.15.28:0/1655355922 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73389 (secure 0 0 0) 0x7fb1e4042ec0 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.742+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) -- 0x7fb1f01a27d0 con 0x7fb1f007ed40 2024-05-20T19:52:01.453 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.745+0000 7fb1eaa00700 1 -- 172.21.15.28:0/1655355922 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) ==== 124+0+0 (secure 0 0 0) 0x7fb1e4031020 con 0x7fb1f007ed40 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.746+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 msgr2=0x7fb1f019f140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.746+0000 7fb1f8800700 1 --2- 172.21.15.28:0/1655355922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f019f140 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fb1e4011ac0 tx=0x7fb1e4011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.746+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 shutdown_connections 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.746+0000 7fb1f8800700 1 --2- 172.21.15.28:0/1655355922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1f007ed40 0x7fb1f019f140 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.746+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 >> 172.21.15.28:0/1655355922 conn(0x7fb1f0104680 msgr2=0x7fb1f0105210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.746+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 shutdown_connections 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:00.746+0000 7fb1f8800700 1 -- 172.21.15.28:0/1655355922 wait complete. 2024-05-20T19:52:01.454 INFO:teuthology.orchestra.run.smithi028.stdout:Wrote config to /etc/ceph/ceph.conf 2024-05-20T19:52:01.476 INFO:teuthology.orchestra.run.smithi028.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-20T19:52:01.477 INFO:teuthology.orchestra.run.smithi028.stdout:Creating mgr... 2024-05-20T19:52:01.478 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying port 0.0.0.0:9283 ... 2024-05-20T19:52:01.478 INFO:teuthology.orchestra.run.smithi028.stdout:Verifying port 0.0.0.0:8765 ... 2024-05-20T19:52:01.955 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.x 2024-05-20T19:52:01.955 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Failed to reset failed state of unit ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.x.service: Unit ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.x.service not loaded. 2024-05-20T19:52:02.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:01 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1655355922' entity='client.admin' 2024-05-20T19:52:02.218 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968.target.wants/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.x.service → /etc/systemd/system/ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@.service. 2024-05-20T19:52:02.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:02 smithi028 systemd[1]: Starting Ceph mgr.x for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T19:52:02.934 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:02 smithi028 podman[31392]: 2024-05-20 19:52:02.531982278 +0000 UTC m=+0.033126712 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T19:52:02.934 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:02 smithi028 podman[31392]: 2024-05-20 19:52:02.654193013 +0000 UTC m=+0.155337437 container create f9f76a24a6606ebf569f656c2589b1febd68650f1ce00b86bafdf6865bc9e98b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-20T19:52:02.934 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:02 smithi028 podman[31392]: 2024-05-20 19:52:02.933174971 +0000 UTC m=+0.434319463 container init f9f76a24a6606ebf569f656c2589b1febd68650f1ce00b86bafdf6865bc9e98b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-20T19:52:03.074 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-05-20T19:52:03.075 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout disabled 2024-05-20T19:52:03.093 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-05-20T19:52:03.093 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout inactive 2024-05-20T19:52:03.093 INFO:teuthology.orchestra.run.smithi028.stdout:firewalld.service is not enabled 2024-05-20T19:52:03.093 INFO:teuthology.orchestra.run.smithi028.stdout:Not possible to enable service . firewalld.service is not available 2024-05-20T19:52:03.117 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-05-20T19:52:03.117 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout disabled 2024-05-20T19:52:03.135 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-05-20T19:52:03.135 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout inactive 2024-05-20T19:52:03.135 INFO:teuthology.orchestra.run.smithi028.stdout:firewalld.service is not enabled 2024-05-20T19:52:03.135 INFO:teuthology.orchestra.run.smithi028.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-05-20T19:52:03.135 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for mgr to start... 2024-05-20T19:52:03.135 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for mgr... 2024-05-20T19:52:03.204 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:02 smithi028 podman[31392]: 2024-05-20 19:52:02.941860209 +0000 UTC m=+0.443004636 container start f9f76a24a6606ebf569f656c2589b1febd68650f1ce00b86bafdf6865bc9e98b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True) 2024-05-20T19:52:03.204 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:03 smithi028 bash[31392]: f9f76a24a6606ebf569f656c2589b1febd68650f1ce00b86bafdf6865bc9e98b 2024-05-20T19:52:03.204 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:03 smithi028 systemd[1]: Started Ceph mgr.x for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:52:03.478 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:03.203+0000 7fc4f7cf3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T19:52:03.479 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:03.477+0000 7fc4f7cf3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:52:03.760 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:03.618+0000 7fc4f7cf3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T19:52:04.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:03 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3960230086' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T19:52:04.378 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:04 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:04.063+0000 7fc4f7cf3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T19:52:04.378 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:04 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:04.186+0000 7fc4f7cf3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T19:52:04.723 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:52:04.723 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:52:04.723 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "health": { 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 0 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "a" 2024-05-20T19:52:04.724 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "monmap": { 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-05-20T19:52:04.725 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-05-20T19:52:04.726 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "available": false, 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-05-20T19:52:04.727 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modules": [ 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "iostat", 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "nfs", 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "restful" 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modified": "2024-05-20T19:51:52.747334+0000", 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-05-20T19:52:04.728 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.759+0000 7f992a200700 1 Processor -- start 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.759+0000 7f992a200700 1 -- start start 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.759+0000 7f992a200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f9924109460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.759+0000 7f992a200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f99241099a0 con 0x7f9924109040 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.760+0000 7f9923400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f9924109460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.760+0000 7f9923400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f9924109460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:46432/0 (socket says 172.21.15.28:46432) 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.760+0000 7f9923400700 1 -- 172.21.15.28:0/3896447315 learned_addr learned my addr 172.21.15.28:0/3896447315 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:04.729 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.761+0000 7f9923400700 1 -- 172.21.15.28:0/3896447315 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9924109b20 con 0x7f9924109040 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.761+0000 7f9923400700 1 --2- 172.21.15.28:0/3896447315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f9924109460 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f99140098a0 tx=0x7f9914009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c3cae89fedb435ed server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.761+0000 7f9922000700 1 -- 172.21.15.28:0/3896447315 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9914004160 con 0x7f9924109040 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.761+0000 7f9922000700 1 -- 172.21.15.28:0/3896447315 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f9914004300 con 0x7f9924109040 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.761+0000 7f9922000700 1 -- 172.21.15.28:0/3896447315 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9914011aa0 con 0x7f9924109040 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.762+0000 7f992a200700 1 -- 172.21.15.28:0/3896447315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 msgr2=0x7f9924109460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.762+0000 7f992a200700 1 --2- 172.21.15.28:0/3896447315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f9924109460 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f99140098a0 tx=0x7f9914009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.762+0000 7f992a200700 1 -- 172.21.15.28:0/3896447315 shutdown_connections 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.762+0000 7f992a200700 1 --2- 172.21.15.28:0/3896447315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f9924109460 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.762+0000 7f992a200700 1 -- 172.21.15.28:0/3896447315 >> 172.21.15.28:0/3896447315 conn(0x7f9924104680 msgr2=0x7f9924106ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:04.730 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.763+0000 7f992a200700 1 -- 172.21.15.28:0/3896447315 shutdown_connections 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.763+0000 7f992a200700 1 -- 172.21.15.28:0/3896447315 wait complete. 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.764+0000 7f992a200700 1 Processor -- start 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.764+0000 7f992a200700 1 -- start start 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.764+0000 7f992a200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f992419cf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.764+0000 7f992a200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f992410afd0 con 0x7f9924109040 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.765+0000 7f9923400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f992419cf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.765+0000 7f9923400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f992419cf90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:46436/0 (socket says 172.21.15.28:46436) 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.765+0000 7f9923400700 1 -- 172.21.15.28:0/3960230086 learned_addr learned my addr 172.21.15.28:0/3960230086 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:04.731 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.765+0000 7f9923400700 1 -- 172.21.15.28:0/3960230086 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f992419d4d0 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.766+0000 7f9923400700 1 --2- 172.21.15.28:0/3960230086 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f992419cf90 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f9914011aa0 tx=0x7f9914011b80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.766+0000 7f991be00700 1 -- 172.21.15.28:0/3960230086 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f9914003a00 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.766+0000 7f991be00700 1 -- 172.21.15.28:0/3960230086 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f9914003ba0 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.766+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f992419d760 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.766+0000 7f991be00700 1 -- 172.21.15.28:0/3960230086 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f991401a6e0 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.766+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f99241a0400 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.766+0000 7f991be00700 1 -- 172.21.15.28:0/3960230086 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f991401ad00 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.767+0000 7f991be00700 1 -- 172.21.15.28:0/3960230086 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f9914019700 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.767+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9924064ba0 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.770+0000 7f991be00700 1 -- 172.21.15.28:0/3960230086 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73389 (secure 0 0 0) 0x7f99140042e0 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.850+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7f99241a08a0 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.851+0000 7f991be00700 1 -- 172.21.15.28:0/3960230086 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7f9914024b10 con 0x7f9924109040 2024-05-20T19:52:04.732 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.852+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 msgr2=0x7f992419cf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:04.733 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.852+0000 7f992a200700 1 --2- 172.21.15.28:0/3960230086 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f992419cf90 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f9914011aa0 tx=0x7f9914011b80 comp rx=0 tx=0).stop 2024-05-20T19:52:04.733 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.853+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 shutdown_connections 2024-05-20T19:52:04.733 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.853+0000 7f992a200700 1 --2- 172.21.15.28:0/3960230086 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9924109040 0x7f992419cf90 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:04.733 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.853+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 >> 172.21.15.28:0/3960230086 conn(0x7f9924104680 msgr2=0x7f99241051e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:04.733 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.853+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 shutdown_connections 2024-05-20T19:52:04.733 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:03.853+0000 7f992a200700 1 -- 172.21.15.28:0/3960230086 wait complete. 2024-05-20T19:52:04.733 INFO:teuthology.orchestra.run.smithi028.stdout:mgr not available, waiting (1/15)... 2024-05-20T19:52:06.303 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:05 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:05.836+0000 7fc4f7cf3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T19:52:06.303 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:06.070+0000 7fc4f7cf3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T19:52:06.303 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:06.301+0000 7fc4f7cf3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T19:52:07.037 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:06.691+0000 7fc4f7cf3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T19:52:07.037 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:06.818+0000 7fc4f7cf3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T19:52:07.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:07 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:07.080+0000 7fc4f7cf3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T19:52:07.769 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:07 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3061178641' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T19:52:07.770 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:07 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:07.357+0000 7fc4f7cf3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:52:08.097 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:52:08.097 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:52:08.097 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "health": { 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 0 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "a" 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2024-05-20T19:52:08.098 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "monmap": { 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-05-20T19:52:08.099 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-05-20T19:52:08.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-05-20T19:52:08.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-05-20T19:52:08.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-05-20T19:52:08.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:08.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-05-20T19:52:08.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-05-20T19:52:08.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "available": false, 2024-05-20T19:52:08.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-05-20T19:52:08.112 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modules": [ 2024-05-20T19:52:08.112 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "iostat", 2024-05-20T19:52:08.112 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "nfs", 2024-05-20T19:52:08.112 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "restful" 2024-05-20T19:52:08.112 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:08.113 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:08.113 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:08.113 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-05-20T19:52:08.113 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:08.113 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modified": "2024-05-20T19:51:52.747334+0000", 2024-05-20T19:52:08.113 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:08.113 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:08.114 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-05-20T19:52:08.114 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:52:08.114 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.345+0000 7f87a1200700 1 Processor -- start 2024-05-20T19:52:08.114 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.346+0000 7f87a1200700 1 -- start start 2024-05-20T19:52:08.114 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.346+0000 7f87a1200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c10be40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:08.114 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.346+0000 7f87a1200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f879c07ce40 con 0x7f879c109a50 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.347+0000 7f8799e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c10be40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.347+0000 7f8799e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c10be40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51734/0 (socket says 172.21.15.28:51734) 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.347+0000 7f8799e00700 1 -- 172.21.15.28:0/3354594447 learned_addr learned my addr 172.21.15.28:0/3354594447 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.347+0000 7f8799e00700 1 -- 172.21.15.28:0/3354594447 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f879c10c380 con 0x7f879c109a50 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.348+0000 7f8799e00700 1 --2- 172.21.15.28:0/3354594447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c10be40 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f878c0098a0 tx=0x7f878c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f683d1dbb6820c98 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.348+0000 7f8798a00700 1 -- 172.21.15.28:0/3354594447 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f878c004160 con 0x7f879c109a50 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.348+0000 7f8798a00700 1 -- 172.21.15.28:0/3354594447 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f878c004300 con 0x7f879c109a50 2024-05-20T19:52:08.115 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.348+0000 7f8798a00700 1 -- 172.21.15.28:0/3354594447 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f878c011af0 con 0x7f879c109a50 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.349+0000 7f87a1200700 1 -- 172.21.15.28:0/3354594447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 msgr2=0x7f879c10be40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.349+0000 7f87a1200700 1 --2- 172.21.15.28:0/3354594447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c10be40 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f878c0098a0 tx=0x7f878c009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.349+0000 7f87a1200700 1 -- 172.21.15.28:0/3354594447 shutdown_connections 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.349+0000 7f87a1200700 1 --2- 172.21.15.28:0/3354594447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c10be40 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.349+0000 7f87a1200700 1 -- 172.21.15.28:0/3354594447 >> 172.21.15.28:0/3354594447 conn(0x7f879c103480 msgr2=0x7f879c1058a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.349+0000 7f87a1200700 1 -- 172.21.15.28:0/3354594447 shutdown_connections 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.349+0000 7f87a1200700 1 -- 172.21.15.28:0/3354594447 wait complete. 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.351+0000 7f87a1200700 1 Processor -- start 2024-05-20T19:52:08.116 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.351+0000 7f87a1200700 1 -- start start 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.351+0000 7f87a1200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c19f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.351+0000 7f87a1200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f879c10d340 con 0x7f879c109a50 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.351+0000 7f8799e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c19f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.352+0000 7f8799e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c19f110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51736/0 (socket says 172.21.15.28:51736) 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.352+0000 7f8799e00700 1 -- 172.21.15.28:0/3061178641 learned_addr learned my addr 172.21.15.28:0/3061178641 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.352+0000 7f8799e00700 1 -- 172.21.15.28:0/3061178641 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f879c19f650 con 0x7f879c109a50 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.352+0000 7f8799e00700 1 --2- 172.21.15.28:0/3061178641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c19f110 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f878c011ac0 tx=0x7f878c011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.352+0000 7f8792a00700 1 -- 172.21.15.28:0/3061178641 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f878c003b20 con 0x7f879c109a50 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.353+0000 7f8792a00700 1 -- 172.21.15.28:0/3061178641 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f878c01a8c0 con 0x7f879c109a50 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.353+0000 7f8792a00700 1 -- 172.21.15.28:0/3061178641 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f878c0248d0 con 0x7f879c109a50 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.353+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f879c19f8e0 con 0x7f879c109a50 2024-05-20T19:52:08.117 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.353+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f879c1a25d0 con 0x7f879c109a50 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.353+0000 7f8792a00700 1 -- 172.21.15.28:0/3061178641 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f878c022020 con 0x7f879c109a50 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.353+0000 7f8792a00700 1 -- 172.21.15.28:0/3061178641 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f878c0193c0 con 0x7f879c109a50 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.354+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f879c064ba0 con 0x7f879c109a50 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.357+0000 7f8792a00700 1 -- 172.21.15.28:0/3061178641 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73389 (secure 0 0 0) 0x7f878c02ddf0 con 0x7f879c109a50 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.435+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7f879c1a2a70 con 0x7f879c109a50 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.435+0000 7f8792a00700 1 -- 172.21.15.28:0/3061178641 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7f878c01aa70 con 0x7f879c109a50 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.436+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 msgr2=0x7f879c19f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.436+0000 7f87a1200700 1 --2- 172.21.15.28:0/3061178641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c19f110 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f878c011ac0 tx=0x7f878c011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.437+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 shutdown_connections 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.437+0000 7f87a1200700 1 --2- 172.21.15.28:0/3061178641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f879c109a50 0x7f879c19f110 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.437+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 >> 172.21.15.28:0/3061178641 conn(0x7f879c103480 msgr2=0x7f879c103f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.437+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 shutdown_connections 2024-05-20T19:52:08.118 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:07.437+0000 7f87a1200700 1 -- 172.21.15.28:0/3061178641 wait complete. 2024-05-20T19:52:08.119 INFO:teuthology.orchestra.run.smithi028.stdout:mgr not available, waiting (2/15)... 2024-05-20T19:52:08.522 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:08 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:08.222+0000 7fc4f7cf3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T19:52:08.522 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:08 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:08.519+0000 7fc4f7cf3200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T19:52:09.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:09.091+0000 7fc4f7cf3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T19:52:09.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:09.221+0000 7fc4f7cf3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T19:52:10.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:09.693+0000 7fc4f7cf3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T19:52:10.440 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:10.165+0000 7fc4f7cf3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T19:52:10.440 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:10.293+0000 7fc4f7cf3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T19:52:10.740 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:10.633+0000 7fc4f7cf3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T19:52:11.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:10 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/808343251' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T19:52:11.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:10.822+0000 7fc4f7cf3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T19:52:11.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:10.958+0000 7fc4f7cf3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T19:52:11.502 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:11.370+0000 7fc4f7cf3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T19:52:11.566 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:52:11.566 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:52:11.566 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T19:52:11.566 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "health": { 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 0 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "a" 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-05-20T19:52:11.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "monmap": { 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:11.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-05-20T19:52:11.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "available": false, 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modules": [ 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "iostat", 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "nfs", 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "restful" 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:11.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modified": "2024-05-20T19:51:52.747334+0000", 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.738+0000 7f376be00700 1 Processor -- start 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.738+0000 7f376be00700 1 -- start start 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.738+0000 7f376be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c07d2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.738+0000 7f376be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f376c07d7f0 con 0x7f376c07ed40 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.739+0000 7f376aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c07d2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.739+0000 7f376aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c07d2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51742/0 (socket says 172.21.15.28:51742) 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.739+0000 7f376aa00700 1 -- 172.21.15.28:0/3415424037 learned_addr learned my addr 172.21.15.28:0/3415424037 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.739+0000 7f376aa00700 1 -- 172.21.15.28:0/3415424037 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f376c07d970 con 0x7f376c07ed40 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.740+0000 7f376aa00700 1 --2- 172.21.15.28:0/3415424037 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c07d2b0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f375c0098a0 tx=0x7f375c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ac82c4b148762c4e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:11.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.740+0000 7f3769600700 1 -- 172.21.15.28:0/3415424037 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f375c004160 con 0x7f376c07ed40 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.740+0000 7f3769600700 1 -- 172.21.15.28:0/3415424037 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f375c004300 con 0x7f376c07ed40 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.740+0000 7f3769600700 1 -- 172.21.15.28:0/3415424037 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f375c011af0 con 0x7f376c07ed40 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.741+0000 7f376be00700 1 -- 172.21.15.28:0/3415424037 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 msgr2=0x7f376c07d2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.741+0000 7f376be00700 1 --2- 172.21.15.28:0/3415424037 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c07d2b0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f375c0098a0 tx=0x7f375c009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.741+0000 7f376be00700 1 -- 172.21.15.28:0/3415424037 shutdown_connections 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.741+0000 7f376be00700 1 --2- 172.21.15.28:0/3415424037 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c07d2b0 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.741+0000 7f376be00700 1 -- 172.21.15.28:0/3415424037 >> 172.21.15.28:0/3415424037 conn(0x7f376c104460 msgr2=0x7f376c1068c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.741+0000 7f376be00700 1 -- 172.21.15.28:0/3415424037 shutdown_connections 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.741+0000 7f376be00700 1 -- 172.21.15.28:0/3415424037 wait complete. 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.742+0000 7f376be00700 1 Processor -- start 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.742+0000 7f376be00700 1 -- start start 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.743+0000 7f376be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c19ef90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.743+0000 7f376be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f376c10f2d0 con 0x7f376c07ed40 2024-05-20T19:52:11.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.743+0000 7f376aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c19ef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.743+0000 7f376aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c19ef90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51746/0 (socket says 172.21.15.28:51746) 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.743+0000 7f376aa00700 1 -- 172.21.15.28:0/808343251 learned_addr learned my addr 172.21.15.28:0/808343251 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.743+0000 7f376aa00700 1 -- 172.21.15.28:0/808343251 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f376c19f4d0 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.743+0000 7f376aa00700 1 --2- 172.21.15.28:0/808343251 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c19ef90 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f375c011ac0 tx=0x7f375c011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.744+0000 7f3763400700 1 -- 172.21.15.28:0/808343251 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f375c003970 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.744+0000 7f3763400700 1 -- 172.21.15.28:0/808343251 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f375c003b10 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.744+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f376c19f760 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.744+0000 7f3763400700 1 -- 172.21.15.28:0/808343251 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f375c024610 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.744+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f376c1a2450 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.744+0000 7f3763400700 1 -- 172.21.15.28:0/808343251 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f375c024bd0 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.744+0000 7f3763400700 1 -- 172.21.15.28:0/808343251 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f375c019930 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.745+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f376c06a670 con 0x7f376c07ed40 2024-05-20T19:52:11.573 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.748+0000 7f3763400700 1 -- 172.21.15.28:0/808343251 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+73389 (secure 0 0 0) 0x7f375c043ec0 con 0x7f376c07ed40 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.829+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7f376c1a2620 con 0x7f376c07ed40 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.830+0000 7f3763400700 1 -- 172.21.15.28:0/808343251 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1240 (secure 0 0 0) 0x7f375c032030 con 0x7f376c07ed40 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.831+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 msgr2=0x7f376c19ef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.831+0000 7f376be00700 1 --2- 172.21.15.28:0/808343251 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c19ef90 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f375c011ac0 tx=0x7f375c011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.831+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 shutdown_connections 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.831+0000 7f376be00700 1 --2- 172.21.15.28:0/808343251 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f376c07ed40 0x7f376c19ef90 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.831+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 >> 172.21.15.28:0/808343251 conn(0x7f376c104460 msgr2=0x7f376c105080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.831+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 shutdown_connections 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:10.832+0000 7f376be00700 1 -- 172.21.15.28:0/808343251 wait complete. 2024-05-20T19:52:11.574 INFO:teuthology.orchestra.run.smithi028.stdout:mgr not available, waiting (3/15)... 2024-05-20T19:52:11.804 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:11.500+0000 7fc4f7cf3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T19:52:12.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:12.294+0000 7fc4f7cf3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T19:52:13.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:12.586+0000 7fc4f7cf3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T19:52:13.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: Activating manager daemon x 2024-05-20T19:52:13.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: mgrmap e2: x(active, starting, since 0.0124434s) 2024-05-20T19:52:13.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T19:52:13.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T19:52:13.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T19:52:13.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:52:13.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T19:52:13.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: Manager daemon x is now available 2024-05-20T19:52:13.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T19:52:13.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:12 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T19:52:13.908 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:13 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' 2024-05-20T19:52:13.909 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:13 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' 2024-05-20T19:52:13.909 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:13 smithi028 ceph-mon[31178]: from='mgr.14100 172.21.15.28:0/963047242' entity='mgr.x' 2024-05-20T19:52:13.909 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:13 smithi028 ceph-mon[31178]: mgrmap e3: x(active, since 1.01196s) 2024-05-20T19:52:15.329 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:52:15.329 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:52:15.329 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T19:52:15.329 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "health": { 2024-05-20T19:52:15.329 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-05-20T19:52:15.329 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 0 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "a" 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "monmap": { 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:15.330 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "squid", 2024-05-20T19:52:15.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-05-20T19:52:15.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:15.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-05-20T19:52:15.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:15.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-05-20T19:52:15.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-05-20T19:52:15.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-05-20T19:52:15.340 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "available": true, 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modules": [ 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "iostat", 2024-05-20T19:52:15.341 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "nfs", 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "restful" 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ], 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "modified": "2024-05-20T19:51:52.747334+0000", 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "services": {} 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout }, 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.322+0000 7fdc95800700 1 Processor -- start 2024-05-20T19:52:15.342 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.323+0000 7fdc95800700 1 -- start start 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.323+0000 7fdc95800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc90109430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.323+0000 7fdc95800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdc90109a00 con 0x7fdc90109010 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.323+0000 7fdc8ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc90109430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.323+0000 7fdc8ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc90109430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51820/0 (socket says 172.21.15.28:51820) 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.323+0000 7fdc8ea00700 1 -- 172.21.15.28:0/3757525835 learned_addr learned my addr 172.21.15.28:0/3757525835 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.324+0000 7fdc8ea00700 1 -- 172.21.15.28:0/3757525835 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdc9010a210 con 0x7fdc90109010 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.324+0000 7fdc8ea00700 1 --2- 172.21.15.28:0/3757525835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc90109430 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fdc800098a0 tx=0x7fdc80009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=30715a0b81d89ed0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.325+0000 7fdc8e000700 1 -- 172.21.15.28:0/3757525835 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc80004160 con 0x7fdc90109010 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.325+0000 7fdc8e000700 1 -- 172.21.15.28:0/3757525835 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fdc80004300 con 0x7fdc90109010 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.325+0000 7fdc8e000700 1 -- 172.21.15.28:0/3757525835 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc80011af0 con 0x7fdc90109010 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.325+0000 7fdc95800700 1 -- 172.21.15.28:0/3757525835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 msgr2=0x7fdc90109430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.326+0000 7fdc95800700 1 --2- 172.21.15.28:0/3757525835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc90109430 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fdc800098a0 tx=0x7fdc80009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:15.343 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.326+0000 7fdc95800700 1 -- 172.21.15.28:0/3757525835 shutdown_connections 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.326+0000 7fdc95800700 1 --2- 172.21.15.28:0/3757525835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc90109430 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.326+0000 7fdc95800700 1 -- 172.21.15.28:0/3757525835 >> 172.21.15.28:0/3757525835 conn(0x7fdc90104590 msgr2=0x7fdc901069f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.326+0000 7fdc95800700 1 -- 172.21.15.28:0/3757525835 shutdown_connections 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.326+0000 7fdc95800700 1 -- 172.21.15.28:0/3757525835 wait complete. 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.327+0000 7fdc95800700 1 Processor -- start 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.328+0000 7fdc95800700 1 -- start start 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.328+0000 7fdc95800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc9007f3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.328+0000 7fdc95800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdc9010d5c0 con 0x7fdc90109010 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.328+0000 7fdc8ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc9007f3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.329+0000 7fdc8ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc9007f3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51824/0 (socket says 172.21.15.28:51824) 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.329+0000 7fdc8ea00700 1 -- 172.21.15.28:0/4031377494 learned_addr learned my addr 172.21.15.28:0/4031377494 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.329+0000 7fdc8ea00700 1 -- 172.21.15.28:0/4031377494 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdc9007f910 con 0x7fdc90109010 2024-05-20T19:52:15.344 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.329+0000 7fdc8ea00700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc9007f3d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fdc800117b0 tx=0x7fdc80011b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.330+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc80003af0 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.330+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fdc80020070 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.330+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdc9007db00 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.330+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fdc80019630 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.330+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdc9007e040 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.332+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 3) ==== 50105+0+0 (secure 0 0 0) 0x7fdc8001a9c0 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.331+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdc90064ba0 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.332+0000 7fdc87e00700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fdc7403e350 0x7fdc74040810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.332+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fdc80053850 con 0x7fdc90109010 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.332+0000 7fdc86000700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fdc7403e350 0x7fdc74040810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.333+0000 7fdc86000700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fdc7403e350 0x7fdc74040810 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fdc78006fd0 tx=0x7fdc78006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:15.345 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.339+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fdc800042a0 con 0x7fdc90109010 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.653+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mgrmap(e 4) ==== 50211+0+0 (secure 0 0 0) 0x7fdc80055ce0 con 0x7fdc90109010 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.676+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) -- 0x7fdc9007e2c0 con 0x7fdc90109010 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.677+0000 7fdc87e00700 1 -- 172.21.15.28:0/4031377494 <== mon.0 v2:172.21.15.28:3300/0 8 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) ==== 79+0+1239 (secure 0 0 0) 0x7fdc8002b260 con 0x7fdc90109010 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fdc7403e350 msgr2=0x7fdc74040810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fdc7403e350 0x7fdc74040810 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fdc78006fd0 tx=0x7fdc78006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 msgr2=0x7fdc9007f3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc9007f3d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fdc800117b0 tx=0x7fdc80011b70 comp rx=0 tx=0).stop 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 shutdown_connections 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fdc7403e350 0x7fdc74040810 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 --2- 172.21.15.28:0/4031377494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdc90109010 0x7fdc9007f3d0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 >> 172.21.15.28:0/4031377494 conn(0x7fdc90104590 msgr2=0x7fdc90105160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 shutdown_connections 2024-05-20T19:52:15.346 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:14.681+0000 7fdc95800700 1 -- 172.21.15.28:0/4031377494 wait complete. 2024-05-20T19:52:15.347 INFO:teuthology.orchestra.run.smithi028.stdout:mgr is available 2024-05-20T19:52:15.946 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:15 smithi028 ceph-mon[31178]: mgrmap e4: x(active, since 2s) 2024-05-20T19:52:15.946 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:15 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/4031377494' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T19:52:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:16 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3261961858' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-20T19:52:16.835 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:52:16.835 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout [global] 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout fsid = 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout [mgr] 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout [osd] 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-05-20T19:52:16.836 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.944+0000 7f5707c00700 1 Processor -- start 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.945+0000 7f5707c00700 1 -- start start 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.945+0000 7f5707c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f57001094a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.945+0000 7f5707c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5700109a70 con 0x7f5700109080 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.946+0000 7f5705200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f57001094a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.946+0000 7f5705200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f57001094a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51832/0 (socket says 172.21.15.28:51832) 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.946+0000 7f5705200700 1 -- 172.21.15.28:0/4082182343 learned_addr learned my addr 172.21.15.28:0/4082182343 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.946+0000 7f5705200700 1 -- 172.21.15.28:0/4082182343 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f570010a230 con 0x7f5700109080 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.946+0000 7f5705200700 1 --2- 172.21.15.28:0/4082182343 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f57001094a0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f56f40098a0 tx=0x7f56f4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4f0dcb2be8f59668 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.947+0000 7f56ff400700 1 -- 172.21.15.28:0/4082182343 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f56f4004160 con 0x7f5700109080 2024-05-20T19:52:16.837 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.947+0000 7f56ff400700 1 -- 172.21.15.28:0/4082182343 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f56f4004300 con 0x7f5700109080 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.947+0000 7f56ff400700 1 -- 172.21.15.28:0/4082182343 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f56f4011af0 con 0x7f5700109080 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.947+0000 7f5707c00700 1 -- 172.21.15.28:0/4082182343 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 msgr2=0x7f57001094a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.947+0000 7f5707c00700 1 --2- 172.21.15.28:0/4082182343 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f57001094a0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f56f40098a0 tx=0x7f56f4009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.948+0000 7f5707c00700 1 -- 172.21.15.28:0/4082182343 shutdown_connections 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.948+0000 7f5707c00700 1 --2- 172.21.15.28:0/4082182343 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f57001094a0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.948+0000 7f5707c00700 1 -- 172.21.15.28:0/4082182343 >> 172.21.15.28:0/4082182343 conn(0x7f5700104620 msgr2=0x7f5700106a60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.948+0000 7f5707c00700 1 -- 172.21.15.28:0/4082182343 shutdown_connections 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.948+0000 7f5707c00700 1 -- 172.21.15.28:0/4082182343 wait complete. 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.949+0000 7f5707c00700 1 Processor -- start 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.949+0000 7f5707c00700 1 -- start start 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.950+0000 7f5707c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f570007f3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.950+0000 7f5707c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f570010d5e0 con 0x7f5700109080 2024-05-20T19:52:16.838 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.950+0000 7f5705200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f570007f3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.950+0000 7f5705200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f570007f3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51848/0 (socket says 172.21.15.28:51848) 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.950+0000 7f5705200700 1 -- 172.21.15.28:0/3261961858 learned_addr learned my addr 172.21.15.28:0/3261961858 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.950+0000 7f5705200700 1 -- 172.21.15.28:0/3261961858 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f570007f910 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.951+0000 7f5705200700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f570007f3d0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f56f40117b0 tx=0x7f56f4011b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.951+0000 7f56fd600700 1 -- 172.21.15.28:0/3261961858 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f56f4003af0 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.951+0000 7f56fd600700 1 -- 172.21.15.28:0/3261961858 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f56f4020070 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.951+0000 7f56fd600700 1 -- 172.21.15.28:0/3261961858 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f56f4019630 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.951+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f570007db00 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.951+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f570007e040 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.952+0000 7f56fd600700 1 -- 172.21.15.28:0/3261961858 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 4) ==== 50211+0+0 (secure 0 0 0) 0x7f56f401a9c0 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.952+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5700064ba0 con 0x7f5700109080 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.953+0000 7f56fd600700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f56e803e470 0x7f56e8040930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:16.839 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.953+0000 7f56fd600700 1 -- 172.21.15.28:0/3261961858 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f56f4053a60 con 0x7f5700109080 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.954+0000 7f56ffe00700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f56e803e470 0x7f56e8040930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.954+0000 7f56ffe00700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f56e803e470 0x7f56e8040930 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f56ec006fd0 tx=0x7f56ec006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:15.959+0000 7f56fd600700 1 -- 172.21.15.28:0/3261961858 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f56f40042a0 con 0x7f5700109080 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.193+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) -- 0x7f570007e2c0 con 0x7f5700109080 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.194+0000 7f56fd600700 1 -- 172.21.15.28:0/3261961858 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v3) ==== 70+0+409 (secure 0 0 0) 0x7f56f40172b0 con 0x7f5700109080 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f56e803e470 msgr2=0x7f56e8040930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f56e803e470 0x7f56e8040930 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f56ec006fd0 tx=0x7f56ec006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 msgr2=0x7f570007f3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f570007f3d0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f56f40117b0 tx=0x7f56f4011b70 comp rx=0 tx=0).stop 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 shutdown_connections 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f56e803e470 0x7f56e8040930 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 --2- 172.21.15.28:0/3261961858 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5700109080 0x7f570007f3d0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.198+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 >> 172.21.15.28:0/3261961858 conn(0x7f5700104620 msgr2=0x7f5700105180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:16.840 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.199+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 shutdown_connections 2024-05-20T19:52:16.841 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:16.199+0000 7f5707c00700 1 -- 172.21.15.28:0/3261961858 wait complete. 2024-05-20T19:52:16.841 INFO:teuthology.orchestra.run.smithi028.stdout:Enabling cephadm module... 2024-05-20T19:52:18.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:17 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2362457634' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-20T19:52:18.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: ignoring --setuser ceph since I am not root 2024-05-20T19:52:18.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: ignoring --setgroup ceph since I am not root 2024-05-20T19:52:18.388 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:18.136+0000 7fb2814f4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T19:52:18.562 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.470+0000 7fc3de600700 1 Processor -- start 2024-05-20T19:52:18.562 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.470+0000 7fc3de600700 1 -- start start 2024-05-20T19:52:18.562 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.471+0000 7fc3de600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d8109280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:18.562 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.471+0000 7fc3de600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc3d81097c0 con 0x7fc3d8108e60 2024-05-20T19:52:18.562 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.471+0000 7fc3dd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d8109280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:18.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.471+0000 7fc3dd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d8109280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37504/0 (socket says 172.21.15.28:37504) 2024-05-20T19:52:18.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.471+0000 7fc3dd200700 1 -- 172.21.15.28:0/1523452735 learned_addr learned my addr 172.21.15.28:0/1523452735 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:18.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.472+0000 7fc3dd200700 1 -- 172.21.15.28:0/1523452735 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc3d8109f70 con 0x7fc3d8108e60 2024-05-20T19:52:18.567 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.472+0000 7fc3dd200700 1 --2- 172.21.15.28:0/1523452735 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d8109280 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc0098a0 tx=0x7fc3cc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e86c2f1e8b7e992 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.472+0000 7fc3d7400700 1 -- 172.21.15.28:0/1523452735 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc3cc004160 con 0x7fc3d8108e60 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.472+0000 7fc3d7400700 1 -- 172.21.15.28:0/1523452735 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fc3cc004300 con 0x7fc3d8108e60 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.472+0000 7fc3d7400700 1 -- 172.21.15.28:0/1523452735 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc3cc011af0 con 0x7fc3d8108e60 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.473+0000 7fc3de600700 1 -- 172.21.15.28:0/1523452735 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 msgr2=0x7fc3d8109280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.473+0000 7fc3de600700 1 --2- 172.21.15.28:0/1523452735 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d8109280 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc0098a0 tx=0x7fc3cc009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.473+0000 7fc3de600700 1 -- 172.21.15.28:0/1523452735 shutdown_connections 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.473+0000 7fc3de600700 1 --2- 172.21.15.28:0/1523452735 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d8109280 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.473+0000 7fc3de600700 1 -- 172.21.15.28:0/1523452735 >> 172.21.15.28:0/1523452735 conn(0x7fc3d8104480 msgr2=0x7fc3d81068a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.473+0000 7fc3de600700 1 -- 172.21.15.28:0/1523452735 shutdown_connections 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.474+0000 7fc3de600700 1 -- 172.21.15.28:0/1523452735 wait complete. 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.475+0000 7fc3de600700 1 Processor -- start 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.475+0000 7fc3de600700 1 -- start start 2024-05-20T19:52:18.568 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.475+0000 7fc3de600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d807eb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.475+0000 7fc3de600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc3d810d3b0 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.475+0000 7fc3dd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d807eb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.475+0000 7fc3dd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d807eb40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37518/0 (socket says 172.21.15.28:37518) 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.475+0000 7fc3dd200700 1 -- 172.21.15.28:0/2362457634 learned_addr learned my addr 172.21.15.28:0/2362457634 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.476+0000 7fc3dd200700 1 -- 172.21.15.28:0/2362457634 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc3d807f080 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.476+0000 7fc3dd200700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d807eb40 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc011ac0 tx=0x7fc3cc011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.476+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc3cc003970 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.477+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fc3cc003b10 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.477+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fc3cc024610 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.477+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc3d807d270 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.477+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc3d807d7b0 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.478+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 4) ==== 50211+0+0 (secure 0 0 0) 0x7fc3cc01a9e0 con 0x7fc3d8108e60 2024-05-20T19:52:18.569 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.478+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc3d806a670 con 0x7fc3d8108e60 2024-05-20T19:52:18.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.479+0000 7fc3d5600700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fc3c003e470 0x7fc3c0040930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:18.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.479+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fc3cc053840 con 0x7fc3d8108e60 2024-05-20T19:52:18.570 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.479+0000 7fc3d7e00700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fc3c003e470 0x7fc3c0040930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.480+0000 7fc3d7e00700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fc3c003e470 0x7fc3c0040930 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fc3c4006fd0 tx=0x7fc3c4006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.484+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fc3cc0247f0 con 0x7fc3d8108e60 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.784+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) -- 0x7fc3d807da30 con 0x7fc3d8108e60 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.838+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) ==== 86+0+0 (secure 0 0 0) 0x7fc3cc053260 con 0x7fc3d8108e60 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.841+0000 7fc3d5600700 1 -- 172.21.15.28:0/2362457634 <== mon.0 v2:172.21.15.28:3300/0 8 ==== mgrmap(e 5) ==== 50222+0+0 (secure 0 0 0) 0x7fc3cc055c80 con 0x7fc3d8108e60 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fc3c003e470 msgr2=0x7fc3c0040930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fc3c003e470 0x7fc3c0040930 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fc3c4006fd0 tx=0x7fc3c4006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 msgr2=0x7fc3d807eb40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d807eb40 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc011ac0 tx=0x7fc3cc011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 shutdown_connections 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7fc3c003e470 0x7fc3c0040930 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:18.571 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 --2- 172.21.15.28:0/2362457634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc3d8108e60 0x7fc3d807eb40 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:18.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.844+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 >> 172.21.15.28:0/2362457634 conn(0x7fc3d8104480 msgr2=0x7fc3d8104fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:18.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.845+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 shutdown_connections 2024-05-20T19:52:18.572 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:17.845+0000 7fc3de600700 1 -- 172.21.15.28:0/2362457634 wait complete. 2024-05-20T19:52:18.717 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:18.386+0000 7fb2814f4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:52:18.718 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:18.525+0000 7fb2814f4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T19:52:18.980 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:18.946+0000 7fb2814f4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T19:52:18.981 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:18 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2362457634' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-20T19:52:18.981 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:18 smithi028 ceph-mon[31178]: mgrmap e5: x(active, since 5s) 2024-05-20T19:52:19.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:19 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:19.066+0000 7fb2814f4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T19:52:20.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:19 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2377923797' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "available": true, 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.135+0000 7f7016a00700 1 Processor -- start 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.135+0000 7f7016a00700 1 -- start start 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.136+0000 7f7016a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f7010109460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.136+0000 7f7016a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f70101099a0 con 0x7f7010109040 2024-05-20T19:52:20.107 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.136+0000 7f700fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f7010109460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.136+0000 7f700fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f7010109460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37536/0 (socket says 172.21.15.28:37536) 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.136+0000 7f700fe00700 1 -- 172.21.15.28:0/2281628334 learned_addr learned my addr 172.21.15.28:0/2281628334 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.137+0000 7f700fe00700 1 -- 172.21.15.28:0/2281628334 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7010109b20 con 0x7f7010109040 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.137+0000 7f700fe00700 1 --2- 172.21.15.28:0/2281628334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f7010109460 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f70000098a0 tx=0x7f7000009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a0e0fd3f22a3236c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.137+0000 7f700ea00700 1 -- 172.21.15.28:0/2281628334 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7000004160 con 0x7f7010109040 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.137+0000 7f700ea00700 1 -- 172.21.15.28:0/2281628334 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f7000004300 con 0x7f7010109040 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.137+0000 7f700ea00700 1 -- 172.21.15.28:0/2281628334 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7000011af0 con 0x7f7010109040 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.138+0000 7f7016a00700 1 -- 172.21.15.28:0/2281628334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 msgr2=0x7f7010109460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.138+0000 7f7016a00700 1 --2- 172.21.15.28:0/2281628334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f7010109460 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f70000098a0 tx=0x7f7000009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.138+0000 7f7016a00700 1 -- 172.21.15.28:0/2281628334 shutdown_connections 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.138+0000 7f7016a00700 1 --2- 172.21.15.28:0/2281628334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f7010109460 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.138+0000 7f7016a00700 1 -- 172.21.15.28:0/2281628334 >> 172.21.15.28:0/2281628334 conn(0x7f7010104680 msgr2=0x7f7010106ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:20.108 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.138+0000 7f7016a00700 1 -- 172.21.15.28:0/2281628334 shutdown_connections 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.138+0000 7f7016a00700 1 -- 172.21.15.28:0/2281628334 wait complete. 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.139+0000 7f7016a00700 1 Processor -- start 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.139+0000 7f7016a00700 1 -- start start 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.140+0000 7f7016a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f701007eaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.140+0000 7f7016a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f701010afd0 con 0x7f7010109040 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.140+0000 7f700fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f701007eaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.140+0000 7f700fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f701007eaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37540/0 (socket says 172.21.15.28:37540) 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.140+0000 7f700fe00700 1 -- 172.21.15.28:0/2377923797 learned_addr learned my addr 172.21.15.28:0/2377923797 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.140+0000 7f700fe00700 1 -- 172.21.15.28:0/2377923797 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f701007f030 con 0x7f7010109040 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.141+0000 7f700fe00700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f701007eaf0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f7000011ac0 tx=0x7f7000011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.141+0000 7f700cc00700 1 -- 172.21.15.28:0/2377923797 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7000003b20 con 0x7f7010109040 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.141+0000 7f700cc00700 1 -- 172.21.15.28:0/2377923797 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f700001a8c0 con 0x7f7010109040 2024-05-20T19:52:20.109 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.141+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f701007d270 con 0x7f7010109040 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.141+0000 7f700cc00700 1 -- 172.21.15.28:0/2377923797 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f70000248d0 con 0x7f7010109040 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.141+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f701007d750 con 0x7f7010109040 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.142+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7010064ba0 con 0x7f7010109040 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.143+0000 7f700cc00700 1 -- 172.21.15.28:0/2377923797 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 5) ==== 50222+0+0 (secure 0 0 0) 0x7f7000022020 con 0x7f7010109040 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.143+0000 7f700cc00700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 0x7f6ff4040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.143+0000 7f700cc00700 1 -- 172.21.15.28:0/2377923797 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f7000053430 con 0x7f7010109040 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.143+0000 7f700f400700 1 -- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 msgr2=0x7f6ff4040980 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.143+0000 7f700f400700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 0x7f6ff4040980 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.149+0000 7f700cc00700 1 -- 172.21.15.28:0/2377923797 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f700001aa70 con 0x7f7010109040 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.344+0000 7f700f400700 1 -- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 msgr2=0x7f6ff4040980 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.344+0000 7f700f400700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 0x7f6ff4040980 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-05-20T19:52:20.110 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.450+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) -- 0x7f701007de80 con 0x7f7010109040 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.450+0000 7f700cc00700 1 -- 172.21.15.28:0/2377923797 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) ==== 56+0+88 (secure 0 0 0) 0x7f7000053190 con 0x7f7010109040 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 msgr2=0x7f6ff4040980 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 0x7f6ff4040980 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 msgr2=0x7f701007eaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f701007eaf0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f7000011ac0 tx=0x7f7000011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 shutdown_connections 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f6ff403e4c0 0x7f6ff4040980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 --2- 172.21.15.28:0/2377923797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7010109040 0x7f701007eaf0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.453+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 >> 172.21.15.28:0/2377923797 conn(0x7f7010104680 msgr2=0x7f70101051e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.454+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 shutdown_connections 2024-05-20T19:52:20.111 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:19.454+0000 7f7016a00700 1 -- 172.21.15.28:0/2377923797 wait complete. 2024-05-20T19:52:20.112 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for the mgr to restart... 2024-05-20T19:52:20.112 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for mgr epoch 5... 2024-05-20T19:52:20.937 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:20.685+0000 7fb2814f4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T19:52:20.937 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:20.935+0000 7fb2814f4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T19:52:21.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:21.159+0000 7fb2814f4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T19:52:21.682 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:21.553+0000 7fb2814f4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T19:52:21.940 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:21.680+0000 7fb2814f4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T19:52:22.204 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:21.938+0000 7fb2814f4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T19:52:22.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:22 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:22.203+0000 7fb2814f4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:52:23.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:23.022+0000 7fb2814f4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T19:52:23.715 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:23.319+0000 7fb2814f4200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T19:52:24.027 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:23.893+0000 7fb2814f4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T19:52:24.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:24.025+0000 7fb2814f4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T19:52:24.804 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:24.525+0000 7fb2814f4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T19:52:25.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:24.995+0000 7fb2814f4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T19:52:25.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:25 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:25.125+0000 7fb2814f4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T19:52:25.778 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:25 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:25.460+0000 7fb2814f4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T19:52:25.779 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:25 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:25.645+0000 7fb2814f4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T19:52:25.779 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:25 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:25.777+0000 7fb2814f4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T19:52:26.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:26 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:26.184+0000 7fb2814f4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T19:52:26.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:26 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:26.313+0000 7fb2814f4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T19:52:27.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:27 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:27.107+0000 7fb2814f4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T19:52:27.804 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:27 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:27.401+0000 7fb2814f4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: Active manager daemon x restarted 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: Activating manager daemon x 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: osdmap e2: 0 total, 0 up, 0 in 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: mgrmap e6: x(active, starting, since 0.0054352s) 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: Manager daemon x is now available 2024-05-20T19:52:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:27.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:27 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: Found migration_current of "None". Setting to last migration. 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:28 smithi028 ceph-mon[31178]: mgrmap e7: x(active, since 1.01089s) 2024-05-20T19:52:29.139 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:52:29.139 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "initialized": true 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.699+0000 7f4418a00700 1 Processor -- start 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.699+0000 7f4418a00700 1 -- start start 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.700+0000 7f4418a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441407d2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.700+0000 7f4418a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f441407d810 con 0x7f441407ed60 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.700+0000 7f4413400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441407d2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.700+0000 7f4413400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441407d2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37544/0 (socket says 172.21.15.28:37544) 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.700+0000 7f4413400700 1 -- 172.21.15.28:0/2812291871 learned_addr learned my addr 172.21.15.28:0/2812291871 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:29.140 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.701+0000 7f4413400700 1 -- 172.21.15.28:0/2812291871 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f441407d990 con 0x7f441407ed60 2024-05-20T19:52:29.145 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.701+0000 7f4413400700 1 --2- 172.21.15.28:0/2812291871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441407d2d0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f44040098a0 tx=0x7f4404009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c4f3d481343ccacd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:29.145 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.701+0000 7f4412000700 1 -- 172.21.15.28:0/2812291871 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4404004160 con 0x7f441407ed60 2024-05-20T19:52:29.145 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.701+0000 7f4412000700 1 -- 172.21.15.28:0/2812291871 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f4404004300 con 0x7f441407ed60 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.701+0000 7f4412000700 1 -- 172.21.15.28:0/2812291871 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4404011af0 con 0x7f441407ed60 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.702+0000 7f4418a00700 1 -- 172.21.15.28:0/2812291871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 msgr2=0x7f441407d2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.702+0000 7f4418a00700 1 --2- 172.21.15.28:0/2812291871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441407d2d0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f44040098a0 tx=0x7f4404009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.702+0000 7f4418a00700 1 -- 172.21.15.28:0/2812291871 shutdown_connections 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.702+0000 7f4418a00700 1 --2- 172.21.15.28:0/2812291871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441407d2d0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.702+0000 7f4418a00700 1 -- 172.21.15.28:0/2812291871 >> 172.21.15.28:0/2812291871 conn(0x7f4414104460 msgr2=0x7f44141068c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.702+0000 7f4418a00700 1 -- 172.21.15.28:0/2812291871 shutdown_connections 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.702+0000 7f4418a00700 1 -- 172.21.15.28:0/2812291871 wait complete. 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.703+0000 7f4418a00700 1 Processor -- start 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.703+0000 7f4418a00700 1 -- start start 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.704+0000 7f4418a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441419efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:29.146 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.704+0000 7f4418a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f441410f2b0 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.704+0000 7f4413400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441419efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.704+0000 7f4413400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441419efc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37554/0 (socket says 172.21.15.28:37554) 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.704+0000 7f4413400700 1 -- 172.21.15.28:0/3730983549 learned_addr learned my addr 172.21.15.28:0/3730983549 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.704+0000 7f4413400700 1 -- 172.21.15.28:0/3730983549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f441419f500 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.705+0000 7f4413400700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441419efc0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f4404011ac0 tx=0x7f4404011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.705+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4404003970 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.705+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f4404003b10 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.705+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4404024610 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.705+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f441419f790 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.705+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f44141a2480 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.707+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 5) ==== 50222+0+0 (secure 0 0 0) 0x7f440401a9e0 con 0x7f441407ed60 2024-05-20T19:52:29.147 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.708+0000 7f440be00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.708+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 --> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f43f80410f0 con 0x7f43f803e520 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.708+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f4404053860 con 0x7f441407ed60 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.708+0000 7f4412a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 msgr2=0x7f43f80409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.708+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.909+0000 7f4412a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 msgr2=0x7f43f80409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:20.909+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:21.310+0000 7f4412a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 msgr2=0x7f43f80409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:21.310+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:22.111+0000 7f4412a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 msgr2=0x7f43f80409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:22.111+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:23.713+0000 7f4412a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 msgr2=0x7f43f80409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:29.148 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:23.713+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-05-20T19:52:29.149 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:26.917+0000 7f4412a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 msgr2=0x7f43f80409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/1506400984 2024-05-20T19:52:29.149 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:26.918+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 6.400000 2024-05-20T19:52:29.149 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:27.411+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mgrmap(e 6) ==== 49989+0+0 (secure 0 0 0) 0x7f440401a430 con 0x7f441407ed60 2024-05-20T19:52:29.149 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:27.411+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 msgr2=0x7f43f80409e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T19:52:29.149 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:27.411+0000 7f440be00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.149 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.414+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mgrmap(e 7) ==== 50116+0+0 (secure 0 0 0) 0x7f440402cc70 con 0x7f441407ed60 2024-05-20T19:52:29.151 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.415+0000 7f440be00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f43f8041e10 0x7f43f8044200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:29.151 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.415+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f4404021e70 con 0x7f43f8041e10 2024-05-20T19:52:29.151 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.417+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f43f8041e10 0x7f43f8044200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:29.151 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.419+0000 7f4412a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f43f8041e10 0x7f43f8044200 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f4400003a10 tx=0x7f44000092b0 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:29.151 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.421+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== command_reply(tid 0: 0 ) ==== 8+0+9564 (secure 0 0 0) 0x7f4404021e70 con 0x7f43f8041e10 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.430+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- command(tid 1: {"prefix": "mgr_status"}) -- 0x7f441406a670 con 0x7f43f8041e10 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f440be00700 1 -- 172.21.15.28:0/3730983549 <== mgr.14120 v2:172.21.15.28:6800/811935446 2 ==== command_reply(tid 1: 0 ) ==== 8+0+51 (secure 0 0 0) 0x7f441406a670 con 0x7f43f8041e10 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f43f8041e10 msgr2=0x7f43f8044200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f43f8041e10 0x7f43f8044200 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f4400003a10 tx=0x7f44000092b0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 msgr2=0x7f441419efc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441419efc0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f4404011ac0 tx=0x7f4404011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 shutdown_connections 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f43f8041e10 0x7f43f8044200 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f441407ed60 0x7f441419efc0 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 --2- 172.21.15.28:0/3730983549 >> [v2:172.21.15.28:6800/1506400984,v1:172.21.15.28:6801/1506400984] conn(0x7f43f803e520 0x7f43f80409e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 >> 172.21.15.28:0/3730983549 conn(0x7f4414104460 msgr2=0x7f4414105000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:29.152 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.431+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 shutdown_connections 2024-05-20T19:52:29.153 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:28.432+0000 7f4418a00700 1 -- 172.21.15.28:0/3730983549 wait complete. 2024-05-20T19:52:29.153 INFO:teuthology.orchestra.run.smithi028.stdout:mgr epoch 5 is available 2024-05-20T19:52:29.153 INFO:teuthology.orchestra.run.smithi028.stdout:Setting orchestrator backend to cephadm... 2024-05-20T19:52:29.765 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:29 smithi028 ceph-mon[31178]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-20T19:52:29.765 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:29 smithi028 ceph-mon[31178]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-20T19:52:29.765 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:29 smithi028 ceph-mon[31178]: [20/May/2024:19:52:28] ENGINE Bus STARTING 2024-05-20T19:52:29.765 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:29 smithi028 ceph-mon[31178]: [20/May/2024:19:52:28] ENGINE Serving on https://172.21.15.28:7150 2024-05-20T19:52:29.765 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:29 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:52:30.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:30 smithi028 ceph-mon[31178]: [20/May/2024:19:52:28] ENGINE Serving on http://172.21.15.28:8765 2024-05-20T19:52:30.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:30 smithi028 ceph-mon[31178]: [20/May/2024:19:52:28] ENGINE Bus STARTED 2024-05-20T19:52:30.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:30 smithi028 ceph-mon[31178]: mgrmap e8: x(active, since 2s) 2024-05-20T19:52:30.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:30 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:30.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:30 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:52:31.002 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.764+0000 7f0626a00700 1 Processor -- start 2024-05-20T19:52:31.003 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.764+0000 7f0626a00700 1 -- start start 2024-05-20T19:52:31.003 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.765+0000 7f0626a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062010b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:31.003 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.765+0000 7f0626a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f062010bb30 con 0x7f062010b1d0 2024-05-20T19:52:31.003 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.765+0000 7f061fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062010b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:31.004 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.765+0000 7f061fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062010b5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48060/0 (socket says 172.21.15.28:48060) 2024-05-20T19:52:31.004 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.765+0000 7f061fe00700 1 -- 172.21.15.28:0/4015595368 learned_addr learned my addr 172.21.15.28:0/4015595368 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:31.004 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.766+0000 7f061fe00700 1 -- 172.21.15.28:0/4015595368 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f062010bcb0 con 0x7f062010b1d0 2024-05-20T19:52:31.004 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.766+0000 7f061fe00700 1 --2- 172.21.15.28:0/4015595368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062010b5f0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f06100098a0 tx=0x7f0610009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1107abf4d93666f1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:31.004 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.766+0000 7f061ea00700 1 -- 172.21.15.28:0/4015595368 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0610004160 con 0x7f062010b1d0 2024-05-20T19:52:31.004 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.766+0000 7f061ea00700 1 -- 172.21.15.28:0/4015595368 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f0610004300 con 0x7f062010b1d0 2024-05-20T19:52:31.005 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.766+0000 7f061ea00700 1 -- 172.21.15.28:0/4015595368 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0610011af0 con 0x7f062010b1d0 2024-05-20T19:52:31.005 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.767+0000 7f0626a00700 1 -- 172.21.15.28:0/4015595368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 msgr2=0x7f062010b5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:31.005 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.767+0000 7f0626a00700 1 --2- 172.21.15.28:0/4015595368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062010b5f0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f06100098a0 tx=0x7f0610009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:31.006 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.767+0000 7f0626a00700 1 -- 172.21.15.28:0/4015595368 shutdown_connections 2024-05-20T19:52:31.006 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.767+0000 7f0626a00700 1 --2- 172.21.15.28:0/4015595368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062010b5f0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:31.006 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.767+0000 7f0626a00700 1 -- 172.21.15.28:0/4015595368 >> 172.21.15.28:0/4015595368 conn(0x7f062007e6e0 msgr2=0x7f062007eaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:31.007 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.768+0000 7f0626a00700 1 -- 172.21.15.28:0/4015595368 shutdown_connections 2024-05-20T19:52:31.007 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.768+0000 7f0626a00700 1 -- 172.21.15.28:0/4015595368 wait complete. 2024-05-20T19:52:31.007 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.769+0000 7f0626a00700 1 Processor -- start 2024-05-20T19:52:31.008 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.769+0000 7f0626a00700 1 -- start start 2024-05-20T19:52:31.008 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.769+0000 7f0626a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062019f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:31.008 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.769+0000 7f0626a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f062010d210 con 0x7f062010b1d0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.770+0000 7f061fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062019f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.770+0000 7f061fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062019f100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48070/0 (socket says 172.21.15.28:48070) 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.770+0000 7f061fe00700 1 -- 172.21.15.28:0/1713005711 learned_addr learned my addr 172.21.15.28:0/1713005711 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.770+0000 7f061fe00700 1 -- 172.21.15.28:0/1713005711 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f062019f640 con 0x7f062010b1d0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.770+0000 7f061fe00700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062019f100 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f0610011ac0 tx=0x7f0610011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.771+0000 7f061cc00700 1 -- 172.21.15.28:0/1713005711 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f0610003b20 con 0x7f062010b1d0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.771+0000 7f061cc00700 1 -- 172.21.15.28:0/1713005711 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f061001a8c0 con 0x7f062010b1d0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.771+0000 7f061cc00700 1 -- 172.21.15.28:0/1713005711 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f06100248d0 con 0x7f062010b1d0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.771+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f062019f8d0 con 0x7f062010b1d0 2024-05-20T19:52:31.009 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.771+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f06201a25c0 con 0x7f062010b1d0 2024-05-20T19:52:31.011 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.772+0000 7f061cc00700 1 -- 172.21.15.28:0/1713005711 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7f0610022020 con 0x7f062010b1d0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.772+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0620064ba0 con 0x7f062010b1d0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.773+0000 7f061cc00700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f060403e4c0 0x7f0604040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.773+0000 7f061cc00700 1 -- 172.21.15.28:0/1713005711 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f0610053810 con 0x7f062010b1d0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.773+0000 7f061f400700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f060403e4c0 0x7f0604040980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.774+0000 7f061f400700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f060403e4c0 0x7f0604040980 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f060c006fd0 tx=0x7f060c006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:29.779+0000 7f061cc00700 1 -- 172.21.15.28:0/1713005711 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f06100172b0 con 0x7f062010b1d0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.036+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) -- 0x7f0620109460 con 0x7f060403e4c0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.049+0000 7f061cc00700 1 -- 172.21.15.28:0/1713005711 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f0620109460 con 0x7f060403e4c0 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f060403e4c0 msgr2=0x7f0604040980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f060403e4c0 0x7f0604040980 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f060c006fd0 tx=0x7f060c006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 msgr2=0x7f062019f100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:31.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062019f100 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f0610011ac0 tx=0x7f0610011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:31.013 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 shutdown_connections 2024-05-20T19:52:31.013 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f060403e4c0 0x7f0604040980 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:31.013 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 --2- 172.21.15.28:0/1713005711 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062010b1d0 0x7f062019f100 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:31.013 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.052+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 >> 172.21.15.28:0/1713005711 conn(0x7f062007e6e0 msgr2=0x7f0620108d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:31.013 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.053+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 shutdown_connections 2024-05-20T19:52:31.013 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:30.053+0000 7f0626a00700 1 -- 172.21.15.28:0/1713005711 wait complete. 2024-05-20T19:52:31.572 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:31 smithi028 ceph-mon[31178]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:52:32.534 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:32 smithi028 ceph-mon[31178]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout value unchanged 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.641+0000 7fe79d800700 1 Processor -- start 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.642+0000 7fe79d800700 1 -- start start 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.642+0000 7fe79d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79810b4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.642+0000 7fe79d800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe79810bac0 con 0x7fe79810b0d0 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.642+0000 7fe797e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79810b4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.643+0000 7fe797e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79810b4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48084/0 (socket says 172.21.15.28:48084) 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.643+0000 7fe797e00700 1 -- 172.21.15.28:0/2630466121 learned_addr learned my addr 172.21.15.28:0/2630466121 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.643+0000 7fe797e00700 1 -- 172.21.15.28:0/2630466121 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe79810c210 con 0x7fe79810b0d0 2024-05-20T19:52:32.550 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.643+0000 7fe797e00700 1 --2- 172.21.15.28:0/2630466121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79810b4f0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fe7880098a0 tx=0x7fe788009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d1814db0ba256f8e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.644+0000 7fe796a00700 1 -- 172.21.15.28:0/2630466121 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe788004160 con 0x7fe79810b0d0 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.644+0000 7fe796a00700 1 -- 172.21.15.28:0/2630466121 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fe788004300 con 0x7fe79810b0d0 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.644+0000 7fe796a00700 1 -- 172.21.15.28:0/2630466121 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe788011af0 con 0x7fe79810b0d0 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.644+0000 7fe79d800700 1 -- 172.21.15.28:0/2630466121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 msgr2=0x7fe79810b4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.644+0000 7fe79d800700 1 --2- 172.21.15.28:0/2630466121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79810b4f0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fe7880098a0 tx=0x7fe788009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.645+0000 7fe79d800700 1 -- 172.21.15.28:0/2630466121 shutdown_connections 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.645+0000 7fe79d800700 1 --2- 172.21.15.28:0/2630466121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79810b4f0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.645+0000 7fe79d800700 1 -- 172.21.15.28:0/2630466121 >> 172.21.15.28:0/2630466121 conn(0x7fe79807e410 msgr2=0x7fe79807e840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.645+0000 7fe79d800700 1 -- 172.21.15.28:0/2630466121 shutdown_connections 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.645+0000 7fe79d800700 1 -- 172.21.15.28:0/2630466121 wait complete. 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.646+0000 7fe79d800700 1 Processor -- start 2024-05-20T19:52:32.551 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.646+0000 7fe79d800700 1 -- start start 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.646+0000 7fe79d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79819eef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.646+0000 7fe79d800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe79810f5c0 con 0x7fe79810b0d0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.646+0000 7fe797e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79819eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.646+0000 7fe797e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79819eef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48092/0 (socket says 172.21.15.28:48092) 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.646+0000 7fe797e00700 1 -- 172.21.15.28:0/1700032038 learned_addr learned my addr 172.21.15.28:0/1700032038 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.647+0000 7fe797e00700 1 -- 172.21.15.28:0/1700032038 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe79819f430 con 0x7fe79810b0d0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.647+0000 7fe797e00700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79819eef0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fe788011ac0 tx=0x7fe788011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.647+0000 7fe794c00700 1 -- 172.21.15.28:0/1700032038 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe788003b20 con 0x7fe79810b0d0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.647+0000 7fe794c00700 1 -- 172.21.15.28:0/1700032038 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7fe78801a8c0 con 0x7fe79810b0d0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.647+0000 7fe794c00700 1 -- 172.21.15.28:0/1700032038 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe7880248d0 con 0x7fe79810b0d0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.648+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe79819f6c0 con 0x7fe79810b0d0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.648+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe7981a23b0 con 0x7fe79810b0d0 2024-05-20T19:52:32.552 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.649+0000 7fe794c00700 1 -- 172.21.15.28:0/1700032038 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7fe788022020 con 0x7fe79810b0d0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.649+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe79806a670 con 0x7fe79810b0d0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.649+0000 7fe794c00700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fe77c03e4c0 0x7fe77c040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.649+0000 7fe794c00700 1 -- 172.21.15.28:0/1700032038 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fe7880538e0 con 0x7fe79810b0d0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.650+0000 7fe797400700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fe77c03e4c0 0x7fe77c040980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.650+0000 7fe797400700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fe77c03e4c0 0x7fe77c040980 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fe784006fd0 tx=0x7fe784006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.656+0000 7fe794c00700 1 -- 172.21.15.28:0/1700032038 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fe7880172b0 con 0x7fe79810b0d0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.895+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) -- 0x7fe798109390 con 0x7fe77c03e4c0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.897+0000 7fe794c00700 1 -- 172.21.15.28:0/1700032038 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+16 (secure 0 0 0) 0x7fe798109390 con 0x7fe77c03e4c0 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fe77c03e4c0 msgr2=0x7fe77c040980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fe77c03e4c0 0x7fe77c040980 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fe784006fd0 tx=0x7fe784006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 msgr2=0x7fe79819eef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79819eef0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fe788011ac0 tx=0x7fe788011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:32.553 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 shutdown_connections 2024-05-20T19:52:32.554 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fe77c03e4c0 0x7fe77c040980 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:32.554 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 --2- 172.21.15.28:0/1700032038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe79810b0d0 0x7fe79819eef0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:32.554 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.900+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 >> 172.21.15.28:0/1700032038 conn(0x7fe79807e410 msgr2=0x7fe798108c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:32.554 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.901+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 shutdown_connections 2024-05-20T19:52:32.554 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:31.901+0000 7fe79d800700 1 -- 172.21.15.28:0/1700032038 wait complete. 2024-05-20T19:52:32.554 INFO:teuthology.orchestra.run.smithi028.stdout:Generating ssh key... 2024-05-20T19:52:34.056 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: Generating public/private rsa key pair. 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: Your identification has been saved in /tmp/tmpa9wahw_a/key. 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: Your public key has been saved in /tmp/tmpa9wahw_a/key.pub. 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: The key fingerprint is: 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: SHA256:h8z0a2J4wacuneTAJckGzv0xyLvkHp4mLxsNHfkKI7g ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: The key's randomart image is: 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: +---[RSA 3072]----+ 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: | | 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: | . . | 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: | o =oo. | 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: | . o.OB+o | 2024-05-20T19:52:34.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: |. . +o.=So+ | 2024-05-20T19:52:34.058 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: | . . ==oo= . | 2024-05-20T19:52:34.058 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: |E .o=*=.o | 2024-05-20T19:52:34.058 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: | oo=*+o | 2024-05-20T19:52:34.058 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: | .B=.. | 2024-05-20T19:52:34.058 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: +----[SHA256]-----+ 2024-05-20T19:52:34.528 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.198+0000 7f787c000700 1 Processor -- start 2024-05-20T19:52:34.529 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.198+0000 7f787c000700 1 -- start start 2024-05-20T19:52:34.529 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.199+0000 7f787c000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787410b630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:34.529 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.199+0000 7f787c000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f787410bc00 con 0x7f787410b210 2024-05-20T19:52:34.529 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.199+0000 7f7879600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787410b630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:34.529 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.199+0000 7f7879600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787410b630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48102/0 (socket says 172.21.15.28:48102) 2024-05-20T19:52:34.529 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.199+0000 7f7879600700 1 -- 172.21.15.28:0/1102957963 learned_addr learned my addr 172.21.15.28:0/1102957963 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:34.529 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.200+0000 7f7879600700 1 -- 172.21.15.28:0/1102957963 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f787410c410 con 0x7f787410b210 2024-05-20T19:52:34.533 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.200+0000 7f7879600700 1 --2- 172.21.15.28:0/1102957963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787410b630 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f78680098a0 tx=0x7f7868009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=48e0c3a315164778 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:34.533 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.200+0000 7f7873e00700 1 -- 172.21.15.28:0/1102957963 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7868004160 con 0x7f787410b210 2024-05-20T19:52:34.533 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.200+0000 7f7873e00700 1 -- 172.21.15.28:0/1102957963 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f7868004300 con 0x7f787410b210 2024-05-20T19:52:34.533 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.200+0000 7f7873e00700 1 -- 172.21.15.28:0/1102957963 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7868011af0 con 0x7f787410b210 2024-05-20T19:52:34.533 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.201+0000 7f787c000700 1 -- 172.21.15.28:0/1102957963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 msgr2=0x7f787410b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:34.533 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.201+0000 7f787c000700 1 --2- 172.21.15.28:0/1102957963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787410b630 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f78680098a0 tx=0x7f7868009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.201+0000 7f787c000700 1 -- 172.21.15.28:0/1102957963 shutdown_connections 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.201+0000 7f787c000700 1 --2- 172.21.15.28:0/1102957963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787410b630 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.201+0000 7f787c000700 1 -- 172.21.15.28:0/1102957963 >> 172.21.15.28:0/1102957963 conn(0x7f787407e650 msgr2=0x7f787407eaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.201+0000 7f787c000700 1 -- 172.21.15.28:0/1102957963 shutdown_connections 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.201+0000 7f787c000700 1 -- 172.21.15.28:0/1102957963 wait complete. 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.202+0000 7f787c000700 1 Processor -- start 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.202+0000 7f787c000700 1 -- start start 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.203+0000 7f787c000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787419f1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.203+0000 7f787c000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f787410f7c0 con 0x7f787410b210 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.203+0000 7f7879600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787419f1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.203+0000 7f7879600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787419f1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48108/0 (socket says 172.21.15.28:48108) 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.203+0000 7f7879600700 1 -- 172.21.15.28:0/3722397847 learned_addr learned my addr 172.21.15.28:0/3722397847 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:34.534 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.203+0000 7f7879600700 1 -- 172.21.15.28:0/3722397847 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f787419f730 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.204+0000 7f7879600700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787419f1f0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f78680117b0 tx=0x7f7868011b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.204+0000 7f7872000700 1 -- 172.21.15.28:0/3722397847 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7868003af0 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.204+0000 7f7872000700 1 -- 172.21.15.28:0/3722397847 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f7868020070 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.204+0000 7f7872000700 1 -- 172.21.15.28:0/3722397847 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f7868019630 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.204+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f787419f9c0 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.204+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f78741a26b0 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.205+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7874064ba0 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.206+0000 7f7872000700 1 -- 172.21.15.28:0/3722397847 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7f786801a9c0 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.206+0000 7f7872000700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f785c03e470 0x7f785c040930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.206+0000 7f7872000700 1 -- 172.21.15.28:0/3722397847 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f7868053ae0 con 0x7f787410b210 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.207+0000 7f7878c00700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f785c03e470 0x7f785c040930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.207+0000 7f7878c00700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f785c03e470 0x7f785c040930 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7860006fd0 tx=0x7f7860006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:34.535 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.212+0000 7f7872000700 1 -- 172.21.15.28:0/3722397847 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f78680172b0 con 0x7f787410b210 2024-05-20T19:52:34.536 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.448+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) -- 0x7f7874109540 con 0x7f785c03e470 2024-05-20T19:52:34.536 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.805+0000 7f7872000700 1 -- 172.21.15.28:0/3722397847 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f7874109540 con 0x7f785c03e470 2024-05-20T19:52:34.536 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.808+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f785c03e470 msgr2=0x7f785c040930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:34.536 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.808+0000 7f787c000700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f785c03e470 0x7f785c040930 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7860006fd0 tx=0x7f7860006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:34.536 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.808+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 msgr2=0x7f787419f1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:34.537 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.808+0000 7f787c000700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787419f1f0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f78680117b0 tx=0x7f7868011b70 comp rx=0 tx=0).stop 2024-05-20T19:52:34.537 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.808+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 shutdown_connections 2024-05-20T19:52:34.537 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.808+0000 7f787c000700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f785c03e470 0x7f785c040930 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:34.537 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.809+0000 7f787c000700 1 --2- 172.21.15.28:0/3722397847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f787410b210 0x7f787419f1f0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:34.537 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.809+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 >> 172.21.15.28:0/3722397847 conn(0x7f787407e650 msgr2=0x7f7874108e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:34.537 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.809+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 shutdown_connections 2024-05-20T19:52:34.537 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:33.809+0000 7f787c000700 1 -- 172.21.15.28:0/3722397847 wait complete. 2024-05-20T19:52:35.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:34 smithi028 ceph-mon[31178]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:52:35.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:34 smithi028 ceph-mon[31178]: Generating ssh key... 2024-05-20T19:52:35.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:34 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:35.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:34 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:36.011 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ6OKcFk30zzzgO8qZZ+GR+Ud+IVR4ZAvakRYwOGSfY62Gw+VP4VoR1B2vVhPW1Fvx1Ez/UFEEIc/WTwzc1K+kTOUrpRzEOolfo2eyGDIBi/+eS0yzscZpWajV1Fz8/tJO3+8EFkHE4ld2ibTcCHwedS762vVEmAj2lMxHw+Aau4Xb6qqbso/QKoYPJ1bnvi3agHYpPcfdq9pGS4sHarYZ+SfRPlE2MHYwklqVTcvx20UCDDiMtNZjYBp2aXHkRdZV57CG8Pj+B8DolUkfC4vg1l0g3/z16ps7FY6qAoqWdoaOKCUX3C08hRKRvHVcnQmPPRppp9fO/68iicPfXZGHRMMNOgTrVTjWVOhsTZKaRn86SrOGrvILK/G5fYOeGyLasXx8KatxjFbdViwtVVacu7OHlQr3mn4QakgOh2F1JpvzL7VPeORLr4JAc92FYDFm3FOv203D8Xxgu5eZD6fXmNSP0W38u+fGbAe2sPyY7DB/7Wa37nPIuuCebeNTvck= ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:52:36.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.108+0000 7f6333e00700 1 Processor -- start 2024-05-20T19:52:36.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.108+0000 7f6333e00700 1 -- start start 2024-05-20T19:52:36.012 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.108+0000 7f6333e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f633410bd40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.108+0000 7f6333e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f633407cde0 con 0x7f6334109950 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.109+0000 7f6332a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f633410bd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.109+0000 7f6332a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f633410bd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48122/0 (socket says 172.21.15.28:48122) 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.109+0000 7f6332a00700 1 -- 172.21.15.28:0/1629031389 learned_addr learned my addr 172.21.15.28:0/1629031389 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.110+0000 7f6332a00700 1 -- 172.21.15.28:0/1629031389 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f633410c280 con 0x7f6334109950 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.110+0000 7f6332a00700 1 --2- 172.21.15.28:0/1629031389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f633410bd40 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f63240098a0 tx=0x7f6324009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7862cf6d5cb6c1f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.110+0000 7f6331600700 1 -- 172.21.15.28:0/1629031389 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6324004160 con 0x7f6334109950 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.110+0000 7f6331600700 1 -- 172.21.15.28:0/1629031389 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f6324004300 con 0x7f6334109950 2024-05-20T19:52:36.014 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.110+0000 7f6331600700 1 -- 172.21.15.28:0/1629031389 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6324011af0 con 0x7f6334109950 2024-05-20T19:52:36.015 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.111+0000 7f6333e00700 1 -- 172.21.15.28:0/1629031389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 msgr2=0x7f633410bd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.111+0000 7f6333e00700 1 --2- 172.21.15.28:0/1629031389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f633410bd40 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f63240098a0 tx=0x7f6324009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.111+0000 7f6333e00700 1 -- 172.21.15.28:0/1629031389 shutdown_connections 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.111+0000 7f6333e00700 1 --2- 172.21.15.28:0/1629031389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f633410bd40 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.111+0000 7f6333e00700 1 -- 172.21.15.28:0/1629031389 >> 172.21.15.28:0/1629031389 conn(0x7f63341032e0 msgr2=0x7f6334105720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.111+0000 7f6333e00700 1 -- 172.21.15.28:0/1629031389 shutdown_connections 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.111+0000 7f6333e00700 1 -- 172.21.15.28:0/1629031389 wait complete. 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.112+0000 7f6333e00700 1 Processor -- start 2024-05-20T19:52:36.016 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.112+0000 7f6333e00700 1 -- start start 2024-05-20T19:52:36.018 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.113+0000 7f6333e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f63341091d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.113+0000 7f6333e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f633410d1e0 con 0x7f6334109950 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.113+0000 7f6332a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f63341091d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.113+0000 7f6332a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f63341091d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48130/0 (socket says 172.21.15.28:48130) 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.113+0000 7f6332a00700 1 -- 172.21.15.28:0/4236325953 learned_addr learned my addr 172.21.15.28:0/4236325953 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.113+0000 7f6332a00700 1 -- 172.21.15.28:0/4236325953 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6334109710 con 0x7f6334109950 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.114+0000 7f6332a00700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f63341091d0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6324011ac0 tx=0x7f6324011ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.114+0000 7f632b400700 1 -- 172.21.15.28:0/4236325953 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6324003970 con 0x7f6334109950 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.114+0000 7f632b400700 1 -- 172.21.15.28:0/4236325953 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f6324003b10 con 0x7f6334109950 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.114+0000 7f632b400700 1 -- 172.21.15.28:0/4236325953 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6324024610 con 0x7f6334109950 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.114+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6334107900 con 0x7f6334109950 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.114+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6334107e40 con 0x7f6334109950 2024-05-20T19:52:36.019 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.116+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f633406a670 con 0x7f6334109950 2024-05-20T19:52:36.020 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.116+0000 7f632b400700 1 -- 172.21.15.28:0/4236325953 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7f632401a9e0 con 0x7f6334109950 2024-05-20T19:52:36.020 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.116+0000 7f632b400700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f631803e4c0 0x7f6318040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:36.020 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.116+0000 7f632b400700 1 -- 172.21.15.28:0/4236325953 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f6324053930 con 0x7f6334109950 2024-05-20T19:52:36.020 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.117+0000 7f6332000700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f631803e4c0 0x7f6318040980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:36.020 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.117+0000 7f6332000700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f631803e4c0 0x7f6318040980 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f6320006fd0 tx=0x7f6320006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:36.020 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.123+0000 7f632b400700 1 -- 172.21.15.28:0/4236325953 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f6324053280 con 0x7f6334109950 2024-05-20T19:52:36.020 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.355+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) -- 0x7f6334108590 con 0x7f631803e4c0 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.357+0000 7f632b400700 1 -- 172.21.15.28:0/4236325953 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+595 (secure 0 0 0) 0x7f6334108590 con 0x7f631803e4c0 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f631803e4c0 msgr2=0x7f6318040980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f631803e4c0 0x7f6318040980 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f6320006fd0 tx=0x7f6320006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 msgr2=0x7f63341091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f63341091d0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6324011ac0 tx=0x7f6324011ba0 comp rx=0 tx=0).stop 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 shutdown_connections 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f631803e4c0 0x7f6318040980 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:36.022 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 --2- 172.21.15.28:0/4236325953 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6334109950 0x7f63341091d0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:36.023 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 >> 172.21.15.28:0/4236325953 conn(0x7f63341032e0 msgr2=0x7f6334103e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:36.023 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 shutdown_connections 2024-05-20T19:52:36.023 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:35.360+0000 7f6333e00700 1 -- 172.21.15.28:0/4236325953 wait complete. 2024-05-20T19:52:36.023 INFO:teuthology.orchestra.run.smithi028.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-20T19:52:36.023 INFO:teuthology.orchestra.run.smithi028.stdout:Adding key to root@localhost authorized_keys... 2024-05-20T19:52:36.023 INFO:teuthology.orchestra.run.smithi028.stdout:Adding host smithi028... 2024-05-20T19:52:36.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:36 smithi028 ceph-mon[31178]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:52:37.192 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:37 smithi028 ceph-mon[31178]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi028", "addr": "172.21.15.28", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:52:39.272 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:39 smithi028 ceph-mon[31178]: Deploying cephadm binary to smithi028 2024-05-20T19:52:44.041 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout Added host 'smithi028' with addr '172.21.15.28' 2024-05-20T19:52:44.041 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.681+0000 7f4777200700 1 Processor -- start 2024-05-20T19:52:44.041 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.681+0000 7f4777200700 1 -- start start 2024-05-20T19:52:44.041 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.682+0000 7f4777200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477010b640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:44.041 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.682+0000 7f4777200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f477010bc10 con 0x7f477010b220 2024-05-20T19:52:44.041 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.682+0000 7f476fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477010b640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.682+0000 7f476fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477010b640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48140/0 (socket says 172.21.15.28:48140) 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.682+0000 7f476fe00700 1 -- 172.21.15.28:0/319765347 learned_addr learned my addr 172.21.15.28:0/319765347 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.683+0000 7f476fe00700 1 -- 172.21.15.28:0/319765347 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f477010c3d0 con 0x7f477010b220 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.683+0000 7f476fe00700 1 --2- 172.21.15.28:0/319765347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477010b640 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f47600098a0 tx=0x7f4760009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ee6bd8501b045fb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.684+0000 7f476ea00700 1 -- 172.21.15.28:0/319765347 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4760004160 con 0x7f477010b220 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.684+0000 7f476ea00700 1 -- 172.21.15.28:0/319765347 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f4760004300 con 0x7f477010b220 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.684+0000 7f476ea00700 1 -- 172.21.15.28:0/319765347 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4760011af0 con 0x7f477010b220 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.684+0000 7f4777200700 1 -- 172.21.15.28:0/319765347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 msgr2=0x7f477010b640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.684+0000 7f4777200700 1 --2- 172.21.15.28:0/319765347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477010b640 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f47600098a0 tx=0x7f4760009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:44.042 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.685+0000 7f4777200700 1 -- 172.21.15.28:0/319765347 shutdown_connections 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.685+0000 7f4777200700 1 --2- 172.21.15.28:0/319765347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477010b640 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.685+0000 7f4777200700 1 -- 172.21.15.28:0/319765347 >> 172.21.15.28:0/319765347 conn(0x7f477007e8b0 msgr2=0x7f47701094a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.685+0000 7f4777200700 1 -- 172.21.15.28:0/319765347 shutdown_connections 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.685+0000 7f4777200700 1 -- 172.21.15.28:0/319765347 wait complete. 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.686+0000 7f4777200700 1 Processor -- start 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.686+0000 7f4777200700 1 -- start start 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.687+0000 7f4777200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477019f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:44.043 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.687+0000 7f4777200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f477010f780 con 0x7f477010b220 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.687+0000 7f476fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477019f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.687+0000 7f476fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477019f0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48150/0 (socket says 172.21.15.28:48150) 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.687+0000 7f476fe00700 1 -- 172.21.15.28:0/1150043980 learned_addr learned my addr 172.21.15.28:0/1150043980 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.688+0000 7f476fe00700 1 -- 172.21.15.28:0/1150043980 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f477019f630 con 0x7f477010b220 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.688+0000 7f476fe00700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477019f0f0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f4760000c00 tx=0x7f4760011b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.688+0000 7f476cc00700 1 -- 172.21.15.28:0/1150043980 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4760003b20 con 0x7f477010b220 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.688+0000 7f476cc00700 1 -- 172.21.15.28:0/1150043980 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f476001a8c0 con 0x7f477010b220 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.688+0000 7f476cc00700 1 -- 172.21.15.28:0/1150043980 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4760024940 con 0x7f477010b220 2024-05-20T19:52:44.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.688+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f477019f8c0 con 0x7f477010b220 2024-05-20T19:52:44.045 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.688+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f47701a25b0 con 0x7f477010b220 2024-05-20T19:52:44.045 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.690+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4770064ba0 con 0x7f477010b220 2024-05-20T19:52:44.045 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.690+0000 7f476cc00700 1 -- 172.21.15.28:0/1150043980 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7f4760022020 con 0x7f477010b220 2024-05-20T19:52:44.045 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.690+0000 7f476cc00700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f475403e470 0x7f4754040930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:44.045 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.690+0000 7f476cc00700 1 -- 172.21.15.28:0/1150043980 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f4760053820 con 0x7f477010b220 2024-05-20T19:52:44.045 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.691+0000 7f476f400700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f475403e470 0x7f4754040930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.691+0000 7f476f400700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f475403e470 0x7f4754040930 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f475c006fd0 tx=0x7f475c006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.697+0000 7f476cc00700 1 -- 172.21.15.28:0/1150043980 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f47600042a0 con 0x7f477010b220 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:36.940+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi028", "addr": "172.21.15.28", "target": ["mon-mgr", ""]}) -- 0x7f4770109360 con 0x7f475403e470 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.292+0000 7f476cc00700 1 -- 172.21.15.28:0/1150043980 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+48 (secure 0 0 0) 0x7f4770109360 con 0x7f475403e470 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f475403e470 msgr2=0x7f4754040930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f475403e470 0x7f4754040930 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f475c006fd0 tx=0x7f475c006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 msgr2=0x7f477019f0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477019f0f0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f4760000c00 tx=0x7f4760011b70 comp rx=0 tx=0).stop 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 shutdown_connections 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f475403e470 0x7f4754040930 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 --2- 172.21.15.28:0/1150043980 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f477010b220 0x7f477019f0f0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:44.046 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.297+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 >> 172.21.15.28:0/1150043980 conn(0x7f477007e8b0 msgr2=0x7f4770108c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:44.047 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.298+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 shutdown_connections 2024-05-20T19:52:44.047 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:43.298+0000 7f4777200700 1 -- 172.21.15.28:0/1150043980 wait complete. 2024-05-20T19:52:44.047 INFO:teuthology.orchestra.run.smithi028.stdout:Deploying unmanaged mon service... 2024-05-20T19:52:44.542 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:44 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:44.543 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:44 smithi028 ceph-mon[31178]: Added host smithi028 2024-05-20T19:52:44.543 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:44 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:52:46.191 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:46 smithi028 ceph-mon[31178]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:52:46.192 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:46 smithi028 ceph-mon[31178]: Saving service mon spec with placement count:5 2024-05-20T19:52:46.192 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:46 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:46.590 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-05-20T19:52:46.590 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.718+0000 7f418b000700 1 Processor -- start 2024-05-20T19:52:46.590 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.718+0000 7f418b000700 1 -- start start 2024-05-20T19:52:46.590 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.719+0000 7f418b000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418410b630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.719+0000 7f418b000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f418410bc00 con 0x7f418410b210 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.719+0000 7f4183e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418410b630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.719+0000 7f4183e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418410b630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51970/0 (socket says 172.21.15.28:51970) 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.719+0000 7f4183e00700 1 -- 172.21.15.28:0/1292139567 learned_addr learned my addr 172.21.15.28:0/1292139567 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.720+0000 7f4183e00700 1 -- 172.21.15.28:0/1292139567 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f418410c410 con 0x7f418410b210 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.720+0000 7f4183e00700 1 --2- 172.21.15.28:0/1292139567 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418410b630 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f41740098a0 tx=0x7f4174009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bce11ae949da12ee server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.720+0000 7f4182a00700 1 -- 172.21.15.28:0/1292139567 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4174004160 con 0x7f418410b210 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.720+0000 7f4182a00700 1 -- 172.21.15.28:0/1292139567 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f4174004300 con 0x7f418410b210 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.720+0000 7f4182a00700 1 -- 172.21.15.28:0/1292139567 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4174011af0 con 0x7f418410b210 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.721+0000 7f418b000700 1 -- 172.21.15.28:0/1292139567 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 msgr2=0x7f418410b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:46.591 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.721+0000 7f418b000700 1 --2- 172.21.15.28:0/1292139567 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418410b630 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f41740098a0 tx=0x7f4174009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.721+0000 7f418b000700 1 -- 172.21.15.28:0/1292139567 shutdown_connections 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.721+0000 7f418b000700 1 --2- 172.21.15.28:0/1292139567 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418410b630 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.721+0000 7f418b000700 1 -- 172.21.15.28:0/1292139567 >> 172.21.15.28:0/1292139567 conn(0x7f418407e650 msgr2=0x7f418407eaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.721+0000 7f418b000700 1 -- 172.21.15.28:0/1292139567 shutdown_connections 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.721+0000 7f418b000700 1 -- 172.21.15.28:0/1292139567 wait complete. 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.722+0000 7f418b000700 1 Processor -- start 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.722+0000 7f418b000700 1 -- start start 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.723+0000 7f418b000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418419f1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.723+0000 7f418b000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f418410f7c0 con 0x7f418410b210 2024-05-20T19:52:46.592 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.723+0000 7f4183e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418419f1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:46.593 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.723+0000 7f4183e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418419f1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51982/0 (socket says 172.21.15.28:51982) 2024-05-20T19:52:46.593 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.723+0000 7f4183e00700 1 -- 172.21.15.28:0/570181001 learned_addr learned my addr 172.21.15.28:0/570181001 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:46.593 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.723+0000 7f4183e00700 1 -- 172.21.15.28:0/570181001 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f418419f730 con 0x7f418410b210 2024-05-20T19:52:46.593 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.724+0000 7f4183e00700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418419f1f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f41740117b0 tx=0x7f4174011b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:46.593 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.724+0000 7f4180c00700 1 -- 172.21.15.28:0/570181001 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4174003af0 con 0x7f418410b210 2024-05-20T19:52:46.593 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.724+0000 7f4180c00700 1 -- 172.21.15.28:0/570181001 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1070+0+0 (secure 0 0 0) 0x7f4174020070 con 0x7f418410b210 2024-05-20T19:52:46.593 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.724+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f418419f9c0 con 0x7f418410b210 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.724+0000 7f4180c00700 1 -- 172.21.15.28:0/570181001 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4174019630 con 0x7f418410b210 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.724+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f41841a26b0 con 0x7f418410b210 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.725+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4184064ba0 con 0x7f418410b210 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.726+0000 7f4180c00700 1 -- 172.21.15.28:0/570181001 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7f417401a9c0 con 0x7f418410b210 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.727+0000 7f4180c00700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f416803e4c0 0x7f4168040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.727+0000 7f4180c00700 1 -- 172.21.15.28:0/570181001 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f4174053a60 con 0x7f418410b210 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.727+0000 7f4183400700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f416803e4c0 0x7f4168040980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.728+0000 7f4183400700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f416803e4c0 0x7f4168040980 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4170006fd0 tx=0x7f4170006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:44.733+0000 7f4180c00700 1 -- 172.21.15.28:0/570181001 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f417400bdd0 con 0x7f418410b210 2024-05-20T19:52:46.594 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.008+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}) -- 0x7f4184109540 con 0x7f416803e4c0 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.021+0000 7f4180c00700 1 -- 172.21.15.28:0/570181001 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f4184109540 con 0x7f416803e4c0 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f416803e4c0 msgr2=0x7f4168040980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f416803e4c0 0x7f4168040980 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4170006fd0 tx=0x7f4170006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 msgr2=0x7f418419f1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418419f1f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f41740117b0 tx=0x7f4174011b70 comp rx=0 tx=0).stop 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 shutdown_connections 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f416803e4c0 0x7f4168040980 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 --2- 172.21.15.28:0/570181001 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f418410b210 0x7f418419f1f0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.024+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 >> 172.21.15.28:0/570181001 conn(0x7f418407e650 msgr2=0x7f4184108e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.025+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 shutdown_connections 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:45.025+0000 7f418b000700 1 -- 172.21.15.28:0/570181001 wait complete. 2024-05-20T19:52:46.595 INFO:teuthology.orchestra.run.smithi028.stdout:Deploying unmanaged mgr service... 2024-05-20T19:52:47.533 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:47 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:48.167 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-05-20T19:52:48.167 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.238+0000 7f182c600700 1 Processor -- start 2024-05-20T19:52:48.167 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.238+0000 7f182c600700 1 -- start start 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.239+0000 7f182c600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f1824109460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.239+0000 7f182c600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1824109a30 con 0x7f1824109040 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.239+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f1824109460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.239+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f1824109460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59612/0 (socket says 172.21.15.28:59612) 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.239+0000 7f1829c00700 1 -- 172.21.15.28:0/4026729310 learned_addr learned my addr 172.21.15.28:0/4026729310 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.240+0000 7f1829c00700 1 -- 172.21.15.28:0/4026729310 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f182410a1a0 con 0x7f1824109040 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.240+0000 7f1829c00700 1 --2- 172.21.15.28:0/4026729310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f1824109460 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f18180098a0 tx=0x7f1818009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c2ba4b6c1e756973 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.240+0000 7f1823e00700 1 -- 172.21.15.28:0/4026729310 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1818004160 con 0x7f1824109040 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.240+0000 7f1823e00700 1 -- 172.21.15.28:0/4026729310 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1818004300 con 0x7f1824109040 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.240+0000 7f1823e00700 1 -- 172.21.15.28:0/4026729310 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1818011a20 con 0x7f1824109040 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.241+0000 7f182c600700 1 -- 172.21.15.28:0/4026729310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 msgr2=0x7f1824109460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.241+0000 7f182c600700 1 --2- 172.21.15.28:0/4026729310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f1824109460 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f18180098a0 tx=0x7f1818009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:48.168 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.241+0000 7f182c600700 1 -- 172.21.15.28:0/4026729310 shutdown_connections 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.241+0000 7f182c600700 1 --2- 172.21.15.28:0/4026729310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f1824109460 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.241+0000 7f182c600700 1 -- 172.21.15.28:0/4026729310 >> 172.21.15.28:0/4026729310 conn(0x7f1824104640 msgr2=0x7f1824106a60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.241+0000 7f182c600700 1 -- 172.21.15.28:0/4026729310 shutdown_connections 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.241+0000 7f182c600700 1 -- 172.21.15.28:0/4026729310 wait complete. 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.242+0000 7f182c600700 1 Processor -- start 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.243+0000 7f182c600700 1 -- start start 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.243+0000 7f182c600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f182419f120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.243+0000 7f182c600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f182410d550 con 0x7f1824109040 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.243+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f182419f120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.243+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f182419f120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59624/0 (socket says 172.21.15.28:59624) 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.243+0000 7f1829c00700 1 -- 172.21.15.28:0/731548899 learned_addr learned my addr 172.21.15.28:0/731548899 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:48.169 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.244+0000 7f1829c00700 1 -- 172.21.15.28:0/731548899 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f182419f660 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.244+0000 7f1829c00700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f182419f120 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f1818009520 tx=0x7f1818011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.244+0000 7f1822000700 1 -- 172.21.15.28:0/731548899 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f18180039e0 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.244+0000 7f1822000700 1 -- 172.21.15.28:0/731548899 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f181801b8f0 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.244+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f182419f8f0 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.244+0000 7f1822000700 1 -- 172.21.15.28:0/731548899 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f1818019770 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.244+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f18241a25e0 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.246+0000 7f1822000700 1 -- 172.21.15.28:0/731548899 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7f1818019a20 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.246+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1824064ba0 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.246+0000 7f1822000700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f180c03e4c0 0x7f180c040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.246+0000 7f1822000700 1 -- 172.21.15.28:0/731548899 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f1818054000 con 0x7f1824109040 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.247+0000 7f1829200700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f180c03e4c0 0x7f180c040980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:48.170 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.247+0000 7f1829200700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f180c03e4c0 0x7f180c040980 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f1810006fd0 tx=0x7f1810006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.253+0000 7f1822000700 1 -- 172.21.15.28:0/731548899 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f181801f030 con 0x7f1824109040 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.532+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}) -- 0x7f1824105790 con 0x7f180c03e4c0 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.539+0000 7f1822000700 1 -- 172.21.15.28:0/731548899 <== mgr.14120 v2:172.21.15.28:6800/811935446 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f1824105790 con 0x7f180c03e4c0 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f180c03e4c0 msgr2=0x7f180c040980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f180c03e4c0 0x7f180c040980 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f1810006fd0 tx=0x7f1810006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 msgr2=0x7f182419f120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f182419f120 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f1818009520 tx=0x7f1818011aa0 comp rx=0 tx=0).stop 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 shutdown_connections 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f180c03e4c0 0x7f180c040980 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 --2- 172.21.15.28:0/731548899 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1824109040 0x7f182419f120 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 >> 172.21.15.28:0/731548899 conn(0x7f1824104640 msgr2=0x7f1824105080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 shutdown_connections 2024-05-20T19:52:48.171 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:47.542+0000 7f182c600700 1 -- 172.21.15.28:0/731548899 wait complete. 2024-05-20T19:52:48.467 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:48 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:48.468 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:48 smithi028 ceph-mon[31178]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:52:48.468 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:48 smithi028 ceph-mon[31178]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:52:48.468 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:48 smithi028 ceph-mon[31178]: Saving service mgr spec with placement count:2 2024-05-20T19:52:48.468 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:48 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:50.260 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.787+0000 7f720c800700 1 Processor -- start 2024-05-20T19:52:50.260 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.787+0000 7f720c800700 1 -- start start 2024-05-20T19:52:50.261 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.787+0000 7f720c800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f7204109470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:50.261 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.787+0000 7f720c800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7204109a40 con 0x7f7204109050 2024-05-20T19:52:50.261 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.788+0000 7f7209e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f7204109470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:50.261 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.788+0000 7f7209e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f7204109470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59630/0 (socket says 172.21.15.28:59630) 2024-05-20T19:52:50.261 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.788+0000 7f7209e00700 1 -- 172.21.15.28:0/418247582 learned_addr learned my addr 172.21.15.28:0/418247582 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:50.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.788+0000 7f7209e00700 1 -- 172.21.15.28:0/418247582 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f720410a200 con 0x7f7204109050 2024-05-20T19:52:50.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.788+0000 7f7209e00700 1 --2- 172.21.15.28:0/418247582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f7204109470 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f71f80098a0 tx=0x7f71f8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ba4634c7c573465 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:50.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.789+0000 7f7208a00700 1 -- 172.21.15.28:0/418247582 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71f8004160 con 0x7f7204109050 2024-05-20T19:52:50.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.789+0000 7f7208a00700 1 -- 172.21.15.28:0/418247582 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f71f8004300 con 0x7f7204109050 2024-05-20T19:52:50.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.789+0000 7f7208a00700 1 -- 172.21.15.28:0/418247582 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71f8011a20 con 0x7f7204109050 2024-05-20T19:52:50.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.790+0000 7f720c800700 1 -- 172.21.15.28:0/418247582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 msgr2=0x7f7204109470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:50.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.790+0000 7f720c800700 1 --2- 172.21.15.28:0/418247582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f7204109470 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f71f80098a0 tx=0x7f71f8009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:50.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.790+0000 7f720c800700 1 -- 172.21.15.28:0/418247582 shutdown_connections 2024-05-20T19:52:50.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.790+0000 7f720c800700 1 --2- 172.21.15.28:0/418247582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f7204109470 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:50.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.790+0000 7f720c800700 1 -- 172.21.15.28:0/418247582 >> 172.21.15.28:0/418247582 conn(0x7f72041045f0 msgr2=0x7f7204106a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:50.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.790+0000 7f720c800700 1 -- 172.21.15.28:0/418247582 shutdown_connections 2024-05-20T19:52:50.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.790+0000 7f720c800700 1 -- 172.21.15.28:0/418247582 wait complete. 2024-05-20T19:52:50.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.791+0000 7f720c800700 1 Processor -- start 2024-05-20T19:52:50.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.792+0000 7f720c800700 1 -- start start 2024-05-20T19:52:50.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.792+0000 7f720c800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f720419f150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:50.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.792+0000 7f720c800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f720410d5b0 con 0x7f7204109050 2024-05-20T19:52:50.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.792+0000 7f7209e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f720419f150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:50.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.793+0000 7f7209e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f720419f150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59634/0 (socket says 172.21.15.28:59634) 2024-05-20T19:52:50.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.793+0000 7f7209e00700 1 -- 172.21.15.28:0/903600244 learned_addr learned my addr 172.21.15.28:0/903600244 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:50.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.793+0000 7f7209e00700 1 -- 172.21.15.28:0/903600244 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f720419f690 con 0x7f7204109050 2024-05-20T19:52:50.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.793+0000 7f7209e00700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f720419f150 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f71f80119f0 tx=0x7f71f8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:50.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.794+0000 7f71fea00700 1 -- 172.21.15.28:0/903600244 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71f80039e0 con 0x7f7204109050 2024-05-20T19:52:50.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.794+0000 7f71fea00700 1 -- 172.21.15.28:0/903600244 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f71f801b8f0 con 0x7f7204109050 2024-05-20T19:52:50.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.794+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f720419f920 con 0x7f7204109050 2024-05-20T19:52:50.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.794+0000 7f71fea00700 1 -- 172.21.15.28:0/903600244 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f71f80237a0 con 0x7f7204109050 2024-05-20T19:52:50.267 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.794+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f72041a2610 con 0x7f7204109050 2024-05-20T19:52:50.267 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.795+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7204064ba0 con 0x7f7204109050 2024-05-20T19:52:50.267 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.795+0000 7f71fea00700 1 -- 172.21.15.28:0/903600244 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7f71f8023aa0 con 0x7f7204109050 2024-05-20T19:52:50.267 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.796+0000 7f71fea00700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f71ec03e420 0x7f71ec0408e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:50.267 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.796+0000 7f71fea00700 1 -- 172.21.15.28:0/903600244 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7f71f8053ea0 con 0x7f7204109050 2024-05-20T19:52:50.267 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.796+0000 7f7209400700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f71ec03e420 0x7f71ec0408e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:50.268 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.797+0000 7f7209400700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f71ec03e420 0x7f71ec0408e0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f71f4006fd0 tx=0x7f71f4006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:50.268 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:48.801+0000 7f71fea00700 1 -- 172.21.15.28:0/903600244 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f71f801f030 con 0x7f7204109050 2024-05-20T19:52:50.268 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.052+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) -- 0x7f72041a2b20 con 0x7f7204109050 2024-05-20T19:52:50.268 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.056+0000 7f71fea00700 1 -- 172.21.15.28:0/903600244 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v7)=0 v7) ==== 142+0+0 (secure 0 0 0) 0x7f71f80202b0 con 0x7f7204109050 2024-05-20T19:52:50.268 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f71ec03e420 msgr2=0x7f71ec0408e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:50.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f71ec03e420 0x7f71ec0408e0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f71f4006fd0 tx=0x7f71f4006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:50.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 msgr2=0x7f720419f150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:50.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f720419f150 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f71f80119f0 tx=0x7f71f8011ad0 comp rx=0 tx=0).stop 2024-05-20T19:52:50.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 shutdown_connections 2024-05-20T19:52:50.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7f71ec03e420 0x7f71ec0408e0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:50.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 --2- 172.21.15.28:0/903600244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7204109050 0x7f720419f150 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:50.270 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 >> 172.21.15.28:0/903600244 conn(0x7f72041045f0 msgr2=0x7f72041050e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:50.270 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 shutdown_connections 2024-05-20T19:52:50.270 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:49.059+0000 7f720c800700 1 -- 172.21.15.28:0/903600244 wait complete. 2024-05-20T19:52:50.415 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:50 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/903600244' entity='client.admin' 2024-05-20T19:52:50.415 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:50 smithi028 ceph-mon[31178]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:52:51.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:51 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:51.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:51 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/967611267' entity='client.admin' 2024-05-20T19:52:51.879 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.894+0000 7ff6ffc00700 1 Processor -- start 2024-05-20T19:52:51.879 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.894+0000 7ff6ffc00700 1 -- start start 2024-05-20T19:52:51.879 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.895+0000 7ff6ffc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f8107c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:51.879 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.895+0000 7ff6ffc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff6f807ce80 con 0x7ff6f8107800 2024-05-20T19:52:51.879 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.895+0000 7ff6fd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f8107c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:51.879 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.895+0000 7ff6fd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f8107c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59646/0 (socket says 172.21.15.28:59646) 2024-05-20T19:52:51.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.895+0000 7ff6fd200700 1 -- 172.21.15.28:0/3545758266 learned_addr learned my addr 172.21.15.28:0/3545758266 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:51.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.896+0000 7ff6fd200700 1 -- 172.21.15.28:0/3545758266 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff6f8108160 con 0x7ff6f8107800 2024-05-20T19:52:51.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.896+0000 7ff6fd200700 1 --2- 172.21.15.28:0/3545758266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f8107c20 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7ff6ec009e80 tx=0x7ff6ec00b330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5be5f21c3b5df69a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:51.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.897+0000 7ff6f7400700 1 -- 172.21.15.28:0/3545758266 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff6ec017070 con 0x7ff6f8107800 2024-05-20T19:52:51.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.897+0000 7ff6f7400700 1 -- 172.21.15.28:0/3545758266 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6ec00b8c0 con 0x7ff6f8107800 2024-05-20T19:52:51.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.897+0000 7ff6f7400700 1 -- 172.21.15.28:0/3545758266 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff6ec004490 con 0x7ff6f8107800 2024-05-20T19:52:51.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.897+0000 7ff6ffc00700 1 -- 172.21.15.28:0/3545758266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 msgr2=0x7ff6f8107c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:51.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.897+0000 7ff6ffc00700 1 --2- 172.21.15.28:0/3545758266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f8107c20 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7ff6ec009e80 tx=0x7ff6ec00b330 comp rx=0 tx=0).stop 2024-05-20T19:52:51.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.898+0000 7ff6ffc00700 1 -- 172.21.15.28:0/3545758266 shutdown_connections 2024-05-20T19:52:51.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.898+0000 7ff6ffc00700 1 --2- 172.21.15.28:0/3545758266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f8107c20 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:51.882 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.898+0000 7ff6ffc00700 1 -- 172.21.15.28:0/3545758266 >> 172.21.15.28:0/3545758266 conn(0x7ff6f8103480 msgr2=0x7ff6f81058a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:51.882 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.898+0000 7ff6ffc00700 1 -- 172.21.15.28:0/3545758266 shutdown_connections 2024-05-20T19:52:51.886 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.898+0000 7ff6ffc00700 1 -- 172.21.15.28:0/3545758266 wait complete. 2024-05-20T19:52:51.886 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.899+0000 7ff6ffc00700 1 Processor -- start 2024-05-20T19:52:51.886 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.899+0000 7ff6ffc00700 1 -- start start 2024-05-20T19:52:51.886 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.899+0000 7ff6ffc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f819f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:51.886 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.899+0000 7ff6ffc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff6f81091d0 con 0x7ff6f8107800 2024-05-20T19:52:51.886 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.900+0000 7ff6fd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f819f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.900+0000 7ff6fd200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f819f0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59658/0 (socket says 172.21.15.28:59658) 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.900+0000 7ff6fd200700 1 -- 172.21.15.28:0/967611267 learned_addr learned my addr 172.21.15.28:0/967611267 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.900+0000 7ff6fd200700 1 -- 172.21.15.28:0/967611267 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff6f819f630 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.900+0000 7ff6fd200700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f819f0f0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7ff6ec004220 tx=0x7ff6ec0093e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.901+0000 7ff6f5600700 1 -- 172.21.15.28:0/967611267 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff6ec020020 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.901+0000 7ff6f5600700 1 -- 172.21.15.28:0/967611267 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6ec003b50 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.901+0000 7ff6f5600700 1 -- 172.21.15.28:0/967611267 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7ff6ec017040 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.901+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff6f819f8c0 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.901+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff6f81a25b0 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.902+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff6f8064ba0 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.902+0000 7ff6f5600700 1 -- 172.21.15.28:0/967611267 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7ff6ec023020 con 0x7ff6f8107800 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.903+0000 7ff6f5600700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7ff6e003e4c0 0x7ff6e0040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:51.887 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.903+0000 7ff6f5600700 1 -- 172.21.15.28:0/967611267 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7ff6ec00f050 con 0x7ff6f8107800 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.903+0000 7ff6f7e00700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7ff6e003e4c0 0x7ff6e0040980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.904+0000 7ff6f7e00700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7ff6e003e4c0 0x7ff6e0040980 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7ff6e4006fd0 tx=0x7ff6e4006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:50.910+0000 7ff6f5600700 1 -- 172.21.15.28:0/967611267 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7ff6ec02a020 con 0x7ff6f8107800 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.152+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) -- 0x7ff6f802e760 con 0x7ff6f8107800 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.157+0000 7ff6f5600700 1 -- 172.21.15.28:0/967611267 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v8)=0 v8) ==== 130+0+0 (secure 0 0 0) 0x7ff6ec02b260 con 0x7ff6f8107800 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7ff6e003e4c0 msgr2=0x7ff6e0040980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7ff6e003e4c0 0x7ff6e0040980 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7ff6e4006fd0 tx=0x7ff6e4006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 msgr2=0x7ff6f819f0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f819f0f0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7ff6ec004220 tx=0x7ff6ec0093e0 comp rx=0 tx=0).stop 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 shutdown_connections 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7ff6e003e4c0 0x7ff6e0040980 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:51.888 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 --2- 172.21.15.28:0/967611267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff6f8107800 0x7ff6f819f0f0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:51.889 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.160+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 >> 172.21.15.28:0/967611267 conn(0x7ff6f8103480 msgr2=0x7ff6f8195d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:51.889 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.161+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 shutdown_connections 2024-05-20T19:52:51.889 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:51.161+0000 7ff6ffc00700 1 -- 172.21.15.28:0/967611267 wait complete. 2024-05-20T19:52:51.889 INFO:teuthology.orchestra.run.smithi028.stdout:Enabling the dashboard module... 2024-05-20T19:52:52.450 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:52 smithi028 ceph-mon[31178]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:52:53.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:53 smithi028 ceph-mon[31178]: from='mgr.14120 172.21.15.28:0/20988467' entity='mgr.x' 2024-05-20T19:52:53.820 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:53 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3922751771' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-20T19:52:53.821 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: ignoring --setuser ceph since I am not root 2024-05-20T19:52:53.821 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: ignoring --setgroup ceph since I am not root 2024-05-20T19:52:54.139 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:53.883+0000 7f7c4e49b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T19:52:54.140 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:54.137+0000 7f7c4e49b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:52:54.390 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:54.274+0000 7f7c4e49b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T19:52:54.690 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:54 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3922751771' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-20T19:52:54.690 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:54 smithi028 ceph-mon[31178]: mgrmap e9: x(active, since 26s) 2024-05-20T19:52:55.035 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:54.688+0000 7f7c4e49b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T19:52:55.036 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:54.820+0000 7f7c4e49b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T19:52:55.044 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.481+0000 7fa268800700 1 Processor -- start 2024-05-20T19:52:55.050 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.481+0000 7fa268800700 1 -- start start 2024-05-20T19:52:55.050 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.481+0000 7fa268800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26007d2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:55.050 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.482+0000 7fa268800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa26007d7f0 con 0x7fa26007edb0 2024-05-20T19:52:55.050 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.482+0000 7fa265e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26007d2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:55.050 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.482+0000 7fa265e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26007d2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59670/0 (socket says 172.21.15.28:59670) 2024-05-20T19:52:55.050 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.482+0000 7fa265e00700 1 -- 172.21.15.28:0/1118827677 learned_addr learned my addr 172.21.15.28:0/1118827677 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.483+0000 7fa265e00700 1 -- 172.21.15.28:0/1118827677 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa26007d970 con 0x7fa26007edb0 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.483+0000 7fa265e00700 1 --2- 172.21.15.28:0/1118827677 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26007d2b0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fa2540098a0 tx=0x7fa254009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=68a69cdd8270641e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.484+0000 7fa264a00700 1 -- 172.21.15.28:0/1118827677 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa254004160 con 0x7fa26007edb0 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.484+0000 7fa264a00700 1 -- 172.21.15.28:0/1118827677 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa254004300 con 0x7fa26007edb0 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.484+0000 7fa264a00700 1 -- 172.21.15.28:0/1118827677 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa254011a20 con 0x7fa26007edb0 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.484+0000 7fa268800700 1 -- 172.21.15.28:0/1118827677 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 msgr2=0x7fa26007d2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.484+0000 7fa268800700 1 --2- 172.21.15.28:0/1118827677 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26007d2b0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fa2540098a0 tx=0x7fa254009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.485+0000 7fa268800700 1 -- 172.21.15.28:0/1118827677 shutdown_connections 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.485+0000 7fa268800700 1 --2- 172.21.15.28:0/1118827677 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26007d2b0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.485+0000 7fa268800700 1 -- 172.21.15.28:0/1118827677 >> 172.21.15.28:0/1118827677 conn(0x7fa260104620 msgr2=0x7fa260106a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:55.051 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.485+0000 7fa268800700 1 -- 172.21.15.28:0/1118827677 shutdown_connections 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.485+0000 7fa268800700 1 -- 172.21.15.28:0/1118827677 wait complete. 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.486+0000 7fa268800700 1 Processor -- start 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.486+0000 7fa268800700 1 -- start start 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.486+0000 7fa268800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.486+0000 7fa268800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa26010f560 con 0x7fa26007edb0 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.487+0000 7fa265e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.487+0000 7fa265e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26019f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59678/0 (socket says 172.21.15.28:59678) 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.487+0000 7fa265e00700 1 -- 172.21.15.28:0/3922751771 learned_addr learned my addr 172.21.15.28:0/3922751771 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.487+0000 7fa265e00700 1 -- 172.21.15.28:0/3922751771 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa26019f590 con 0x7fa26007edb0 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.487+0000 7fa265e00700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26019f050 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fa254000c00 tx=0x7fa254011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.488+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa254003aa0 con 0x7fa26007edb0 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.488+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa25401b8f0 con 0x7fa26007edb0 2024-05-20T19:52:55.052 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.488+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa26019f820 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.488+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fa254023870 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.488+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa2601a2510 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.489+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa260064ba0 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.490+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 8) ==== 50222+0+0 (secure 0 0 0) 0x7fa254023b20 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.490+0000 7fa25aa00700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fa24803e4c0 0x7fa248040980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.490+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fa254054110 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.490+0000 7fa265400700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fa24803e4c0 0x7fa248040980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.491+0000 7fa265400700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fa24803e4c0 0x7fa248040980 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fa250006fd0 tx=0x7fa250006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.500+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa25401f030 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:52.795+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) -- 0x7fa2601a26e0 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.557+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v9) ==== 88+0+0 (secure 0 0 0) 0x7fa25400b3e0 con 0x7fa26007edb0 2024-05-20T19:52:55.053 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.561+0000 7fa25aa00700 1 -- 172.21.15.28:0/3922751771 <== mon.0 v2:172.21.15.28:3300/0 8 ==== mgrmap(e 9) ==== 50235+0+0 (secure 0 0 0) 0x7fa254019ce0 con 0x7fa26007edb0 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fa24803e4c0 msgr2=0x7fa248040980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fa24803e4c0 0x7fa248040980 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fa250006fd0 tx=0x7fa250006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 msgr2=0x7fa26019f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26019f050 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fa254000c00 tx=0x7fa254011aa0 comp rx=0 tx=0).stop 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 shutdown_connections 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fa24803e4c0 0x7fa248040980 secure :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fa250006fd0 tx=0x7fa250006e40 comp rx=0 tx=0).stop 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 --2- 172.21.15.28:0/3922751771 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa26007edb0 0x7fa26019f050 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.564+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 >> 172.21.15.28:0/3922751771 conn(0x7fa260104620 msgr2=0x7fa260105070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.565+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 shutdown_connections 2024-05-20T19:52:55.054 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:53.565+0000 7fa268800700 1 -- 172.21.15.28:0/3922751771 wait complete. 2024-05-20T19:52:56.246 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:52:55 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2197014347' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-20T19:52:56.541 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:56.446+0000 7f7c4e49b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "available": true, 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.606+0000 7fbf2da00700 1 Processor -- start 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.606+0000 7fbf2da00700 1 -- start start 2024-05-20T19:52:56.555 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.606+0000 7fbf2da00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2807d2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.606+0000 7fbf2da00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbf2807d7f0 con 0x7fbf2807ed40 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.607+0000 7fbf27e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2807d2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.607+0000 7fbf27e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2807d2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59704/0 (socket says 172.21.15.28:59704) 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.607+0000 7fbf27e00700 1 -- 172.21.15.28:0/875775107 learned_addr learned my addr 172.21.15.28:0/875775107 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.607+0000 7fbf27e00700 1 -- 172.21.15.28:0/875775107 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbf2807d970 con 0x7fbf2807ed40 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.607+0000 7fbf27e00700 1 --2- 172.21.15.28:0/875775107 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2807d2b0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fbf180098a0 tx=0x7fbf18009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=909f2ee943076be3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.608+0000 7fbf26a00700 1 -- 172.21.15.28:0/875775107 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbf18004160 con 0x7fbf2807ed40 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.608+0000 7fbf26a00700 1 -- 172.21.15.28:0/875775107 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbf18004300 con 0x7fbf2807ed40 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.608+0000 7fbf26a00700 1 -- 172.21.15.28:0/875775107 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbf18011a20 con 0x7fbf2807ed40 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.608+0000 7fbf2da00700 1 -- 172.21.15.28:0/875775107 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 msgr2=0x7fbf2807d2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.608+0000 7fbf2da00700 1 --2- 172.21.15.28:0/875775107 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2807d2b0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fbf180098a0 tx=0x7fbf18009bb0 comp rx=0 tx=0).stop 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.609+0000 7fbf2da00700 1 -- 172.21.15.28:0/875775107 shutdown_connections 2024-05-20T19:52:56.556 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.609+0000 7fbf2da00700 1 --2- 172.21.15.28:0/875775107 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2807d2b0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.609+0000 7fbf2da00700 1 -- 172.21.15.28:0/875775107 >> 172.21.15.28:0/875775107 conn(0x7fbf28104460 msgr2=0x7fbf281068c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.609+0000 7fbf2da00700 1 -- 172.21.15.28:0/875775107 shutdown_connections 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.609+0000 7fbf2da00700 1 -- 172.21.15.28:0/875775107 wait complete. 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.610+0000 7fbf2da00700 1 Processor -- start 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.610+0000 7fbf2da00700 1 -- start start 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.610+0000 7fbf2da00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2819ef30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.610+0000 7fbf2da00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbf2810f310 con 0x7fbf2807ed40 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.611+0000 7fbf27e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2819ef30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.611+0000 7fbf27e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2819ef30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59706/0 (socket says 172.21.15.28:59706) 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.611+0000 7fbf27e00700 1 -- 172.21.15.28:0/2197014347 learned_addr learned my addr 172.21.15.28:0/2197014347 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.611+0000 7fbf27e00700 1 -- 172.21.15.28:0/2197014347 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbf2819f470 con 0x7fbf2807ed40 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.611+0000 7fbf27e00700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2819ef30 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbf18004660 tx=0x7fbf18011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:52:56.557 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.612+0000 7fbf24c00700 1 -- 172.21.15.28:0/2197014347 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbf18003aa0 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.612+0000 7fbf24c00700 1 -- 172.21.15.28:0/2197014347 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbf1801b8f0 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.612+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbf2819f700 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.612+0000 7fbf24c00700 1 -- 172.21.15.28:0/2197014347 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fbf18023900 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.612+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbf281a23f0 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.613+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbf2806a670 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.613+0000 7fbf24c00700 1 -- 172.21.15.28:0/2197014347 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 9) ==== 50235+0+0 (secure 0 0 0) 0x7fbf1801baa0 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.614+0000 7fbf24c00700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 0x7fbf0c0409e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.614+0000 7fbf24c00700 1 -- 172.21.15.28:0/2197014347 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fbf18053e70 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.614+0000 7fbf27400700 1 -- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 msgr2=0x7fbf0c0409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/811935446 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.614+0000 7fbf27400700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 0x7fbf0c0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.620+0000 7fbf24c00700 1 -- 172.21.15.28:0/2197014347 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fbf1801f030 con 0x7fbf2807ed40 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.815+0000 7fbf27400700 1 -- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 msgr2=0x7fbf0c0409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/811935446 2024-05-20T19:52:56.558 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.815+0000 7fbf27400700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 0x7fbf0c0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.911+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) -- 0x7fbf281a25c0 con 0x7fbf2807ed40 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.912+0000 7fbf24c00700 1 -- 172.21.15.28:0/2197014347 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v9) ==== 56+0+88 (secure 0 0 0) 0x7fbf180202b0 con 0x7fbf2807ed40 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 msgr2=0x7fbf0c0409e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 0x7fbf0c0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 msgr2=0x7fbf2819ef30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2819ef30 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbf18004660 tx=0x7fbf18011aa0 comp rx=0 tx=0).stop 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 shutdown_connections 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fbf0c03e520 0x7fbf0c0409e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 --2- 172.21.15.28:0/2197014347 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf2807ed40 0x7fbf2819ef30 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 >> 172.21.15.28:0/2197014347 conn(0x7fbf28104460 msgr2=0x7fbf28105000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.916+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 shutdown_connections 2024-05-20T19:52:56.559 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:55.917+0000 7fbf2da00700 1 -- 172.21.15.28:0/2197014347 wait complete. 2024-05-20T19:52:56.560 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for the mgr to restart... 2024-05-20T19:52:56.560 INFO:teuthology.orchestra.run.smithi028.stdout:Waiting for mgr epoch 9... 2024-05-20T19:52:56.804 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:56.698+0000 7f7c4e49b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T19:52:57.192 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:56.924+0000 7f7c4e49b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T19:52:57.457 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:57.327+0000 7f7c4e49b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T19:52:57.736 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:57.455+0000 7f7c4e49b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T19:52:57.736 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:57.734+0000 7f7c4e49b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T19:52:58.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:58.000+0000 7f7c4e49b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:52:59.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:58.822+0000 7f7c4e49b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T19:52:59.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:59.119+0000 7f7c4e49b200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T19:53:00.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:59.693+0000 7f7c4e49b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T19:53:00.057 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:52:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:52:59.827+0000 7f7c4e49b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T19:53:00.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:00 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:00.307+0000 7f7c4e49b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T19:53:01.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:00 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:00.780+0000 7f7c4e49b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T19:53:01.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:00 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:00.913+0000 7f7c4e49b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T19:53:01.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:01.250+0000 7f7c4e49b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T19:53:01.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:01.437+0000 7f7c4e49b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T19:53:01.985 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:01.573+0000 7f7c4e49b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T19:53:02.306 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:01.983+0000 7f7c4e49b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T19:53:02.306 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:02 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:02.114+0000 7f7c4e49b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T19:53:03.219 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:02 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:02.917+0000 7f7c4e49b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T19:53:03.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 19:53:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T19:53:03.217+0000 7f7c4e49b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T19:53:03.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: Active manager daemon x restarted 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: Activating manager daemon x 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: osdmap e3: 0 total, 0 up, 0 in 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: mgrmap e10: x(active, starting, since 0.0100824s) 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T19:53:03.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:03 smithi028 ceph-mon[31178]: Manager daemon x is now available 2024-05-20T19:53:04.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:04.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T19:53:04.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T19:53:04.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:04 smithi028 ceph-mon[31178]: mgrmap e11: x(active, since 1.01136s) 2024-05-20T19:53:05.261 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout { 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout "initialized": true 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout } 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.190+0000 7fb1dc000700 1 Processor -- start 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.190+0000 7fb1dc000700 1 -- start start 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.191+0000 7fb1dc000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d407d2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.191+0000 7fb1dc000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb1d407d7f0 con 0x7fb1d407ed40 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.191+0000 7fb1d9600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d407d2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.192+0000 7fb1d9600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d407d2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56776/0 (socket says 172.21.15.28:56776) 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.192+0000 7fb1d9600700 1 -- 172.21.15.28:0/1352628279 learned_addr learned my addr 172.21.15.28:0/1352628279 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:05.262 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.192+0000 7fb1d9600700 1 -- 172.21.15.28:0/1352628279 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1d407d970 con 0x7fb1d407ed40 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.192+0000 7fb1d9600700 1 --2- 172.21.15.28:0/1352628279 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d407d2b0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fb1c80098a0 tx=0x7fb1c8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6d3d8d961371629 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.193+0000 7fb1d3e00700 1 -- 172.21.15.28:0/1352628279 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1c8004160 con 0x7fb1d407ed40 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.193+0000 7fb1d3e00700 1 -- 172.21.15.28:0/1352628279 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb1c8004300 con 0x7fb1d407ed40 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.193+0000 7fb1d3e00700 1 -- 172.21.15.28:0/1352628279 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1c8011a20 con 0x7fb1d407ed40 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.193+0000 7fb1dc000700 1 -- 172.21.15.28:0/1352628279 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 msgr2=0x7fb1d407d2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.193+0000 7fb1dc000700 1 --2- 172.21.15.28:0/1352628279 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d407d2b0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fb1c80098a0 tx=0x7fb1c8009bb0 comp rx=0 tx=0).stop 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.194+0000 7fb1dc000700 1 -- 172.21.15.28:0/1352628279 shutdown_connections 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.194+0000 7fb1dc000700 1 --2- 172.21.15.28:0/1352628279 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d407d2b0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.194+0000 7fb1dc000700 1 -- 172.21.15.28:0/1352628279 >> 172.21.15.28:0/1352628279 conn(0x7fb1d4104680 msgr2=0x7fb1d4106ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.194+0000 7fb1dc000700 1 -- 172.21.15.28:0/1352628279 shutdown_connections 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.194+0000 7fb1dc000700 1 -- 172.21.15.28:0/1352628279 wait complete. 2024-05-20T19:53:05.263 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.195+0000 7fb1dc000700 1 Processor -- start 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.195+0000 7fb1dc000700 1 -- start start 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.195+0000 7fb1dc000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d419f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.195+0000 7fb1dc000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb1d410f510 con 0x7fb1d407ed40 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.196+0000 7fb1d9600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d419f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.196+0000 7fb1d9600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d419f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56786/0 (socket says 172.21.15.28:56786) 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.196+0000 7fb1d9600700 1 -- 172.21.15.28:0/981181642 learned_addr learned my addr 172.21.15.28:0/981181642 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.196+0000 7fb1d9600700 1 -- 172.21.15.28:0/981181642 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1d419f610 con 0x7fb1d407ed40 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.196+0000 7fb1d9600700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d419f0d0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fb1c8000c00 tx=0x7fb1c8011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.197+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1c8003aa0 con 0x7fb1d407ed40 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.197+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb1c801b8f0 con 0x7fb1d407ed40 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.197+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb1d419f8a0 con 0x7fb1d407ed40 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.197+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fb1c8023870 con 0x7fb1d407ed40 2024-05-20T19:53:05.264 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.197+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb1d41a2590 con 0x7fb1d407ed40 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.199+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 9) ==== 50235+0+0 (secure 0 0 0) 0x7fb1c801b440 con 0x7fb1d407ed40 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.199+0000 7fb1d2000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.199+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 --> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fb1bc0410f0 con 0x7fb1bc03e520 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.199+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(2..2 src has 1..2) ==== 940+0+0 (secure 0 0 0) 0x7fb1c8053ed0 con 0x7fb1d407ed40 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.199+0000 7fb1d8c00700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 msgr2=0x7fb1bc0409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/811935446 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.199+0000 7fb1d8c00700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.400+0000 7fb1d8c00700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 msgr2=0x7fb1bc0409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/811935446 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.400+0000 7fb1d8c00700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.801+0000 7fb1d8c00700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 msgr2=0x7fb1bc0409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/811935446 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:57.801+0000 7fb1d8c00700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-05-20T19:53:05.265 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:58.602+0000 7fb1d8c00700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 msgr2=0x7fb1bc0409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/811935446 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:52:58.602+0000 7fb1d8c00700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:00.204+0000 7fb1d8c00700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 msgr2=0x7fb1bc0409e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/811935446 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:00.204+0000 7fb1d8c00700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:03.226+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mgrmap(e 10) ==== 50002+0+0 (secure 0 0 0) 0x7fb1c801acb0 con 0x7fb1d407ed40 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:03.227+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 msgr2=0x7fb1bc0409e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:03.227+0000 7fb1d2000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.229+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mgrmap(e 11) ==== 50129+0+0 (secure 0 0 0) 0x7fb1c8055300 con 0x7fb1d407ed40 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.230+0000 7fb1d2000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1bc041fa0 0x7fb1bc044390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.230+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fb1c8008e50 con 0x7fb1bc041fa0 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.231+0000 7fb1d8c00700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1bc041fa0 0x7fb1bc044390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:05.266 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.232+0000 7fb1d8c00700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1bc041fa0 0x7fb1bc044390 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fb1c0003a10 tx=0x7fb1c0009270 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:05.267 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.234+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== command_reply(tid 0: 0 ) ==== 8+0+9564 (secure 0 0 0) 0x7fb1c8008e50 con 0x7fb1bc041fa0 2024-05-20T19:53:05.268 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.245+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- command(tid 1: {"prefix": "mgr_status"}) -- 0x7fb1d406a670 con 0x7fb1bc041fa0 2024-05-20T19:53:05.268 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.245+0000 7fb1d2000700 1 -- 172.21.15.28:0/981181642 <== mgr.14152 v2:172.21.15.28:6800/2019900521 2 ==== command_reply(tid 1: 0 ) ==== 8+0+52 (secure 0 0 0) 0x7fb1d406a670 con 0x7fb1bc041fa0 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1bc041fa0 msgr2=0x7fb1bc044390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1bc041fa0 0x7fb1bc044390 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fb1c0003a10 tx=0x7fb1c0009270 comp rx=0 tx=0).stop 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 msgr2=0x7fb1d419f0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d419f0d0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fb1c8000c00 tx=0x7fb1c8011aa0 comp rx=0 tx=0).stop 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 shutdown_connections 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1bc041fa0 0x7fb1bc044390 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb1d407ed40 0x7fb1d419f0d0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 --2- 172.21.15.28:0/981181642 >> [v2:172.21.15.28:6800/811935446,v1:172.21.15.28:6801/811935446] conn(0x7fb1bc03e520 0x7fb1bc0409e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 >> 172.21.15.28:0/981181642 conn(0x7fb1d4104680 msgr2=0x7fb1d4105190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.246+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 shutdown_connections 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:04.247+0000 7fb1dc000700 1 -- 172.21.15.28:0/981181642 wait complete. 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:mgr epoch 9 is available 2024-05-20T19:53:05.269 INFO:teuthology.orchestra.run.smithi028.stdout:Generating a dashboard self-signed certificate... 2024-05-20T19:53:05.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:05 smithi028 ceph-mon[31178]: [20/May/2024:19:53:04] ENGINE Bus STARTING 2024-05-20T19:53:05.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:05 smithi028 ceph-mon[31178]: [20/May/2024:19:53:04] ENGINE Serving on https://172.21.15.28:7150 2024-05-20T19:53:06.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:06 smithi028 ceph-mon[31178]: [20/May/2024:19:53:04] ENGINE Serving on http://172.21.15.28:8765 2024-05-20T19:53:06.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:06 smithi028 ceph-mon[31178]: [20/May/2024:19:53:04] ENGINE Bus STARTED 2024-05-20T19:53:06.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:06 smithi028 ceph-mon[31178]: mgrmap e12: x(active, since 2s) 2024-05-20T19:53:06.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:07.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-05-20T19:53:07.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.194+0000 7f51c5a00700 1 Processor -- start 2024-05-20T19:53:07.331 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.194+0000 7f51c5a00700 1 -- start start 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.195+0000 7f51c5a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c007d310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.195+0000 7f51c5a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f51c007d850 con 0x7f51c007ee10 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.196+0000 7f51bea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c007d310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.196+0000 7f51bea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c007d310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56836/0 (socket says 172.21.15.28:56836) 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.196+0000 7f51bea00700 1 -- 172.21.15.28:0/4204634915 learned_addr learned my addr 172.21.15.28:0/4204634915 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.196+0000 7f51bea00700 1 -- 172.21.15.28:0/4204634915 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51c007d9d0 con 0x7f51c007ee10 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.197+0000 7f51bea00700 1 --2- 172.21.15.28:0/4204634915 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c007d310 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f51b00098a0 tx=0x7f51b0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8e7bc0ad036096b4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.197+0000 7f51be000700 1 -- 172.21.15.28:0/4204634915 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f51b0004160 con 0x7f51c007ee10 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.197+0000 7f51be000700 1 -- 172.21.15.28:0/4204634915 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51b0004300 con 0x7f51c007ee10 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.197+0000 7f51be000700 1 -- 172.21.15.28:0/4204634915 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f51b0011a20 con 0x7f51c007ee10 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.198+0000 7f51c5a00700 1 -- 172.21.15.28:0/4204634915 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 msgr2=0x7f51c007d310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:07.332 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.198+0000 7f51c5a00700 1 --2- 172.21.15.28:0/4204634915 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c007d310 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f51b00098a0 tx=0x7f51b0009bb0 comp rx=0 tx=0).stop 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.198+0000 7f51c5a00700 1 -- 172.21.15.28:0/4204634915 shutdown_connections 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.198+0000 7f51c5a00700 1 --2- 172.21.15.28:0/4204634915 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c007d310 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.198+0000 7f51c5a00700 1 -- 172.21.15.28:0/4204634915 >> 172.21.15.28:0/4204634915 conn(0x7f51c0104620 msgr2=0x7f51c0106a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.198+0000 7f51c5a00700 1 -- 172.21.15.28:0/4204634915 shutdown_connections 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.199+0000 7f51c5a00700 1 -- 172.21.15.28:0/4204634915 wait complete. 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.201+0000 7f51c5a00700 1 Processor -- start 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.201+0000 7f51c5a00700 1 -- start start 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.201+0000 7f51c5a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.201+0000 7f51c5a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f51c010f560 con 0x7f51c007ee10 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.202+0000 7f51bea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.202+0000 7f51bea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c019f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56840/0 (socket says 172.21.15.28:56840) 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.202+0000 7f51bea00700 1 -- 172.21.15.28:0/1284397304 learned_addr learned my addr 172.21.15.28:0/1284397304 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:07.333 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.202+0000 7f51bea00700 1 -- 172.21.15.28:0/1284397304 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51c019f590 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.203+0000 7f51bea00700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c019f050 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f51b0000c00 tx=0x7f51b0011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.203+0000 7f51b7400700 1 -- 172.21.15.28:0/1284397304 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f51b0003aa0 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.203+0000 7f51b7400700 1 -- 172.21.15.28:0/1284397304 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51b001b8f0 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.203+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f51c019f820 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.203+0000 7f51b7400700 1 -- 172.21.15.28:0/1284397304 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f51b0023870 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.203+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f51c01a2510 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.204+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f51c0064ba0 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.205+0000 7f51b7400700 1 -- 172.21.15.28:0/1284397304 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 12) ==== 50235+0+0 (secure 0 0 0) 0x7f51b001b440 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.205+0000 7f51b7400700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f51a403e520 0x7f51a40409e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.205+0000 7f51b7400700 1 -- 172.21.15.28:0/1284397304 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f51b0053f70 con 0x7f51c007ee10 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.206+0000 7f51b7e00700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f51a403e520 0x7f51a40409e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:07.334 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.206+0000 7f51b7e00700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f51a403e520 0x7f51a40409e0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f51a8006fd0 tx=0x7f51a8006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.211+0000 7f51b7400700 1 -- 172.21.15.28:0/1284397304 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f51b001f030 con 0x7f51c007ee10 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.459+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) -- 0x7f51c0105780 con 0x7f51a403e520 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.610+0000 7f51b7400700 1 -- 172.21.15.28:0/1284397304 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+32 (secure 0 0 0) 0x7f51c0105780 con 0x7f51a403e520 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f51a403e520 msgr2=0x7f51a40409e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f51a403e520 0x7f51a40409e0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f51a8006fd0 tx=0x7f51a8006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 msgr2=0x7f51c019f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c019f050 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f51b0000c00 tx=0x7f51b0011aa0 comp rx=0 tx=0).stop 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 shutdown_connections 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f51a403e520 0x7f51a40409e0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 --2- 172.21.15.28:0/1284397304 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51c007ee10 0x7f51c019f050 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 >> 172.21.15.28:0/1284397304 conn(0x7f51c0104620 msgr2=0x7f51c0105070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:07.335 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 shutdown_connections 2024-05-20T19:53:07.336 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:06.613+0000 7f51c5a00700 1 -- 172.21.15.28:0/1284397304 wait complete. 2024-05-20T19:53:07.336 INFO:teuthology.orchestra.run.smithi028.stdout:Creating initial admin user... 2024-05-20T19:53:07.892 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:07 smithi028 ceph-mon[31178]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:07.892 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:07.892 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:08 smithi028 ceph-mon[31178]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:09.065 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$2JouuKEWl0TGe0itV4/ro.ZwyIHjmx1ZwHbMU7.rO8A0uIv2si8Aa", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1716234788, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-20T19:53:09.065 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.891+0000 7f4aa7800700 1 Processor -- start 2024-05-20T19:53:09.065 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.891+0000 7f4aa7800700 1 -- start start 2024-05-20T19:53:09.065 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.892+0000 7f4aa7800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa0107ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.892+0000 7f4aa7800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4aa007ce00 con 0x7f4aa0107880 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.892+0000 7f4aa4e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa0107ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.892+0000 7f4aa4e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa0107ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38772/0 (socket says 172.21.15.28:38772) 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.893+0000 7f4aa4e00700 1 -- 172.21.15.28:0/3645598855 learned_addr learned my addr 172.21.15.28:0/3645598855 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.893+0000 7f4aa4e00700 1 -- 172.21.15.28:0/3645598855 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4aa01081e0 con 0x7f4aa0107880 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.893+0000 7f4aa4e00700 1 --2- 172.21.15.28:0/3645598855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa0107ca0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f4a940098a0 tx=0x7f4a94009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a439b2c8ff2d3ebf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.894+0000 7f4a9f400700 1 -- 172.21.15.28:0/3645598855 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4a94004160 con 0x7f4aa0107880 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.894+0000 7f4a9f400700 1 -- 172.21.15.28:0/3645598855 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a94004300 con 0x7f4aa0107880 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.894+0000 7f4a9f400700 1 -- 172.21.15.28:0/3645598855 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4a94011a20 con 0x7f4aa0107880 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.894+0000 7f4aa7800700 1 -- 172.21.15.28:0/3645598855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 msgr2=0x7f4aa0107ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.894+0000 7f4aa7800700 1 --2- 172.21.15.28:0/3645598855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa0107ca0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f4a940098a0 tx=0x7f4a94009bb0 comp rx=0 tx=0).stop 2024-05-20T19:53:09.066 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.895+0000 7f4aa7800700 1 -- 172.21.15.28:0/3645598855 shutdown_connections 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.895+0000 7f4aa7800700 1 --2- 172.21.15.28:0/3645598855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa0107ca0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.895+0000 7f4aa7800700 1 -- 172.21.15.28:0/3645598855 >> 172.21.15.28:0/3645598855 conn(0x7f4aa0103480 msgr2=0x7f4aa01058a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.895+0000 7f4aa7800700 1 -- 172.21.15.28:0/3645598855 shutdown_connections 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.895+0000 7f4aa7800700 1 -- 172.21.15.28:0/3645598855 wait complete. 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.896+0000 7f4aa7800700 1 Processor -- start 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.896+0000 7f4aa7800700 1 -- start start 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.897+0000 7f4aa7800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa019f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.897+0000 7f4aa7800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4aa0109250 con 0x7f4aa0107880 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.897+0000 7f4aa4e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa019f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.897+0000 7f4aa4e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa019f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38774/0 (socket says 172.21.15.28:38774) 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.897+0000 7f4aa4e00700 1 -- 172.21.15.28:0/2869970306 learned_addr learned my addr 172.21.15.28:0/2869970306 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.897+0000 7f4aa4e00700 1 -- 172.21.15.28:0/2869970306 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4aa019f5c0 con 0x7f4aa0107880 2024-05-20T19:53:09.067 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.898+0000 7f4aa4e00700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa019f080 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4a94000c00 tx=0x7f4a94011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.898+0000 7f4a9d600700 1 -- 172.21.15.28:0/2869970306 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4a94003aa0 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.898+0000 7f4a9d600700 1 -- 172.21.15.28:0/2869970306 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a9401b8f0 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.898+0000 7f4a9d600700 1 -- 172.21.15.28:0/2869970306 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4a94023870 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.898+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4aa019f850 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.898+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4aa01a2540 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.899+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4aa0064ba0 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.900+0000 7f4a9d600700 1 -- 172.21.15.28:0/2869970306 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 12) ==== 50235+0+0 (secure 0 0 0) 0x7f4a9401b440 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.900+0000 7f4a9d600700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a8803e520 0x7f4a880409e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.901+0000 7f4a9d600700 1 -- 172.21.15.28:0/2869970306 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f4a94053fc0 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.901+0000 7f4a9fe00700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a8803e520 0x7f4a880409e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.902+0000 7f4a9fe00700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a8803e520 0x7f4a880409e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f4a8c006fd0 tx=0x7f4a8c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:07.906+0000 7f4a9d600700 1 -- 172.21.15.28:0/2869970306 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f4a9401f030 con 0x7f4aa0107880 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.165+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) -- 0x7f4aa002e6f0 con 0x7f4a8803e520 2024-05-20T19:53:09.068 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.426+0000 7f4a9d600700 1 -- 172.21.15.28:0/2869970306 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+252 (secure 0 0 0) 0x7f4aa002e6f0 con 0x7f4a8803e520 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.429+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a8803e520 msgr2=0x7f4a880409e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.429+0000 7f4aa7800700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a8803e520 0x7f4a880409e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f4a8c006fd0 tx=0x7f4a8c006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.429+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 msgr2=0x7f4aa019f080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.429+0000 7f4aa7800700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa019f080 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f4a94000c00 tx=0x7f4a94011aa0 comp rx=0 tx=0).stop 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.430+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 shutdown_connections 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.430+0000 7f4aa7800700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a8803e520 0x7f4a880409e0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.430+0000 7f4aa7800700 1 --2- 172.21.15.28:0/2869970306 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4aa0107880 0x7f4aa019f080 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.430+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 >> 172.21.15.28:0/2869970306 conn(0x7f4aa0103480 msgr2=0x7f4aa0195cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.430+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 shutdown_connections 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:08.430+0000 7f4aa7800700 1 -- 172.21.15.28:0/2869970306 wait complete. 2024-05-20T19:53:09.069 INFO:teuthology.orchestra.run.smithi028.stdout:Fetching dashboard port number... 2024-05-20T19:53:10.905 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stdout 8443 2024-05-20T19:53:10.905 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.914+0000 7f2e0e200700 1 Processor -- start 2024-05-20T19:53:10.905 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.914+0000 7f2e0e200700 1 -- start start 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.915+0000 7f2e0e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e08108270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.915+0000 7f2e0e200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2e081087b0 con 0x7f2e08107e50 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.915+0000 7f2e07400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e08108270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.915+0000 7f2e07400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e08108270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38780/0 (socket says 172.21.15.28:38780) 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.915+0000 7f2e07400700 1 -- 172.21.15.28:0/1443856440 learned_addr learned my addr 172.21.15.28:0/1443856440 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.916+0000 7f2e07400700 1 -- 172.21.15.28:0/1443856440 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e08109040 con 0x7f2e08107e50 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.916+0000 7f2e07400700 1 --2- 172.21.15.28:0/1443856440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e08108270 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f2df80098a0 tx=0x7f2df8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9c651b1af2585cc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.916+0000 7f2e06000700 1 -- 172.21.15.28:0/1443856440 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2df8004160 con 0x7f2e08107e50 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.917+0000 7f2e06000700 1 -- 172.21.15.28:0/1443856440 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2df8004300 con 0x7f2e08107e50 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.917+0000 7f2e06000700 1 -- 172.21.15.28:0/1443856440 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2df8011a20 con 0x7f2e08107e50 2024-05-20T19:53:10.906 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.917+0000 7f2e0e200700 1 -- 172.21.15.28:0/1443856440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 msgr2=0x7f2e08108270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.917+0000 7f2e0e200700 1 --2- 172.21.15.28:0/1443856440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e08108270 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f2df80098a0 tx=0x7f2df8009bb0 comp rx=0 tx=0).stop 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.918+0000 7f2e0e200700 1 -- 172.21.15.28:0/1443856440 shutdown_connections 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.918+0000 7f2e0e200700 1 --2- 172.21.15.28:0/1443856440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e08108270 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.918+0000 7f2e0e200700 1 -- 172.21.15.28:0/1443856440 >> 172.21.15.28:0/1443856440 conn(0x7f2e08103480 msgr2=0x7f2e081058a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.918+0000 7f2e0e200700 1 -- 172.21.15.28:0/1443856440 shutdown_connections 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.918+0000 7f2e0e200700 1 -- 172.21.15.28:0/1443856440 wait complete. 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.919+0000 7f2e0e200700 1 Processor -- start 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.919+0000 7f2e0e200700 1 -- start start 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.919+0000 7f2e0e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e0819f1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.919+0000 7f2e0e200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2e0810c300 con 0x7f2e08107e50 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.920+0000 7f2e07400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e0819f1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.920+0000 7f2e07400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e0819f1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38792/0 (socket says 172.21.15.28:38792) 2024-05-20T19:53:10.907 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.920+0000 7f2e07400700 1 -- 172.21.15.28:0/1647250750 learned_addr learned my addr 172.21.15.28:0/1647250750 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:10.908 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.920+0000 7f2e07400700 1 -- 172.21.15.28:0/1647250750 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e0819f6f0 con 0x7f2e08107e50 2024-05-20T19:53:10.908 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.920+0000 7f2e07400700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e0819f1b0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f2df8000c00 tx=0x7f2df8011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:10.908 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.921+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2df8003aa0 con 0x7f2e08107e50 2024-05-20T19:53:10.908 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.921+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2df801b8f0 con 0x7f2e08107e50 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.921+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f2df8023890 con 0x7f2e08107e50 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.921+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e0819f980 con 0x7f2e08107e50 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.921+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e081a2670 con 0x7f2e08107e50 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.922+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2e08064ba0 con 0x7f2e08107e50 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.922+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 12) ==== 50235+0+0 (secure 0 0 0) 0x7f2df801b440 con 0x7f2e08107e50 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.923+0000 7f2dffe00700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2dec03e520 0x7f2dec0409e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.923+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f2df8053fb0 con 0x7f2e08107e50 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.923+0000 7f2e06a00700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2dec03e520 0x7f2dec0409e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.924+0000 7f2e06a00700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2dec03e520 0x7f2dec0409e0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f2df4006fd0 tx=0x7f2df4006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:10.911 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.929+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f2df801f030 con 0x7f2e08107e50 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:09.965+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f2df800b3e0 con 0x7f2e08107e50 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.178+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) -- 0x7f2e0802e730 con 0x7f2e08107e50 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.180+0000 7f2dffe00700 1 -- 172.21.15.28:0/1647250750 <== mon.0 v2:172.21.15.28:3300/0 8 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v8) ==== 112+0+5 (secure 0 0 0) 0x7f2df8010380 con 0x7f2e08107e50 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2dec03e520 msgr2=0x7f2dec0409e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2dec03e520 0x7f2dec0409e0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f2df4006fd0 tx=0x7f2df4006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 msgr2=0x7f2e0819f1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e0819f1b0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f2df8000c00 tx=0x7f2df8011aa0 comp rx=0 tx=0).stop 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 shutdown_connections 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2dec03e520 0x7f2dec0409e0 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:10.912 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 --2- 172.21.15.28:0/1647250750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e08107e50 0x7f2e0819f1b0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:10.913 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 >> 172.21.15.28:0/1647250750 conn(0x7f2e08103480 msgr2=0x7f2e08104320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:10.913 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 shutdown_connections 2024-05-20T19:53:10.913 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:10.183+0000 7f2e0e200700 1 -- 172.21.15.28:0/1647250750 wait complete. 2024-05-20T19:53:10.934 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-05-20T19:53:10.934 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout disabled 2024-05-20T19:53:10.955 INFO:teuthology.orchestra.run.smithi028.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-05-20T19:53:10.955 INFO:teuthology.orchestra.run.smithi028.stdout:systemctl: stdout inactive 2024-05-20T19:53:10.955 INFO:teuthology.orchestra.run.smithi028.stdout:firewalld.service is not enabled 2024-05-20T19:53:10.955 INFO:teuthology.orchestra.run.smithi028.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-20T19:53:10.958 INFO:teuthology.orchestra.run.smithi028.stdout:Ceph Dashboard is now available at: 2024-05-20T19:53:10.958 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:10.958 INFO:teuthology.orchestra.run.smithi028.stdout: URL: https://smithi028.front.sepia.ceph.com:8443/ 2024-05-20T19:53:10.958 INFO:teuthology.orchestra.run.smithi028.stdout: User: admin 2024-05-20T19:53:10.958 INFO:teuthology.orchestra.run.smithi028.stdout: Password: y00imqgbor 2024-05-20T19:53:10.959 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:10.960 INFO:teuthology.orchestra.run.smithi028.stdout:Saving cluster configuration to /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config directory 2024-05-20T19:53:10.960 INFO:teuthology.orchestra.run.smithi028.stdout:Enabling autotune for osd_memory_target 2024-05-20T19:53:11.197 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:10 smithi028 ceph-mon[31178]: mgrmap e13: x(active, since 6s) 2024-05-20T19:53:11.197 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:10 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1647250750' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-20T19:53:12.876 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.028+0000 7faa5b400700 1 Processor -- start 2024-05-20T19:53:12.876 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.028+0000 7faa5b400700 1 -- start start 2024-05-20T19:53:12.876 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.028+0000 7faa5b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa54109450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:12.877 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.028+0000 7faa5b400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7faa54109a20 con 0x7faa54109030 2024-05-20T19:53:12.877 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.029+0000 7faa58a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa54109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:12.877 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.029+0000 7faa58a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa54109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38800/0 (socket says 172.21.15.28:38800) 2024-05-20T19:53:12.877 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.029+0000 7faa58a00700 1 -- 172.21.15.28:0/3158976015 learned_addr learned my addr 172.21.15.28:0/3158976015 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:12.877 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.029+0000 7faa58a00700 1 -- 172.21.15.28:0/3158976015 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faa5410a1e0 con 0x7faa54109030 2024-05-20T19:53:12.877 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.030+0000 7faa58a00700 1 --2- 172.21.15.28:0/3158976015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa54109450 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7faa440098a0 tx=0x7faa44009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d647d6203359f32f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:12.878 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.030+0000 7faa53e00700 1 -- 172.21.15.28:0/3158976015 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7faa44004160 con 0x7faa54109030 2024-05-20T19:53:12.878 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.030+0000 7faa53e00700 1 -- 172.21.15.28:0/3158976015 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7faa44004300 con 0x7faa54109030 2024-05-20T19:53:12.878 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.030+0000 7faa53e00700 1 -- 172.21.15.28:0/3158976015 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7faa44011a20 con 0x7faa54109030 2024-05-20T19:53:12.878 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.031+0000 7faa5b400700 1 -- 172.21.15.28:0/3158976015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 msgr2=0x7faa54109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:12.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.031+0000 7faa5b400700 1 --2- 172.21.15.28:0/3158976015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa54109450 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7faa440098a0 tx=0x7faa44009bb0 comp rx=0 tx=0).stop 2024-05-20T19:53:12.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.031+0000 7faa5b400700 1 -- 172.21.15.28:0/3158976015 shutdown_connections 2024-05-20T19:53:12.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.031+0000 7faa5b400700 1 --2- 172.21.15.28:0/3158976015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa54109450 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:12.880 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.031+0000 7faa5b400700 1 -- 172.21.15.28:0/3158976015 >> 172.21.15.28:0/3158976015 conn(0x7faa54104610 msgr2=0x7faa54106a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.031+0000 7faa5b400700 1 -- 172.21.15.28:0/3158976015 shutdown_connections 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.031+0000 7faa5b400700 1 -- 172.21.15.28:0/3158976015 wait complete. 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.032+0000 7faa5b400700 1 Processor -- start 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.032+0000 7faa5b400700 1 -- start start 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.033+0000 7faa5b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa5419f160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.033+0000 7faa5b400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7faa5410d590 con 0x7faa54109030 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.033+0000 7faa58a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa5419f160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.033+0000 7faa58a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa5419f160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38802/0 (socket says 172.21.15.28:38802) 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.033+0000 7faa58a00700 1 -- 172.21.15.28:0/786515493 learned_addr learned my addr 172.21.15.28:0/786515493 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.033+0000 7faa58a00700 1 -- 172.21.15.28:0/786515493 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faa5419f6a0 con 0x7faa54109030 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.034+0000 7faa58a00700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa5419f160 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7faa44000c00 tx=0x7faa44011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.034+0000 7faa52000700 1 -- 172.21.15.28:0/786515493 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7faa44003aa0 con 0x7faa54109030 2024-05-20T19:53:12.881 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.034+0000 7faa52000700 1 -- 172.21.15.28:0/786515493 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7faa4401b8f0 con 0x7faa54109030 2024-05-20T19:53:12.882 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.034+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faa5419f930 con 0x7faa54109030 2024-05-20T19:53:12.882 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.034+0000 7faa52000700 1 -- 172.21.15.28:0/786515493 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7faa44023890 con 0x7faa54109030 2024-05-20T19:53:12.882 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.034+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faa541a2620 con 0x7faa54109030 2024-05-20T19:53:12.882 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.035+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faa54064ba0 con 0x7faa54109030 2024-05-20T19:53:12.882 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.036+0000 7faa52000700 1 -- 172.21.15.28:0/786515493 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7faa4401b440 con 0x7faa54109030 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.036+0000 7faa52000700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7faa3403e560 0x7faa34040a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.036+0000 7faa52000700 1 -- 172.21.15.28:0/786515493 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7faa440540f0 con 0x7faa54109030 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.037+0000 7faa4be00700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7faa3403e560 0x7faa34040a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.037+0000 7faa4be00700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7faa3403e560 0x7faa34040a20 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7faa3c006fd0 tx=0x7faa3c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.042+0000 7faa52000700 1 -- 172.21.15.28:0/786515493 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7faa4401f030 con 0x7faa54109030 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.279+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command([{prefix=config set, name=osd_memory_target_autotune}] v 0) -- 0x7faa541a27f0 con 0x7faa54109030 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.280+0000 7faa52000700 1 -- 172.21.15.28:0/786515493 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_memory_target_autotune}]=0 v8)=0 v8) ==== 127+0+0 (secure 0 0 0) 0x7faa44017e90 con 0x7faa54109030 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.283+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7faa3403e560 msgr2=0x7faa34040a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:12.884 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.283+0000 7faa5b400700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7faa3403e560 0x7faa34040a20 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7faa3c006fd0 tx=0x7faa3c006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.283+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 msgr2=0x7faa5419f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.283+0000 7faa5b400700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa5419f160 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7faa44000c00 tx=0x7faa44011aa0 comp rx=0 tx=0).stop 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.284+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 shutdown_connections 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.284+0000 7faa5b400700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7faa3403e560 0x7faa34040a20 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.284+0000 7faa5b400700 1 --2- 172.21.15.28:0/786515493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa54109030 0x7faa5419f160 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.284+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 >> 172.21.15.28:0/786515493 conn(0x7faa54104610 msgr2=0x7faa54105070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.284+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 shutdown_connections 2024-05-20T19:53:12.885 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:12.284+0000 7faa5b400700 1 -- 172.21.15.28:0/786515493 wait complete. 2024-05-20T19:53:14.508 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.501+0000 7f6d6b400700 1 Processor -- start 2024-05-20T19:53:14.508 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.501+0000 7f6d6b400700 1 -- start start 2024-05-20T19:53:14.508 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.502+0000 7f6d6b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d64109430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:14.508 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.502+0000 7f6d6b400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6d64109a00 con 0x7f6d64109010 2024-05-20T19:53:14.508 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.502+0000 7f6d68a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d64109430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:14.508 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.502+0000 7f6d68a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d64109430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38814/0 (socket says 172.21.15.28:38814) 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.502+0000 7f6d68a00700 1 -- 172.21.15.28:0/1264942911 learned_addr learned my addr 172.21.15.28:0/1264942911 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.502+0000 7f6d68a00700 1 -- 172.21.15.28:0/1264942911 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6d6410a210 con 0x7f6d64109010 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.503+0000 7f6d68a00700 1 --2- 172.21.15.28:0/1264942911 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d64109430 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f6d580098a0 tx=0x7f6d58009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7af2bc16406ba85d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.503+0000 7f6d63400700 1 -- 172.21.15.28:0/1264942911 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d58004160 con 0x7f6d64109010 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.503+0000 7f6d63400700 1 -- 172.21.15.28:0/1264942911 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6d58004300 con 0x7f6d64109010 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.503+0000 7f6d63400700 1 -- 172.21.15.28:0/1264942911 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d58011a20 con 0x7f6d64109010 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.504+0000 7f6d6b400700 1 -- 172.21.15.28:0/1264942911 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 msgr2=0x7f6d64109430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.504+0000 7f6d6b400700 1 --2- 172.21.15.28:0/1264942911 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d64109430 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f6d580098a0 tx=0x7f6d58009bb0 comp rx=0 tx=0).stop 2024-05-20T19:53:14.509 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.504+0000 7f6d6b400700 1 -- 172.21.15.28:0/1264942911 shutdown_connections 2024-05-20T19:53:14.514 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.504+0000 7f6d6b400700 1 --2- 172.21.15.28:0/1264942911 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d64109430 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:14.514 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.504+0000 7f6d6b400700 1 -- 172.21.15.28:0/1264942911 >> 172.21.15.28:0/1264942911 conn(0x7f6d64104590 msgr2=0x7f6d641069f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:14.514 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.504+0000 7f6d6b400700 1 -- 172.21.15.28:0/1264942911 shutdown_connections 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.505+0000 7f6d6b400700 1 -- 172.21.15.28:0/1264942911 wait complete. 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.506+0000 7f6d6b400700 1 Processor -- start 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.506+0000 7f6d6b400700 1 -- start start 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.506+0000 7f6d6b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d6407f3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.506+0000 7f6d6b400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6d6410d5c0 con 0x7f6d64109010 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.507+0000 7f6d68a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d6407f3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.507+0000 7f6d68a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d6407f3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38830/0 (socket says 172.21.15.28:38830) 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.507+0000 7f6d68a00700 1 -- 172.21.15.28:0/2820220689 learned_addr learned my addr 172.21.15.28:0/2820220689 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.507+0000 7f6d68a00700 1 -- 172.21.15.28:0/2820220689 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6d6407f910 con 0x7f6d64109010 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.508+0000 7f6d68a00700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d6407f3d0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f6d580119f0 tx=0x7f6d58011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:14.515 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.508+0000 7f6d61600700 1 -- 172.21.15.28:0/2820220689 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d580039e0 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.508+0000 7f6d61600700 1 -- 172.21.15.28:0/2820220689 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6d5801b8f0 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.508+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6d6407db00 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.508+0000 7f6d61600700 1 -- 172.21.15.28:0/2820220689 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6d580237a0 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.508+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6d6407e040 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.509+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6d64064ba0 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.511+0000 7f6d61600700 1 -- 172.21.15.28:0/2820220689 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f6d5801b440 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.511+0000 7f6d61600700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6d4c03e570 0x7f6d4c040a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.511+0000 7f6d61600700 1 -- 172.21.15.28:0/2820220689 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f6d580547d0 con 0x7f6d64109010 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.512+0000 7f6d63e00700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6d4c03e570 0x7f6d4c040a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.512+0000 7f6d63e00700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6d4c03e570 0x7f6d4c040a30 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f6d50006fd0 tx=0x7f6d50006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:14.516 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.516+0000 7f6d61600700 1 -- 172.21.15.28:0/2820220689 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f6d5801f030 con 0x7f6d64109010 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.877+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command([{prefix=config-key set, key=mgr/dashboard/cluster/status}] v 0) -- 0x7f6d6407e6b0 con 0x7f6d64109010 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.880+0000 7f6d61600700 1 -- 172.21.15.28:0/2820220689 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{prefix=config-key set, key=mgr/dashboard/cluster/status}]=0 set mgr/dashboard/cluster/status v28)=0 set mgr/dashboard/cluster/status v28) ==== 153+0+0 (secure 0 0 0) 0x7f6d580292a0 con 0x7f6d64109010 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6d4c03e570 msgr2=0x7f6d4c040a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6d4c03e570 0x7f6d4c040a30 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f6d50006fd0 tx=0x7f6d50006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 msgr2=0x7f6d6407f3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d6407f3d0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f6d580119f0 tx=0x7f6d58011ad0 comp rx=0 tx=0).stop 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 shutdown_connections 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6d4c03e570 0x7f6d4c040a30 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 --2- 172.21.15.28:0/2820220689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6d64109010 0x7f6d6407f3d0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 >> 172.21.15.28:0/2820220689 conn(0x7f6d64104590 msgr2=0x7f6d641050a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.883+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 shutdown_connections 2024-05-20T19:53:14.517 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr 2024-05-20T19:53:13.884+0000 7f6d6b400700 1 -- 172.21.15.28:0/2820220689 wait complete. 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout:Or, if you are only running a single cluster on this host: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout:Please consider enabling telemetry to help improve Ceph: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.518 INFO:teuthology.orchestra.run.smithi028.stdout: ceph telemetry on 2024-05-20T19:53:14.519 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.519 INFO:teuthology.orchestra.run.smithi028.stdout:For more information see: 2024-05-20T19:53:14.519 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.519 INFO:teuthology.orchestra.run.smithi028.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-05-20T19:53:14.519 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:14.519 INFO:teuthology.orchestra.run.smithi028.stdout:Bootstrap complete. 2024-05-20T19:53:14.626 INFO:tasks.cephadm:Fetching config... 2024-05-20T19:53:14.627 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:53:14.627 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-20T19:53:14.685 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-20T19:53:14.686 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:53:14.686 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-20T19:53:14.729 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-20T19:53:14.729 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:53:14.729 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/keyring of=/dev/stdout 2024-05-20T19:53:14.845 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-20T19:53:14.845 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:53:14.846 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-20T19:53:14.943 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-20T19:53:14.943 DEBUG:teuthology.orchestra.run.smithi028:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ6OKcFk30zzzgO8qZZ+GR+Ud+IVR4ZAvakRYwOGSfY62Gw+VP4VoR1B2vVhPW1Fvx1Ez/UFEEIc/WTwzc1K+kTOUrpRzEOolfo2eyGDIBi/+eS0yzscZpWajV1Fz8/tJO3+8EFkHE4ld2ibTcCHwedS762vVEmAj2lMxHw+Aau4Xb6qqbso/QKoYPJ1bnvi3agHYpPcfdq9pGS4sHarYZ+SfRPlE2MHYwklqVTcvx20UCDDiMtNZjYBp2aXHkRdZV57CG8Pj+B8DolUkfC4vg1l0g3/z16ps7FY6qAoqWdoaOKCUX3C08hRKRvHVcnQmPPRppp9fO/68iicPfXZGHRMMNOgTrVTjWVOhsTZKaRn86SrOGrvILK/G5fYOeGyLasXx8KatxjFbdViwtVVacu7OHlQr3mn4QakgOh2F1JpvzL7VPeORLr4JAc92FYDFm3FOv203D8Xxgu5eZD6fXmNSP0W38u+fGbAe2sPyY7DB/7Wa37nPIuuCebeNTvck= ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-20T19:53:15.051 INFO:teuthology.orchestra.run.smithi028.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ6OKcFk30zzzgO8qZZ+GR+Ud+IVR4ZAvakRYwOGSfY62Gw+VP4VoR1B2vVhPW1Fvx1Ez/UFEEIc/WTwzc1K+kTOUrpRzEOolfo2eyGDIBi/+eS0yzscZpWajV1Fz8/tJO3+8EFkHE4ld2ibTcCHwedS762vVEmAj2lMxHw+Aau4Xb6qqbso/QKoYPJ1bnvi3agHYpPcfdq9pGS4sHarYZ+SfRPlE2MHYwklqVTcvx20UCDDiMtNZjYBp2aXHkRdZV57CG8Pj+B8DolUkfC4vg1l0g3/z16ps7FY6qAoqWdoaOKCUX3C08hRKRvHVcnQmPPRppp9fO/68iicPfXZGHRMMNOgTrVTjWVOhsTZKaRn86SrOGrvILK/G5fYOeGyLasXx8KatxjFbdViwtVVacu7OHlQr3mn4QakgOh2F1JpvzL7VPeORLr4JAc92FYDFm3FOv203D8Xxgu5eZD6fXmNSP0W38u+fGbAe2sPyY7DB/7Wa37nPIuuCebeNTvck= ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:53:15.087 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:14 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2820220689' entity='client.admin' 2024-05-20T19:53:15.102 DEBUG:teuthology.orchestra.run.smithi033:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ6OKcFk30zzzgO8qZZ+GR+Ud+IVR4ZAvakRYwOGSfY62Gw+VP4VoR1B2vVhPW1Fvx1Ez/UFEEIc/WTwzc1K+kTOUrpRzEOolfo2eyGDIBi/+eS0yzscZpWajV1Fz8/tJO3+8EFkHE4ld2ibTcCHwedS762vVEmAj2lMxHw+Aau4Xb6qqbso/QKoYPJ1bnvi3agHYpPcfdq9pGS4sHarYZ+SfRPlE2MHYwklqVTcvx20UCDDiMtNZjYBp2aXHkRdZV57CG8Pj+B8DolUkfC4vg1l0g3/z16ps7FY6qAoqWdoaOKCUX3C08hRKRvHVcnQmPPRppp9fO/68iicPfXZGHRMMNOgTrVTjWVOhsTZKaRn86SrOGrvILK/G5fYOeGyLasXx8KatxjFbdViwtVVacu7OHlQr3mn4QakgOh2F1JpvzL7VPeORLr4JAc92FYDFm3FOv203D8Xxgu5eZD6fXmNSP0W38u+fGbAe2sPyY7DB/7Wa37nPIuuCebeNTvck= ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-20T19:53:15.145 INFO:teuthology.orchestra.run.smithi033.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ6OKcFk30zzzgO8qZZ+GR+Ud+IVR4ZAvakRYwOGSfY62Gw+VP4VoR1B2vVhPW1Fvx1Ez/UFEEIc/WTwzc1K+kTOUrpRzEOolfo2eyGDIBi/+eS0yzscZpWajV1Fz8/tJO3+8EFkHE4ld2ibTcCHwedS762vVEmAj2lMxHw+Aau4Xb6qqbso/QKoYPJ1bnvi3agHYpPcfdq9pGS4sHarYZ+SfRPlE2MHYwklqVTcvx20UCDDiMtNZjYBp2aXHkRdZV57CG8Pj+B8DolUkfC4vg1l0g3/z16ps7FY6qAoqWdoaOKCUX3C08hRKRvHVcnQmPPRppp9fO/68iicPfXZGHRMMNOgTrVTjWVOhsTZKaRn86SrOGrvILK/G5fYOeGyLasXx8KatxjFbdViwtVVacu7OHlQr3mn4QakgOh2F1JpvzL7VPeORLr4JAc92FYDFm3FOv203D8Xxgu5eZD6fXmNSP0W38u+fGbAe2sPyY7DB/7Wa37nPIuuCebeNTvck= ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:53:15.157 DEBUG:teuthology.orchestra.run.smithi154:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ6OKcFk30zzzgO8qZZ+GR+Ud+IVR4ZAvakRYwOGSfY62Gw+VP4VoR1B2vVhPW1Fvx1Ez/UFEEIc/WTwzc1K+kTOUrpRzEOolfo2eyGDIBi/+eS0yzscZpWajV1Fz8/tJO3+8EFkHE4ld2ibTcCHwedS762vVEmAj2lMxHw+Aau4Xb6qqbso/QKoYPJ1bnvi3agHYpPcfdq9pGS4sHarYZ+SfRPlE2MHYwklqVTcvx20UCDDiMtNZjYBp2aXHkRdZV57CG8Pj+B8DolUkfC4vg1l0g3/z16ps7FY6qAoqWdoaOKCUX3C08hRKRvHVcnQmPPRppp9fO/68iicPfXZGHRMMNOgTrVTjWVOhsTZKaRn86SrOGrvILK/G5fYOeGyLasXx8KatxjFbdViwtVVacu7OHlQr3mn4QakgOh2F1JpvzL7VPeORLr4JAc92FYDFm3FOv203D8Xxgu5eZD6fXmNSP0W38u+fGbAe2sPyY7DB/7Wa37nPIuuCebeNTvck= ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-20T19:53:15.199 INFO:teuthology.orchestra.run.smithi154.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ6OKcFk30zzzgO8qZZ+GR+Ud+IVR4ZAvakRYwOGSfY62Gw+VP4VoR1B2vVhPW1Fvx1Ez/UFEEIc/WTwzc1K+kTOUrpRzEOolfo2eyGDIBi/+eS0yzscZpWajV1Fz8/tJO3+8EFkHE4ld2ibTcCHwedS762vVEmAj2lMxHw+Aau4Xb6qqbso/QKoYPJ1bnvi3agHYpPcfdq9pGS4sHarYZ+SfRPlE2MHYwklqVTcvx20UCDDiMtNZjYBp2aXHkRdZV57CG8Pj+B8DolUkfC4vg1l0g3/z16ps7FY6qAoqWdoaOKCUX3C08hRKRvHVcnQmPPRppp9fO/68iicPfXZGHRMMNOgTrVTjWVOhsTZKaRn86SrOGrvILK/G5fYOeGyLasXx8KatxjFbdViwtVVacu7OHlQr3mn4QakgOh2F1JpvzL7VPeORLr4JAc92FYDFm3FOv203D8Xxgu5eZD6fXmNSP0W38u+fGbAe2sPyY7DB/7Wa37nPIuuCebeNTvck= ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:53:15.210 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-20T19:53:15.653 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:53:18.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.327+0000 7f6c4d800700 1 -- 172.21.15.28:0/1334388672 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 msgr2=0x7f6c48109b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.327+0000 7f6c4d800700 1 --2- 172.21.15.28:0/1334388672 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c48109b70 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f6c38009910 tx=0x7f6c38009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.327+0000 7f6c4d800700 1 -- 172.21.15.28:0/1334388672 shutdown_connections 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.327+0000 7f6c4d800700 1 --2- 172.21.15.28:0/1334388672 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c48109b70 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.327+0000 7f6c4d800700 1 -- 172.21.15.28:0/1334388672 >> 172.21.15.28:0/1334388672 conn(0x7f6c48078630 msgr2=0x7f6c4807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.327+0000 7f6c4d800700 1 -- 172.21.15.28:0/1334388672 shutdown_connections 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.327+0000 7f6c4d800700 1 -- 172.21.15.28:0/1334388672 wait complete. 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.328+0000 7f6c4d800700 1 Processor -- start 2024-05-20T19:53:18.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.328+0000 7f6c4d800700 1 -- start start 2024-05-20T19:53:18.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.329+0000 7f6c4d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c4819ec30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:18.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.329+0000 7f6c4d800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6c4810f520 con 0x7f6c48109750 2024-05-20T19:53:18.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.329+0000 7f6c46a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c4819ec30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:18.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.329+0000 7f6c46a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c4819ec30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:55902/0 (socket says 172.21.15.28:55902) 2024-05-20T19:53:18.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.329+0000 7f6c46a00700 1 -- 172.21.15.28:0/751616555 learned_addr learned my addr 172.21.15.28:0/751616555 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:18.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.330+0000 7f6c46a00700 1 -- 172.21.15.28:0/751616555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6c4819f170 con 0x7f6c48109750 2024-05-20T19:53:18.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.330+0000 7f6c46a00700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c4819ec30 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f6c38005950 tx=0x7f6c38004fa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:18.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.330+0000 7f6c3f400700 1 -- 172.21.15.28:0/751616555 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6c38026070 con 0x7f6c48109750 2024-05-20T19:53:18.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.330+0000 7f6c3f400700 1 -- 172.21.15.28:0/751616555 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6c3800bc00 con 0x7f6c48109750 2024-05-20T19:53:18.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.330+0000 7f6c3f400700 1 -- 172.21.15.28:0/751616555 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f6c38003a10 con 0x7f6c48109750 2024-05-20T19:53:18.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.330+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6c4819f400 con 0x7f6c48109750 2024-05-20T19:53:18.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.330+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6c481a20a0 con 0x7f6c48109750 2024-05-20T19:53:18.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.331+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6c4810fca0 con 0x7f6c48109750 2024-05-20T19:53:18.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.332+0000 7f6c3f400700 1 -- 172.21.15.28:0/751616555 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f6c38029070 con 0x7f6c48109750 2024-05-20T19:53:18.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.332+0000 7f6c3f400700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6c2c03e570 0x7f6c2c040a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:18.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.332+0000 7f6c3f400700 1 -- 172.21.15.28:0/751616555 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f6c3801e070 con 0x7f6c48109750 2024-05-20T19:53:18.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.333+0000 7f6c46000700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6c2c03e570 0x7f6c2c040a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:18.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.333+0000 7f6c46000700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6c2c03e570 0x7f6c2c040a30 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f6c34006fd0 tx=0x7f6c34006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:18.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.338+0000 7f6c3f400700 1 -- 172.21.15.28:0/751616555 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f6c38013070 con 0x7f6c48109750 2024-05-20T19:53:18.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.585+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) -- 0x7f6c4802e670 con 0x7f6c48109750 2024-05-20T19:53:18.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.588+0000 7f6c3f400700 1 -- 172.21.15.28:0/751616555 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v9)=0 v9) ==== 125+0+0 (secure 0 0 0) 0x7f6c380300c0 con 0x7f6c48109750 2024-05-20T19:53:18.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6c2c03e570 msgr2=0x7f6c2c040a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:18.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6c2c03e570 0x7f6c2c040a30 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f6c34006fd0 tx=0x7f6c34006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:18.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 msgr2=0x7f6c4819ec30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:18.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c4819ec30 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f6c38005950 tx=0x7f6c38004fa0 comp rx=0 tx=0).stop 2024-05-20T19:53:18.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 shutdown_connections 2024-05-20T19:53:18.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6c2c03e570 0x7f6c2c040a30 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:18.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 --2- 172.21.15.28:0/751616555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6c48109750 0x7f6c4819ec30 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:18.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.593+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 >> 172.21.15.28:0/751616555 conn(0x7f6c48078630 msgr2=0x7f6c48079160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:18.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.594+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 shutdown_connections 2024-05-20T19:53:18.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:18.594+0000 7f6c4d800700 1 -- 172.21.15.28:0/751616555 wait complete. 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi028", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/751616555' entity='client.admin' 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:19.145 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:19.566 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-20T19:53:19.566 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-20T19:53:20.025 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:53:22.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.731+0000 7f4de6c00700 1 -- 172.21.15.28:0/21082142 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 msgr2=0x7f4de0105440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:22.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.731+0000 7f4de6c00700 1 --2- 172.21.15.28:0/21082142 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de0105440 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f4dd4009910 tx=0x7f4dd4009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:22.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.731+0000 7f4de6c00700 1 -- 172.21.15.28:0/21082142 shutdown_connections 2024-05-20T19:53:22.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.731+0000 7f4de6c00700 1 --2- 172.21.15.28:0/21082142 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de0105440 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:22.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.731+0000 7f4de6c00700 1 -- 172.21.15.28:0/21082142 >> 172.21.15.28:0/21082142 conn(0x7f4de01005a0 msgr2=0x7f4de0102a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:22.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.731+0000 7f4de6c00700 1 -- 172.21.15.28:0/21082142 shutdown_connections 2024-05-20T19:53:22.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.732+0000 7f4de6c00700 1 -- 172.21.15.28:0/21082142 wait complete. 2024-05-20T19:53:22.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.733+0000 7f4de6c00700 1 Processor -- start 2024-05-20T19:53:22.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.733+0000 7f4de6c00700 1 -- start start 2024-05-20T19:53:22.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.734+0000 7f4de6c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de019a630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:22.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.734+0000 7f4de6c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4de010af60 con 0x7f4de0105020 2024-05-20T19:53:22.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.734+0000 7f4de5800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de019a630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:22.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.734+0000 7f4de5800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de019a630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:55912/0 (socket says 172.21.15.28:55912) 2024-05-20T19:53:22.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.734+0000 7f4de5800700 1 -- 172.21.15.28:0/257691453 learned_addr learned my addr 172.21.15.28:0/257691453 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:22.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.734+0000 7f4de5800700 1 -- 172.21.15.28:0/257691453 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4de019ab70 con 0x7f4de0105020 2024-05-20T19:53:22.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.735+0000 7f4de5800700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de019a630 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f4dd4005950 tx=0x7f4dd4004fa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:22.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.735+0000 7f4dde000700 1 -- 172.21.15.28:0/257691453 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4dd4026070 con 0x7f4de0105020 2024-05-20T19:53:22.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.735+0000 7f4dde000700 1 -- 172.21.15.28:0/257691453 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4dd400bc00 con 0x7f4de0105020 2024-05-20T19:53:22.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.735+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4de019ae00 con 0x7f4de0105020 2024-05-20T19:53:22.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.735+0000 7f4dde000700 1 -- 172.21.15.28:0/257691453 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f4dd4003a10 con 0x7f4de0105020 2024-05-20T19:53:22.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.735+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4de019daf0 con 0x7f4de0105020 2024-05-20T19:53:22.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.736+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4de010b6e0 con 0x7f4de0105020 2024-05-20T19:53:22.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.736+0000 7f4dde000700 1 -- 172.21.15.28:0/257691453 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f4dd4029070 con 0x7f4de0105020 2024-05-20T19:53:22.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.737+0000 7f4dde000700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4dc803e570 0x7f4dc8040a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:22.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.737+0000 7f4dde000700 1 -- 172.21.15.28:0/257691453 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f4dd401e070 con 0x7f4de0105020 2024-05-20T19:53:22.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.737+0000 7f4de4e00700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4dc803e570 0x7f4dc8040a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:22.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.738+0000 7f4de4e00700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4dc803e570 0x7f4dc8040a30 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc006fd0 tx=0x7f4dcc006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:22.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.743+0000 7f4dde000700 1 -- 172.21.15.28:0/257691453 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f4dd4013070 con 0x7f4de0105020 2024-05-20T19:53:22.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.988+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}) -- 0x7f4de002e670 con 0x7f4dc803e570 2024-05-20T19:53:22.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.994+0000 7f4dde000700 1 -- 172.21.15.28:0/257691453 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f4de002e670 con 0x7f4dc803e570 2024-05-20T19:53:22.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4dc803e570 msgr2=0x7f4dc8040a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:22.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4dc803e570 0x7f4dc8040a30 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc006fd0 tx=0x7f4dcc006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 msgr2=0x7f4de019a630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de019a630 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f4dd4005950 tx=0x7f4dd4004fa0 comp rx=0 tx=0).stop 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 shutdown_connections 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4dc803e570 0x7f4dc8040a30 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 --2- 172.21.15.28:0/257691453 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4de0105020 0x7f4de019a630 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 >> 172.21.15.28:0/257691453 conn(0x7f4de01005a0 msgr2=0x7f4de0101150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 shutdown_connections 2024-05-20T19:53:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:22.998+0000 7f4de6c00700 1 -- 172.21.15.28:0/257691453 wait complete. 2024-05-20T19:53:23.747 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi033 2024-05-20T19:53:23.747 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:53:23.747 DEBUG:teuthology.orchestra.run.smithi033:> dd of=/etc/ceph/ceph.conf 2024-05-20T19:53:23.766 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:53:23.766 DEBUG:teuthology.orchestra.run.smithi033:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:23.824 INFO:tasks.cephadm:Adding host smithi033 to orchestrator... 2024-05-20T19:53:23.824 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch host add smithi033 2024-05-20T19:53:24.243 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:53:24.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:23 smithi028 ceph-mon[31178]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:24.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:24.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:24.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:24.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:53:24.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:23 smithi028 ceph-mon[31178]: Updating smithi028:/etc/ceph/ceph.conf 2024-05-20T19:53:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:23 smithi028 ceph-mon[31178]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:25.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:24 smithi028 ceph-mon[31178]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:53:25.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:24 smithi028 ceph-mon[31178]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:26.048 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:26 smithi028 ceph-mon[31178]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T19:53:26.048 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:26 smithi028 ceph-mon[31178]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:26.049 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:26.049 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:26.049 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:26.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.222+0000 7fe2c8e00700 1 -- 172.21.15.28:0/206590445 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 msgr2=0x7fe2c41055e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:26.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.222+0000 7fe2c8e00700 1 --2- 172.21.15.28:0/206590445 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41055e0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fe2b400b0a0 tx=0x7fe2b400b3b0 comp rx=0 tx=0).stop 2024-05-20T19:53:26.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.222+0000 7fe2c8e00700 1 -- 172.21.15.28:0/206590445 shutdown_connections 2024-05-20T19:53:26.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.222+0000 7fe2c8e00700 1 --2- 172.21.15.28:0/206590445 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41055e0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:26.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.222+0000 7fe2c8e00700 1 -- 172.21.15.28:0/206590445 >> 172.21.15.28:0/206590445 conn(0x7fe2c4100a70 msgr2=0x7fe2c4102e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:26.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.222+0000 7fe2c8e00700 1 -- 172.21.15.28:0/206590445 shutdown_connections 2024-05-20T19:53:26.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.222+0000 7fe2c8e00700 1 -- 172.21.15.28:0/206590445 wait complete. 2024-05-20T19:53:26.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.224+0000 7fe2c8e00700 1 Processor -- start 2024-05-20T19:53:26.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.224+0000 7fe2c8e00700 1 -- start start 2024-05-20T19:53:26.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.225+0000 7fe2c8e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41a0640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:26.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.225+0000 7fe2c8e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe2c410f480 con 0x7fe2c4105200 2024-05-20T19:53:26.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.225+0000 7fe2c3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41a0640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:26.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.225+0000 7fe2c3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41a0640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:55932/0 (socket says 172.21.15.28:55932) 2024-05-20T19:53:26.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.225+0000 7fe2c3400700 1 -- 172.21.15.28:0/2101556881 learned_addr learned my addr 172.21.15.28:0/2101556881 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:26.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.226+0000 7fe2c3400700 1 -- 172.21.15.28:0/2101556881 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe2c41a0b80 con 0x7fe2c4105200 2024-05-20T19:53:26.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.226+0000 7fe2c3400700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41a0640 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7fe2b4006130 tx=0x7fe2b40097d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:26.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.226+0000 7fe2c0c00700 1 -- 172.21.15.28:0/2101556881 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe2b4021020 con 0x7fe2c4105200 2024-05-20T19:53:26.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.226+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe2c41a0e10 con 0x7fe2c4105200 2024-05-20T19:53:26.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.226+0000 7fe2c0c00700 1 -- 172.21.15.28:0/2101556881 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe2b40079b0 con 0x7fe2c4105200 2024-05-20T19:53:26.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.226+0000 7fe2c0c00700 1 -- 172.21.15.28:0/2101556881 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe2b4017040 con 0x7fe2c4105200 2024-05-20T19:53:26.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.226+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe2c419a7c0 con 0x7fe2c4105200 2024-05-20T19:53:26.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.228+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe2c41066e0 con 0x7fe2c4105200 2024-05-20T19:53:26.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.228+0000 7fe2c0c00700 1 -- 172.21.15.28:0/2101556881 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7fe2b4024020 con 0x7fe2c4105200 2024-05-20T19:53:26.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.229+0000 7fe2c0c00700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe2a803e510 0x7fe2a80409d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:26.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.229+0000 7fe2c0c00700 1 -- 172.21.15.28:0/2101556881 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fe2b400f050 con 0x7fe2c4105200 2024-05-20T19:53:26.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.229+0000 7fe2baa00700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe2a803e510 0x7fe2a80409d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:26.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.230+0000 7fe2baa00700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe2a803e510 0x7fe2a80409d0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fe2ac006fd0 tx=0x7fe2ac006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:26.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.234+0000 7fe2c0c00700 1 -- 172.21.15.28:0/2101556881 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fe2b4057030 con 0x7fe2c4105200 2024-05-20T19:53:26.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:26.476+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi033", "target": ["mon-mgr", ""]}) -- 0x7fe2c419bcf0 con 0x7fe2a803e510 2024-05-20T19:53:27.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:27 smithi028 ceph-mon[31178]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi033", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:28.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:28 smithi028 ceph-mon[31178]: Deploying cephadm binary to smithi033 2024-05-20T19:53:28.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:28 smithi028 ceph-mon[31178]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:30.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:30 smithi028 ceph-mon[31178]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:31.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.426+0000 7fe2c0c00700 1 -- 172.21.15.28:0/2101556881 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+48 (secure 0 0 0) 0x7fe2c419bcf0 con 0x7fe2a803e510 2024-05-20T19:53:31.428 INFO:teuthology.orchestra.run.smithi028.stdout:Added host 'smithi033' with addr '172.21.15.33' 2024-05-20T19:53:31.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe2a803e510 msgr2=0x7fe2a80409d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe2a803e510 0x7fe2a80409d0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fe2ac006fd0 tx=0x7fe2ac006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 msgr2=0x7fe2c41a0640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41a0640 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7fe2b4006130 tx=0x7fe2b40097d0 comp rx=0 tx=0).stop 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 shutdown_connections 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe2a803e510 0x7fe2a80409d0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 --2- 172.21.15.28:0/2101556881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe2c4105200 0x7fe2c41a0640 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 >> 172.21.15.28:0/2101556881 conn(0x7fe2c4100a70 msgr2=0x7fe2c4106bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.430+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 shutdown_connections 2024-05-20T19:53:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:31.431+0000 7fe2c8e00700 1 -- 172.21.15.28:0/2101556881 wait complete. 2024-05-20T19:53:32.209 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch host ls --format=json 2024-05-20T19:53:32.655 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:53:32.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:32 smithi028 ceph-mon[31178]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:32.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:32 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:32.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:32 smithi028 ceph-mon[31178]: Added host smithi033 2024-05-20T19:53:32.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:32 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:32.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:32 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:33.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:33 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:34.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.553+0000 7f96e7800700 1 -- 172.21.15.28:0/4280334625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 msgr2=0x7f96e010e960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:34.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.553+0000 7f96e7800700 1 --2- 172.21.15.28:0/4280334625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e010e960 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f96d4009910 tx=0x7f96d4009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:34.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.553+0000 7f96e7800700 1 -- 172.21.15.28:0/4280334625 shutdown_connections 2024-05-20T19:53:34.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.553+0000 7f96e7800700 1 --2- 172.21.15.28:0/4280334625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e010e960 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:34.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.553+0000 7f96e7800700 1 -- 172.21.15.28:0/4280334625 >> 172.21.15.28:0/4280334625 conn(0x7f96e0078ce0 msgr2=0x7f96e007b100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:34.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.553+0000 7f96e7800700 1 -- 172.21.15.28:0/4280334625 shutdown_connections 2024-05-20T19:53:34.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.553+0000 7f96e7800700 1 -- 172.21.15.28:0/4280334625 wait complete. 2024-05-20T19:53:34.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.555+0000 7f96e7800700 1 Processor -- start 2024-05-20T19:53:34.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.555+0000 7f96e7800700 1 -- start start 2024-05-20T19:53:34.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.555+0000 7f96e7800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e019ef90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:34.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.555+0000 7f96e7800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f96e0113aa0 con 0x7f96e010e580 2024-05-20T19:53:34.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.555+0000 7f96e4e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e019ef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:34.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.556+0000 7f96e4e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e019ef90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41486/0 (socket says 172.21.15.28:41486) 2024-05-20T19:53:34.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.556+0000 7f96e4e00700 1 -- 172.21.15.28:0/2869030799 learned_addr learned my addr 172.21.15.28:0/2869030799 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:34.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.556+0000 7f96e4e00700 1 -- 172.21.15.28:0/2869030799 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f96e019f4d0 con 0x7f96e010e580 2024-05-20T19:53:34.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.556+0000 7f96e4e00700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e019ef90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f96d40056d0 tx=0x7f96d40057b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:34.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.557+0000 7f96dd600700 1 -- 172.21.15.28:0/2869030799 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f96d4027070 con 0x7f96e010e580 2024-05-20T19:53:34.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.557+0000 7f96dd600700 1 -- 172.21.15.28:0/2869030799 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f96d400be10 con 0x7f96e010e580 2024-05-20T19:53:34.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.557+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f96e019f760 con 0x7f96e010e580 2024-05-20T19:53:34.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.557+0000 7f96dd600700 1 -- 172.21.15.28:0/2869030799 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f96d4022d20 con 0x7f96e010e580 2024-05-20T19:53:34.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.557+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f96e01a3810 con 0x7f96e010e580 2024-05-20T19:53:34.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.558+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f96e010ac80 con 0x7f96e010e580 2024-05-20T19:53:34.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.559+0000 7f96dd600700 1 -- 172.21.15.28:0/2869030799 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f96d402a070 con 0x7f96e010e580 2024-05-20T19:53:34.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.559+0000 7f96dd600700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f96c803e570 0x7f96c8040a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:34.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.560+0000 7f96dd600700 1 -- 172.21.15.28:0/2869030799 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f96d401e030 con 0x7f96e010e580 2024-05-20T19:53:34.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.560+0000 7f96dfe00700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f96c803e570 0x7f96c8040a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:34.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.561+0000 7f96dfe00700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f96c803e570 0x7f96c8040a30 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f96cc006fd0 tx=0x7f96cc006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:34.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.565+0000 7f96dd600700 1 -- 172.21.15.28:0/2869030799 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f96d4013070 con 0x7f96e010e580 2024-05-20T19:53:34.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:34 smithi028 ceph-mon[31178]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:34.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:34 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:34.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.805+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f96e01a02a0 con 0x7f96c803e570 2024-05-20T19:53:34.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.807+0000 7f96dd600700 1 -- 172.21.15.28:0/2869030799 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+159 (secure 0 0 0) 0x7f96e01a02a0 con 0x7f96c803e570 2024-05-20T19:53:34.808 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:34.808 INFO:teuthology.orchestra.run.smithi028.stdout:[{"addr": "172.21.15.28", "hostname": "smithi028", "labels": [], "status": ""}, {"addr": "172.21.15.33", "hostname": "smithi033", "labels": [], "status": ""}] 2024-05-20T19:53:34.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f96c803e570 msgr2=0x7f96c8040a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:34.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f96c803e570 0x7f96c8040a30 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f96cc006fd0 tx=0x7f96cc006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 msgr2=0x7f96e019ef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e019ef90 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f96d40056d0 tx=0x7f96d40057b0 comp rx=0 tx=0).stop 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 shutdown_connections 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f96c803e570 0x7f96c8040a30 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 --2- 172.21.15.28:0/2869030799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f96e010e580 0x7f96e019ef90 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 >> 172.21.15.28:0/2869030799 conn(0x7f96e0078ce0 msgr2=0x7f96e01110e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 shutdown_connections 2024-05-20T19:53:34.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:34.810+0000 7f96e7800700 1 -- 172.21.15.28:0/2869030799 wait complete. 2024-05-20T19:53:35.458 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi154 2024-05-20T19:53:35.458 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:53:35.458 DEBUG:teuthology.orchestra.run.smithi154:> dd of=/etc/ceph/ceph.conf 2024-05-20T19:53:35.477 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:53:35.477 DEBUG:teuthology.orchestra.run.smithi154:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:35.535 INFO:tasks.cephadm:Adding host smithi154 to orchestrator... 2024-05-20T19:53:35.535 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch host add smithi154 2024-05-20T19:53:35.977 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:53:36.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:36 smithi028 ceph-mon[31178]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T19:53:36.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:36 smithi028 ceph-mon[31178]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:38.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.021+0000 7fe6d5800700 1 -- 172.21.15.28:0/2121089215 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 msgr2=0x7fe6d01057a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:38.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.021+0000 7fe6d5800700 1 --2- 172.21.15.28:0/2121089215 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01057a0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fe6c0009910 tx=0x7fe6c0009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:38.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.021+0000 7fe6d5800700 1 -- 172.21.15.28:0/2121089215 shutdown_connections 2024-05-20T19:53:38.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.021+0000 7fe6d5800700 1 --2- 172.21.15.28:0/2121089215 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01057a0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:38.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.021+0000 7fe6d5800700 1 -- 172.21.15.28:0/2121089215 >> 172.21.15.28:0/2121089215 conn(0x7fe6d0100c90 msgr2=0x7fe6d01030b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:38.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.021+0000 7fe6d5800700 1 -- 172.21.15.28:0/2121089215 shutdown_connections 2024-05-20T19:53:38.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.021+0000 7fe6d5800700 1 -- 172.21.15.28:0/2121089215 wait complete. 2024-05-20T19:53:38.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.022+0000 7fe6d5800700 1 Processor -- start 2024-05-20T19:53:38.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.022+0000 7fe6d5800700 1 -- start start 2024-05-20T19:53:38.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.023+0000 7fe6d5800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01a07c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:38.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.023+0000 7fe6d5800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe6d010f640 con 0x7fe6d01053c0 2024-05-20T19:53:38.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.023+0000 7fe6cea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01a07c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:38.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.023+0000 7fe6cea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01a07c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47924/0 (socket says 172.21.15.28:47924) 2024-05-20T19:53:38.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.023+0000 7fe6cea00700 1 -- 172.21.15.28:0/2991072270 learned_addr learned my addr 172.21.15.28:0/2991072270 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:38.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.024+0000 7fe6cea00700 1 -- 172.21.15.28:0/2991072270 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe6d01a0d00 con 0x7fe6d01053c0 2024-05-20T19:53:38.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.024+0000 7fe6cea00700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01a07c0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fe6c0006010 tx=0x7fe6c0005780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:38.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.024+0000 7fe6c7400700 1 -- 172.21.15.28:0/2991072270 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe6c0026070 con 0x7fe6d01053c0 2024-05-20T19:53:38.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.025+0000 7fe6c7400700 1 -- 172.21.15.28:0/2991072270 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe6c000bba0 con 0x7fe6d01053c0 2024-05-20T19:53:38.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.025+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe6d01a0ff0 con 0x7fe6d01053c0 2024-05-20T19:53:38.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.025+0000 7fe6c7400700 1 -- 172.21.15.28:0/2991072270 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fe6c0003910 con 0x7fe6d01053c0 2024-05-20T19:53:38.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.025+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe6d019a980 con 0x7fe6d01053c0 2024-05-20T19:53:38.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.026+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe6d01068a0 con 0x7fe6d01053c0 2024-05-20T19:53:38.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.026+0000 7fe6c7400700 1 -- 172.21.15.28:0/2991072270 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7fe6c0029070 con 0x7fe6d01053c0 2024-05-20T19:53:38.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.027+0000 7fe6c7400700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe6b403e570 0x7fe6b4040a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:38.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.027+0000 7fe6c7400700 1 -- 172.21.15.28:0/2991072270 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fe6c001e070 con 0x7fe6d01053c0 2024-05-20T19:53:38.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.027+0000 7fe6ce000700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe6b403e570 0x7fe6b4040a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:38.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.028+0000 7fe6ce000700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe6b403e570 0x7fe6b4040a30 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fe6bc006fd0 tx=0x7fe6bc006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:38.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.034+0000 7fe6c7400700 1 -- 172.21.15.28:0/2991072270 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fe6c0013070 con 0x7fe6d01053c0 2024-05-20T19:53:38.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:38.277+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi154", "target": ["mon-mgr", ""]}) -- 0x7fe6d019beb0 con 0x7fe6b403e570 2024-05-20T19:53:38.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:38.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi033", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T19:53:38.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:38 smithi028 ceph-mon[31178]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:53:39.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:39 smithi028 ceph-mon[31178]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:39 smithi028 ceph-mon[31178]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi154", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:39 smithi028 ceph-mon[31178]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T19:53:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:40 smithi028 ceph-mon[31178]: Deploying cephadm binary to smithi154 2024-05-20T19:53:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:40 smithi028 ceph-mon[31178]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:42 smithi028 ceph-mon[31178]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:43.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.096+0000 7fe6c7400700 1 -- 172.21.15.28:0/2991072270 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+49 (secure 0 0 0) 0x7fe6d019beb0 con 0x7fe6b403e570 2024-05-20T19:53:43.097 INFO:teuthology.orchestra.run.smithi028.stdout:Added host 'smithi154' with addr '172.21.15.154' 2024-05-20T19:53:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe6b403e570 msgr2=0x7fe6b4040a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe6b403e570 0x7fe6b4040a30 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fe6bc006fd0 tx=0x7fe6bc006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 msgr2=0x7fe6d01a07c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01a07c0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fe6c0006010 tx=0x7fe6c0005780 comp rx=0 tx=0).stop 2024-05-20T19:53:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 shutdown_connections 2024-05-20T19:53:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fe6b403e570 0x7fe6b4040a30 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 --2- 172.21.15.28:0/2991072270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6d01053c0 0x7fe6d01a07c0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.100+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 >> 172.21.15.28:0/2991072270 conn(0x7fe6d0100c90 msgr2=0x7fe6d0106d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.101+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 shutdown_connections 2024-05-20T19:53:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:43.101+0000 7fe6d5800700 1 -- 172.21.15.28:0/2991072270 wait complete. 2024-05-20T19:53:43.877 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch host ls --format=json 2024-05-20T19:53:44.344 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:53:44.486 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:44.486 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:44 smithi028 ceph-mon[31178]: Added host smithi154 2024-05-20T19:53:44.486 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:44.486 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:44 smithi028 ceph-mon[31178]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:44.486 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:45.440 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:46.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.202+0000 7fca97000700 1 -- 172.21.15.28:0/3660780852 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 msgr2=0x7fca901044f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:46.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.202+0000 7fca97000700 1 --2- 172.21.15.28:0/3660780852 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca901044f0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fca80009910 tx=0x7fca80009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:46.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.202+0000 7fca97000700 1 -- 172.21.15.28:0/3660780852 shutdown_connections 2024-05-20T19:53:46.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.202+0000 7fca97000700 1 --2- 172.21.15.28:0/3660780852 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca901044f0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:46.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.202+0000 7fca97000700 1 -- 172.21.15.28:0/3660780852 >> 172.21.15.28:0/3660780852 conn(0x7fca900fd940 msgr2=0x7fca900ffd60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:46.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.202+0000 7fca97000700 1 -- 172.21.15.28:0/3660780852 shutdown_connections 2024-05-20T19:53:46.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.202+0000 7fca97000700 1 -- 172.21.15.28:0/3660780852 wait complete. 2024-05-20T19:53:46.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.203+0000 7fca97000700 1 Processor -- start 2024-05-20T19:53:46.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.204+0000 7fca97000700 1 -- start start 2024-05-20T19:53:46.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.204+0000 7fca97000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca90101d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:46.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.204+0000 7fca97000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fca9010d490 con 0x7fca90104110 2024-05-20T19:53:46.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.204+0000 7fca8fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca90101d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:46.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.204+0000 7fca8fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca90101d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47948/0 (socket says 172.21.15.28:47948) 2024-05-20T19:53:46.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.205+0000 7fca8fe00700 1 -- 172.21.15.28:0/1346979738 learned_addr learned my addr 172.21.15.28:0/1346979738 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:46.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.205+0000 7fca8fe00700 1 -- 172.21.15.28:0/1346979738 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca90103c60 con 0x7fca90104110 2024-05-20T19:53:46.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.205+0000 7fca8fe00700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca90101d70 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fca80006010 tx=0x7fca80005780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:46.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.206+0000 7fca8cc00700 1 -- 172.21.15.28:0/1346979738 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fca80026070 con 0x7fca90104110 2024-05-20T19:53:46.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.206+0000 7fca8cc00700 1 -- 172.21.15.28:0/1346979738 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fca8000bc00 con 0x7fca90104110 2024-05-20T19:53:46.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.206+0000 7fca8cc00700 1 -- 172.21.15.28:0/1346979738 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fca80003910 con 0x7fca90104110 2024-05-20T19:53:46.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.206+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca901022b0 con 0x7fca90104110 2024-05-20T19:53:46.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.206+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca90102770 con 0x7fca90104110 2024-05-20T19:53:46.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.207+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fca9006bc30 con 0x7fca90104110 2024-05-20T19:53:46.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.207+0000 7fca8cc00700 1 -- 172.21.15.28:0/1346979738 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7fca80029070 con 0x7fca90104110 2024-05-20T19:53:46.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.208+0000 7fca8cc00700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fca7403e570 0x7fca74040a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:46.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.208+0000 7fca8cc00700 1 -- 172.21.15.28:0/1346979738 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7fca8001e070 con 0x7fca90104110 2024-05-20T19:53:46.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.208+0000 7fca8f400700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fca7403e570 0x7fca74040a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:46.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.209+0000 7fca8f400700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fca7403e570 0x7fca74040a30 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fca7c006fd0 tx=0x7fca7c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:46.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.214+0000 7fca8cc00700 1 -- 172.21.15.28:0/1346979738 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fca80013070 con 0x7fca90104110 2024-05-20T19:53:46.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:46 smithi028 ceph-mon[31178]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:46.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:46.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.462+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fca901032a0 con 0x7fca7403e570 2024-05-20T19:53:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.463+0000 7fca8cc00700 1 -- 172.21.15.28:0/1346979738 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+239 (secure 0 0 0) 0x7fca901032a0 con 0x7fca7403e570 2024-05-20T19:53:46.464 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T19:53:46.464 INFO:teuthology.orchestra.run.smithi028.stdout:[{"addr": "172.21.15.28", "hostname": "smithi028", "labels": [], "status": ""}, {"addr": "172.21.15.33", "hostname": "smithi033", "labels": [], "status": ""}, {"addr": "172.21.15.154", "hostname": "smithi154", "labels": [], "status": ""}] 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fca7403e570 msgr2=0x7fca74040a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fca7403e570 0x7fca74040a30 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fca7c006fd0 tx=0x7fca7c006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 msgr2=0x7fca90101d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca90101d70 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fca80006010 tx=0x7fca80005780 comp rx=0 tx=0).stop 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 shutdown_connections 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fca7403e570 0x7fca74040a30 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 --2- 172.21.15.28:0/1346979738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca90104110 0x7fca90101d70 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.466+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 >> 172.21.15.28:0/1346979738 conn(0x7fca900fd940 msgr2=0x7fca90104ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.467+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 shutdown_connections 2024-05-20T19:53:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:46.467+0000 7fca97000700 1 -- 172.21.15.28:0/1346979738 wait complete. 2024-05-20T19:53:47.234 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-20T19:53:47.234 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd crush tunables default 2024-05-20T19:53:47.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:47 smithi028 ceph-mon[31178]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T19:53:47.701 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:53:48.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:48 smithi028 ceph-mon[31178]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:49.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.701+0000 7f7490c00700 1 -- 172.21.15.28:0/3692726061 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 msgr2=0x7f748c1057a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:49.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.701+0000 7f7490c00700 1 --2- 172.21.15.28:0/3692726061 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1057a0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f747c009910 tx=0x7f747c009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:49.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.701+0000 7f7490c00700 1 -- 172.21.15.28:0/3692726061 shutdown_connections 2024-05-20T19:53:49.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.701+0000 7f7490c00700 1 --2- 172.21.15.28:0/3692726061 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1057a0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:49.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.701+0000 7f7490c00700 1 -- 172.21.15.28:0/3692726061 >> 172.21.15.28:0/3692726061 conn(0x7f748c100c90 msgr2=0x7f748c1030b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:49.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.701+0000 7f7490c00700 1 -- 172.21.15.28:0/3692726061 shutdown_connections 2024-05-20T19:53:49.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.702+0000 7f7490c00700 1 -- 172.21.15.28:0/3692726061 wait complete. 2024-05-20T19:53:49.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.703+0000 7f7490c00700 1 Processor -- start 2024-05-20T19:53:49.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.703+0000 7f7490c00700 1 -- start start 2024-05-20T19:53:49.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.704+0000 7f7490c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1a07c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:49.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.704+0000 7f7490c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f748c10f640 con 0x7f748c1053c0 2024-05-20T19:53:49.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.704+0000 7f7489e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1a07c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:49.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.704+0000 7f7489e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1a07c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35368/0 (socket says 172.21.15.28:35368) 2024-05-20T19:53:49.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.704+0000 7f7489e00700 1 -- 172.21.15.28:0/3792071918 learned_addr learned my addr 172.21.15.28:0/3792071918 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:53:49.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.704+0000 7f7489e00700 1 -- 172.21.15.28:0/3792071918 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f748c1a0d00 con 0x7f748c1053c0 2024-05-20T19:53:49.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.705+0000 7f7489e00700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1a07c0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f747c0056d0 tx=0x7f747c0057b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:49.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.705+0000 7f7482a00700 1 -- 172.21.15.28:0/3792071918 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f747c026070 con 0x7f748c1053c0 2024-05-20T19:53:49.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.705+0000 7f7482a00700 1 -- 172.21.15.28:0/3792071918 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f747c00bba0 con 0x7f748c1053c0 2024-05-20T19:53:49.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.705+0000 7f7482a00700 1 -- 172.21.15.28:0/3792071918 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f747c003920 con 0x7f748c1053c0 2024-05-20T19:53:49.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.705+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f748c1a0ff0 con 0x7f748c1053c0 2024-05-20T19:53:49.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.705+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f748c19a980 con 0x7f748c1053c0 2024-05-20T19:53:49.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.707+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f748c1068a0 con 0x7f748c1053c0 2024-05-20T19:53:49.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.707+0000 7f7482a00700 1 -- 172.21.15.28:0/3792071918 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f747c029070 con 0x7f748c1053c0 2024-05-20T19:53:49.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.707+0000 7f7482a00700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f747003e570 0x7f7470040a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:49.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.708+0000 7f7482a00700 1 -- 172.21.15.28:0/3792071918 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(3..3 src has 1..3) ==== 1155+0+0 (secure 0 0 0) 0x7f747c01e070 con 0x7f748c1053c0 2024-05-20T19:53:49.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.708+0000 7f7489400700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f747003e570 0x7f7470040a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:49.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.709+0000 7f7489400700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f747003e570 0x7f7470040a30 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f7478006fd0 tx=0x7f7478006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:49.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.713+0000 7f7482a00700 1 -- 172.21.15.28:0/3792071918 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f747c013070 con 0x7f748c1053c0 2024-05-20T19:53:49.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:49.961+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) -- 0x7f748c19beb0 con 0x7f748c1053c0 2024-05-20T19:53:50.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.232+0000 7f7482a00700 1 -- 172.21.15.28:0/3792071918 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v4) ==== 124+0+0 (secure 0 0 0) 0x7f747c00d340 con 0x7f748c1053c0 2024-05-20T19:53:50.233 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:50.233 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:50.233 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:50.233 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:50.233 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi154", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:53:50.233 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:50.234 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:53:50.234 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T19:53:50.234 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:50.234 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:53:50.234 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:50 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3792071918' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-20T19:53:50.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f747003e570 msgr2=0x7f7470040a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:50.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f747003e570 0x7f7470040a30 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f7478006fd0 tx=0x7f7478006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:50.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 msgr2=0x7f748c1a07c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:50.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1a07c0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f747c0056d0 tx=0x7f747c0057b0 comp rx=0 tx=0).stop 2024-05-20T19:53:50.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 shutdown_connections 2024-05-20T19:53:50.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f747003e570 0x7f7470040a30 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:50.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 --2- 172.21.15.28:0/3792071918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f748c1053c0 0x7f748c1a07c0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:50.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.237+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 >> 172.21.15.28:0/3792071918 conn(0x7f748c100c90 msgr2=0x7f748c106d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:50.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.238+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 shutdown_connections 2024-05-20T19:53:50.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:53:50.238+0000 7f7490c00700 1 -- 172.21.15.28:0/3792071918 wait complete. 2024-05-20T19:53:50.239 INFO:teuthology.orchestra.run.smithi028.stderr:adjusted tunables profile to default 2024-05-20T19:53:50.934 INFO:tasks.cephadm:Adding mon.a on smithi028 2024-05-20T19:53:50.934 INFO:tasks.cephadm:Adding mon.b on smithi033 2024-05-20T19:53:50.934 INFO:tasks.cephadm:Adding mon.c on smithi154 2024-05-20T19:53:50.934 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch apply mon '3;smithi028:172.21.15.28=a;smithi033:172.21.15.33=b;smithi154:172.21.15.154=c' 2024-05-20T19:53:51.159 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:53:51.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:51 smithi028 ceph-mon[31178]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:51.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:51 smithi028 ceph-mon[31178]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T19:53:51.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:51 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3792071918' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-20T19:53:51.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:51 smithi028 ceph-mon[31178]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:53:51.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:51.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:51.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:52.420 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.419+0000 7f29045ea700 1 -- 172.21.15.154:0/3717794169 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 msgr2=0x7f28fc105320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:52.420 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.419+0000 7f29045ea700 1 --2- 172.21.15.154:0/3717794169 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc105320 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f28f8009910 tx=0x7f28f8009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.419+0000 7f29045ea700 1 -- 172.21.15.154:0/3717794169 shutdown_connections 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.419+0000 7f29045ea700 1 --2- 172.21.15.154:0/3717794169 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc105320 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.419+0000 7f29045ea700 1 -- 172.21.15.154:0/3717794169 >> 172.21.15.154:0/3717794169 conn(0x7f28fc0fd0b0 msgr2=0x7f28fc0ff4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.419+0000 7f29045ea700 1 -- 172.21.15.154:0/3717794169 shutdown_connections 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.419+0000 7f29045ea700 1 -- 172.21.15.154:0/3717794169 wait complete. 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.420+0000 7f29045ea700 1 Processor -- start 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.420+0000 7f29045ea700 1 -- start start 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.420+0000 7f29045ea700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc19e730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:52.421 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.420+0000 7f29045ea700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f28fc10d630 con 0x7f28fc104f40 2024-05-20T19:53:52.422 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.421+0000 7f2902386700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc19e730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:52.422 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.421+0000 7f2902386700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc19e730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.154:37144/0 (socket says 172.21.15.154:37144) 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.421+0000 7f2902386700 1 -- 172.21.15.154:0/1310828905 learned_addr learned my addr 172.21.15.154:0/1310828905 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.421+0000 7f2902386700 1 -- 172.21.15.154:0/1310828905 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f28fc19ec70 con 0x7f28fc104f40 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.422+0000 7f2902386700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc19e730 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f28f80056d0 tx=0x7f28f80057b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.422+0000 7f28f37fe700 1 -- 172.21.15.154:0/1310828905 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f28f8026070 con 0x7f28fc104f40 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.422+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f28fc19ef00 con 0x7f28fc104f40 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.422+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f28fc1988f0 con 0x7f28fc104f40 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.422+0000 7f28f37fe700 1 -- 172.21.15.154:0/1310828905 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f28f800b750 con 0x7f28fc104f40 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.422+0000 7f28f37fe700 1 -- 172.21.15.154:0/1310828905 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7f28f8030060 con 0x7f28fc104f40 2024-05-20T19:53:52.423 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.423+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f28e0005330 con 0x7f28fc104f40 2024-05-20T19:53:52.424 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.424+0000 7f28f37fe700 1 -- 172.21.15.154:0/1310828905 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f28f8029070 con 0x7f28fc104f40 2024-05-20T19:53:52.424 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.424+0000 7f28f37fe700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f28e803e510 0x7f28e80409d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:52.424 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.424+0000 7f28f37fe700 1 -- 172.21.15.154:0/1310828905 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f28f801e070 con 0x7f28fc104f40 2024-05-20T19:53:52.424 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.424+0000 7f2901b85700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f28e803e510 0x7f28e80409d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:52.426 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.425+0000 7f2901b85700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f28e803e510 0x7f28e80409d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f28ec006fd0 tx=0x7f28ec006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:52.440 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.440+0000 7f28f37fe700 1 -- 172.21.15.154:0/1310828905 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f28f8013070 con 0x7f28fc104f40 2024-05-20T19:53:52.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:52 smithi028 ceph-mon[31178]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:52.656 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.656+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi028:172.21.15.28=a;smithi033:172.21.15.33=b;smithi154:172.21.15.154=c", "target": ["mon-mgr", ""]}) -- 0x7f28e0000c80 con 0x7f28e803e510 2024-05-20T19:53:52.672 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.671+0000 7f28f37fe700 1 -- 172.21.15.154:0/1310828905 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f28e0000c80 con 0x7f28e803e510 2024-05-20T19:53:52.672 INFO:teuthology.orchestra.run.smithi154.stdout:Scheduled mon update... 2024-05-20T19:53:52.674 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f28e803e510 msgr2=0x7f28e80409d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:52.674 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f28e803e510 0x7f28e80409d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f28ec006fd0 tx=0x7f28ec006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:52.674 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 msgr2=0x7f28fc19e730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:52.674 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc19e730 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f28f80056d0 tx=0x7f28f80057b0 comp rx=0 tx=0).stop 2024-05-20T19:53:52.674 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 shutdown_connections 2024-05-20T19:53:52.675 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f28e803e510 0x7f28e80409d0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:52.675 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 --2- 172.21.15.154:0/1310828905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28fc104f40 0x7f28fc19e730 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:52.675 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 >> 172.21.15.154:0/1310828905 conn(0x7f28fc0fd0b0 msgr2=0x7f28fc10ab30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:52.675 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 shutdown_connections 2024-05-20T19:53:52.675 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:52.674+0000 7f29045ea700 1 -- 172.21.15.154:0/1310828905 wait complete. 2024-05-20T19:53:53.298 DEBUG:teuthology.orchestra.run.smithi033:mon.b> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.b.service 2024-05-20T19:53:53.301 DEBUG:teuthology.orchestra.run.smithi154:mon.c> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.c.service 2024-05-20T19:53:53.303 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-20T19:53:53.304 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T19:53:54.042 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:53:54.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi028:172.21.15.28=a;smithi033:172.21.15.33=b;smithi154:172.21.15.154=c", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:54.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: Saving service mon spec with placement smithi028:172.21.15.28=a;smithi033:172.21.15.33=b;smithi154:172.21.15.154=c;count:3 2024-05-20T19:53:54.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:54.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:54.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:54.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:53:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:53:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:54.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:53 smithi028 ceph-mon[31178]: Deploying daemon mon.c on smithi154 2024-05-20T19:53:55.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:54 smithi028 ceph-mon[31178]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:56.898 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 -- 172.21.15.154:0/1099049363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde81031b0 msgr2=0x7fcde8103590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:56.898 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 --2- 172.21.15.154:0/1099049363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde81031b0 0x7fcde8103590 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fcdd8009910 tx=0x7fcdd8009c20 comp rx=0 tx=0).stop 2024-05-20T19:53:56.898 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 -- 172.21.15.154:0/1099049363 shutdown_connections 2024-05-20T19:53:56.898 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 --2- 172.21.15.154:0/1099049363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde81031b0 0x7fcde8103590 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:56.899 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 -- 172.21.15.154:0/1099049363 >> 172.21.15.154:0/1099049363 conn(0x7fcde80fea60 msgr2=0x7fcde8100e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:56.899 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 -- 172.21.15.154:0/1099049363 shutdown_connections 2024-05-20T19:53:56.899 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 -- 172.21.15.154:0/1099049363 wait complete. 2024-05-20T19:53:56.899 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 Processor -- start 2024-05-20T19:53:56.899 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.897+0000 7fcdefe84700 1 -- start start 2024-05-20T19:53:56.899 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.898+0000 7fcdefe84700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde819e840 0x7fcde819ec20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:56.899 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.898+0000 7fcdefe84700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fcde810d430 con 0x7fcde819e840 2024-05-20T19:53:56.900 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.898+0000 7fcdedc20700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde819e840 0x7fcde819ec20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:56.900 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.898+0000 7fcdedc20700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde819e840 0x7fcde819ec20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.154:49748/0 (socket says 172.21.15.154:49748) 2024-05-20T19:53:56.900 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.898+0000 7fcdedc20700 1 -- 172.21.15.154:0/3803290314 learned_addr learned my addr 172.21.15.154:0/3803290314 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:53:56.900 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.899+0000 7fcdedc20700 1 -- 172.21.15.154:0/3803290314 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcde819f160 con 0x7fcde819e840 2024-05-20T19:53:56.900 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.899+0000 7fcdedc20700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde819e840 0x7fcde819ec20 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fcdd800b5c0 tx=0x7fcdd8005780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:56.901 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.900+0000 7fcddeffd700 1 -- 172.21.15.154:0/3803290314 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcdd8026070 con 0x7fcde819e840 2024-05-20T19:53:56.901 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.900+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcde819f450 con 0x7fcde819e840 2024-05-20T19:53:56.901 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.900+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcde8198c30 con 0x7fcde819e840 2024-05-20T19:53:56.901 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.900+0000 7fcddeffd700 1 -- 172.21.15.154:0/3803290314 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcdd800b6f0 con 0x7fcde819e840 2024-05-20T19:53:56.901 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.900+0000 7fcddeffd700 1 -- 172.21.15.154:0/3803290314 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 205+0+0 (secure 0 0 0) 0x7fcdd8030060 con 0x7fcde819e840 2024-05-20T19:53:56.901 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.900+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcdcc005330 con 0x7fcde819e840 2024-05-20T19:53:56.902 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.901+0000 7fcddeffd700 1 -- 172.21.15.154:0/3803290314 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7fcdd8029070 con 0x7fcde819e840 2024-05-20T19:53:56.902 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.901+0000 7fcddeffd700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcdd403e510 0x7fcdd40409d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:53:56.902 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.901+0000 7fcddeffd700 1 -- 172.21.15.154:0/3803290314 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7fcdd801e070 con 0x7fcde819e840 2024-05-20T19:53:56.902 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.902+0000 7fcded41f700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcdd403e510 0x7fcdd40409d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:53:56.903 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.903+0000 7fcded41f700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcdd403e510 0x7fcdd40409d0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fcde4006fd0 tx=0x7fcde4006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:53:56.918 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:56.918+0000 7fcddeffd700 1 -- 172.21.15.154:0/3803290314 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fcde819f450 con 0x7fcde819e840 2024-05-20T19:53:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:56 smithi028 ceph-mon[31178]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:57.241 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.240+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7fcdcc0058f0 con 0x7fcde819e840 2024-05-20T19:53:57.241 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.241+0000 7fcddeffd700 1 -- 172.21.15.154:0/3803290314 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) ==== 95+0+745 (secure 0 0 0) 0x7fcdd8013070 con 0x7fcde819e840 2024-05-20T19:53:57.242 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:53:57.242 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","modified":"2024-05-20T19:51:39.025045Z","created":"2024-05-20T19:51:39.025045Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T19:53:57.244 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcdd403e510 msgr2=0x7fcdd40409d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:57.244 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcdd403e510 0x7fcdd40409d0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fcde4006fd0 tx=0x7fcde4006e40 comp rx=0 tx=0).stop 2024-05-20T19:53:57.244 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde819e840 msgr2=0x7fcde819ec20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:53:57.244 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde819e840 0x7fcde819ec20 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fcdd800b5c0 tx=0x7fcdd8005780 comp rx=0 tx=0).stop 2024-05-20T19:53:57.244 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 shutdown_connections 2024-05-20T19:53:57.245 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcdd403e510 0x7fcdd40409d0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:57.245 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 --2- 172.21.15.154:0/3803290314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcde819e840 0x7fcde819ec20 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:53:57.245 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 >> 172.21.15.154:0/3803290314 conn(0x7fcde80fea60 msgr2=0x7fcde8100660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:53:57.245 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 shutdown_connections 2024-05-20T19:53:57.245 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:53:57.244+0000 7fcdefe84700 1 -- 172.21.15.154:0/3803290314 wait complete. 2024-05-20T19:53:57.245 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-05-20T19:53:57.604 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:57 smithi154 systemd[1]: Starting Ceph mon.c for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T19:53:58.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:53:57 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/3803290314' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T19:53:58.277 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 podman[36295]: 2024-05-20 19:53:58.005536942 +0000 UTC m=+0.114146780 container create 7216247a8a467d2978d24ab6059ba653b89fb4da236f1cb9b791d9b98a07d077 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-20T19:53:58.277 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 podman[36295]: 2024-05-20 19:53:57.908875729 +0000 UTC m=+0.017485581 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T19:53:58.277 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 podman[36295]: 2024-05-20 19:53:58.216020284 +0000 UTC m=+0.324630125 container init 7216247a8a467d2978d24ab6059ba653b89fb4da236f1cb9b791d9b98a07d077 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T19:53:58.277 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 podman[36295]: 2024-05-20 19:53:58.219077849 +0000 UTC m=+0.327687700 container start 7216247a8a467d2978d24ab6059ba653b89fb4da236f1cb9b791d9b98a07d077 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-05-20T19:53:58.277 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T19:53:58.277 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: ceph version 19.0.0-3830-g4ebd1d53 (4ebd1d5361d457738957dc0455bba90102296634) squid (dev), process ceph-mon, pid 2 2024-05-20T19:53:58.277 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pidfile_write: ignore empty --pid-file 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: load: jerasure load: lrc 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: RocksDB version: 7.9.2 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Git sha 0 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Compile date 2024-05-20 17:11:59 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: DB SUMMARY 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: DB Session ID: YAHEJZHGS28VKTVEDEVZ 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: CURRENT file: CURRENT 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-05-20T19:53:58.278 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.error_if_exists: 0 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.create_if_missing: 0 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.paranoid_checks: 1 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.env: 0x56041bf5acc0 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.fs: PosixFileSystem 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.info_log: 0x56041d635da0 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.statistics: (nil) 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.use_fsync: 0 2024-05-20T19:53:58.279 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_log_file_size: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.allow_fallocate: 1 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.use_direct_reads: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.db_log_dir: 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.wal_dir: 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.write_buffer_manager: 0x56041d6070e0 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T19:53:58.281 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.unordered_write: 0 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T19:53:58.283 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.row_cache: None 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.wal_filter: None 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.two_write_queues: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.wal_compression: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.atomic_flush: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T19:53:58.284 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.log_readahead_size: 0 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_background_jobs: 2 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_background_compactions: -1 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_subcompactions: 1 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T19:53:58.285 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_open_files: -1 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T19:53:58.286 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_background_flushes: -1 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Compression algorithms supported: 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kZSTD supported: 0 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kXpressCompression supported: 0 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kZlibCompression supported: 1 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kSnappyCompression supported: 1 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kLZ4Compression supported: 1 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: kBZip2Compression supported: 0 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-20T19:53:58.287 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.merge_operator: 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_filter: None 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56041d635f60) 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout: cache_index_and_filter_blocks: 1 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout: pin_top_level_index_and_filter: 1 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout: index_type: 0 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout: data_block_index_type: 0 2024-05-20T19:53:58.288 INFO:journalctl@ceph.mon.c.smithi154.stdout: index_shortening: 1 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: checksum: 4 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: no_block_cache: 0 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_cache: 0x56041d6391f0 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_cache_name: BinnedLRUCache 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_cache_options: 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: capacity : 536870912 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: num_shard_bits : 4 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: strict_capacity_limit : 0 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: high_pri_pool_ratio: 0.000 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_cache_compressed: (nil) 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: persistent_cache: (nil) 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_size: 4096 2024-05-20T19:53:58.289 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_size_deviation: 10 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_restart_interval: 16 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: index_block_restart_interval: 1 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: metadata_block_size: 4096 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: partition_filters: 0 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: use_delta_encoding: 1 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: filter_policy: bloomfilter 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: whole_key_filtering: 1 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: verify_compression: 0 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: read_amp_bytes_per_bit: 0 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: format_version: 5 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: enable_index_compression: 1 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: block_align: 0 2024-05-20T19:53:58.290 INFO:journalctl@ceph.mon.c.smithi154.stdout: max_auto_readahead_size: 262144 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout: prepopulate_block_cache: 0 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout: initial_auto_readahead_size: 8192 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout: num_file_reads_for_auto_readahead: 2 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression: NoCompression 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.num_levels: 7 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T19:53:58.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T19:53:58.292 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T19:53:58.293 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.arena_block_size: 1048576 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T19:53:58.294 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.inplace_update_support: 0 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T19:53:58.295 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.bloom_locality: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.max_successive_merges: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.ttl: 2592000 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.enable_blob_files: false 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.min_blob_size: 0 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T19:53:58.296 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.blob_file_starting_level: 0 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 839b1ede-a3b1-4a83-8a70-3f3e4694d98e 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716234838248985, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716234838249408, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1648, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716234838, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "839b1ede-a3b1-4a83-8a70-3f3e4694d98e", "db_session_id": "YAHEJZHGS28VKTVEDEVZ", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-05-20T19:53:58.297 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716234838249499, "job": 1, "event": "recovery_finished"} 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56041d756000 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: DB pointer 0x56041d742000 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: ** DB Stats ** 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T19:53:58.298 INFO:journalctl@ceph.mon.c.smithi154.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: ** Compaction Stats [default] ** 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: L0 1/0 1.61 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: Sum 1/0 1.61 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: ** Compaction Stats [default] ** 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T19:53:58.299 INFO:journalctl@ceph.mon.c.smithi154.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Block cache BinnedLRUCache@0x56041d6391f0#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-05-20T19:53:58.300 INFO:journalctl@ceph.mon.c.smithi154.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout: 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: using public_addr v2:172.21.15.154:0/0 -> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: starting mon.c rank -1 at public addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] at bind addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon_data /var/lib/ceph/mon/ceph-c fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(???) e0 preinit fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).mds e1 new map 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).mds e1 print_map 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout: e1 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout: legacy client fscid: -1 2024-05-20T19:53:58.301 INFO:journalctl@ceph.mon.c.smithi154.stdout: 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout: No filesystems configured 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.302 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Added host smithi033 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.303 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi033", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi154", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Deploying cephadm binary to smithi154 2024-05-20T19:53:58.304 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Added host smithi154 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.305 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi154", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3792071918' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3792071918' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-20T19:53:58.306 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi028:172.21.15.28=a;smithi033:172.21.15.33=b;smithi154:172.21.15.154=c", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Saving service mon spec with placement smithi028:172.21.15.28=a;smithi033:172.21.15.33=b;smithi154:172.21.15.154=c;count:3 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:53:58.307 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: Deploying daemon mon.c on smithi154 2024-05-20T19:53:58.308 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.308 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:53:58.308 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/3803290314' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T19:53:58.308 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 ceph-mon[36314]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-05-20T19:53:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 bash[36295]: 7216247a8a467d2978d24ab6059ba653b89fb4da236f1cb9b791d9b98a07d077 2024-05-20T19:53:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:53:58 smithi154 systemd[1]: Started Ceph mon.c for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:53:58.848 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-20T19:53:58.849 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T19:53:59.084 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:54:03.496 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.494+0000 7f1a7e61f700 1 -- 172.21.15.154:0/3902136829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a78104250 msgr2=0x7f1a60005610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:03.496 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.494+0000 7f1a7e61f700 1 --2- 172.21.15.154:0/3902136829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a78104250 0x7f1a60005610 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c005fa0 tx=0x7f1a6c00ffc0 comp rx=0 tx=0).stop 2024-05-20T19:54:03.496 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.495+0000 7f1a7e61f700 1 -- 172.21.15.154:0/3902136829 shutdown_connections 2024-05-20T19:54:03.496 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.495+0000 7f1a7e61f700 1 --2- 172.21.15.154:0/3902136829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a78104250 0x7f1a60005610 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:03.496 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.495+0000 7f1a7e61f700 1 -- 172.21.15.154:0/3902136829 >> 172.21.15.154:0/3902136829 conn(0x7f1a780fda20 msgr2=0x7f1a780ffe40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:03.496 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.495+0000 7f1a7e61f700 1 -- 172.21.15.154:0/3902136829 shutdown_connections 2024-05-20T19:54:03.497 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.495+0000 7f1a7e61f700 1 -- 172.21.15.154:0/3902136829 wait complete. 2024-05-20T19:54:03.497 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a7e61f700 1 Processor -- start 2024-05-20T19:54:03.497 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a7e61f700 1 -- start start 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a7e61f700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 0x7f1a78101eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a7e61f700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a781023f0 0x7f1a78102870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a7e61f700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1a7810af70 con 0x7f1a781023f0 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a7e61f700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1a7810adf0 con 0x7f1a78104250 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a77fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 0x7f1a78101eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.496+0000 7f1a77fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 0x7f1a78101eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:41468/0 (socket says 172.21.15.154:41468) 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a77fff700 1 -- 172.21.15.154:0/1318537214 learned_addr learned my addr 172.21.15.154:0/1318537214 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a77fff700 1 -- 172.21.15.154:0/1318537214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 msgr2=0x7f1a78101eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a77fff700 1 -- 172.21.15.154:0/1318537214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 msgr2=0x7f1a78101eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a77fff700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 0x7f1a78101eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-05-20T19:54:03.499 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a77fff700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 0x7f1a78101eb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a777fe700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a781023f0 0x7f1a78102870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a777fe700 1 -- 172.21.15.154:0/1318537214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 msgr2=0x7f1a78101eb0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a777fe700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 0x7f1a78101eb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.497+0000 7f1a777fe700 1 -- 172.21.15.154:0/1318537214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1a78103dd0 con 0x7f1a781023f0 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.498+0000 7f1a777fe700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a781023f0 0x7f1a78102870 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f1a6800b4d0 tx=0x7f1a6800b890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.498+0000 7f1a757fa700 1 -- 172.21.15.154:0/1318537214 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f1a68011630 con 0x7f1a781023f0 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.498+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1a781a4e30 con 0x7f1a781023f0 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.498+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1a781a5340 con 0x7f1a781023f0 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.498+0000 7f1a757fa700 1 -- 172.21.15.154:0/1318537214 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1a68011c80 con 0x7f1a781023f0 2024-05-20T19:54:03.500 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.498+0000 7f1a757fa700 1 -- 172.21.15.154:0/1318537214 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f1a68019c20 con 0x7f1a781023f0 2024-05-20T19:54:03.501 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.499+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1a7806b740 con 0x7f1a781023f0 2024-05-20T19:54:03.501 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.500+0000 7f1a757fa700 1 -- 172.21.15.154:0/1318537214 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f1a68018020 con 0x7f1a781023f0 2024-05-20T19:54:03.501 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.500+0000 7f1a757fa700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1a6403e2d0 0x7f1a64040790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:03.501 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.500+0000 7f1a757fa700 1 -- 172.21.15.154:0/1318537214 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f1a68051900 con 0x7f1a781023f0 2024-05-20T19:54:03.501 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.501+0000 7f1a77fff700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1a6403e2d0 0x7f1a64040790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:03.502 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.502+0000 7f1a77fff700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1a6403e2d0 0x7f1a64040790 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c00fb20 tx=0x7f1a6c01d040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:03.516 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.516+0000 7f1a757fa700 1 -- 172.21.15.154:0/1318537214 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f1a6800fcf0 con 0x7f1a781023f0 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: mon.a calling monitor election 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: mon.c calling monitor election 2024-05-20T19:54:03.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: monmap epoch 2 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: last_changed 2024-05-20T19:53:58.282529+0000 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: min_mon_release 19 (squid) 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: election_strategy: 1 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: fsmap 2024-05-20T19:54:03.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:03.628 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: mgrmap e13: x(active, since 60s) 2024-05-20T19:54:03.628 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: overall HEALTH_OK 2024-05-20T19:54:03.628 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:03.628 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: mon.a calling monitor election 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: mon.c calling monitor election 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: monmap epoch 2 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: last_changed 2024-05-20T19:53:58.282529+0000 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: min_mon_release 19 (squid) 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: election_strategy: 1 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: fsmap 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: mgrmap e13: x(active, since 60s) 2024-05-20T19:54:03.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: overall HEALTH_OK 2024-05-20T19:54:03.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:03.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:03.807 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.806+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7f1a7802e560 con 0x7f1a781023f0 2024-05-20T19:54:03.807 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.807+0000 7f1a757fa700 1 -- 172.21.15.154:0/1318537214 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) ==== 95+0+1015 (secure 0 0 0) 0x7f1a6800c150 con 0x7f1a781023f0 2024-05-20T19:54:03.808 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:54:03.808 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":2,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","modified":"2024-05-20T19:53:58.282529Z","created":"2024-05-20T19:51:39.025045Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-20T19:54:03.809 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1a6403e2d0 msgr2=0x7f1a64040790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:03.809 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1a6403e2d0 0x7f1a64040790 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f1a6c00fb20 tx=0x7f1a6c01d040 comp rx=0 tx=0).stop 2024-05-20T19:54:03.809 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a781023f0 msgr2=0x7f1a78102870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a781023f0 0x7f1a78102870 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f1a6800b4d0 tx=0x7f1a6800b890 comp rx=0 tx=0).stop 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 shutdown_connections 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1a6403e2d0 0x7f1a64040790 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a78104250 0x7f1a78101eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 --2- 172.21.15.154:0/1318537214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a781023f0 0x7f1a78102870 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 >> 172.21.15.154:0/1318537214 conn(0x7f1a780fda20 msgr2=0x7f1a780ff900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 shutdown_connections 2024-05-20T19:54:03.810 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:03.809+0000 7f1a7e61f700 1 -- 172.21.15.154:0/1318537214 wait complete. 2024-05-20T19:54:03.811 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 2 2024-05-20T19:54:04.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:04 smithi028 ceph-mon[31178]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:04.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:04.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:04.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:04.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:04 smithi028 ceph-mon[31178]: Deploying daemon mon.b on smithi033 2024-05-20T19:54:04.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:04 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/1318537214' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T19:54:04.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:04.673 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:04 smithi154 ceph-mon[36314]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:04.673 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:04.673 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:04.673 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:04.673 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:04 smithi154 ceph-mon[36314]: Deploying daemon mon.b on smithi033 2024-05-20T19:54:04.673 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:04 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/1318537214' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T19:54:04.673 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:05.331 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-05-20T19:54:05.332 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T19:54:05.565 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:54:06.666 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:06 smithi154 ceph-mon[36314]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:06.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:06 smithi028 ceph-mon[31178]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:07.173 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 -- 172.21.15.154:0/1909404600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 msgr2=0x7f81d4102580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:07.173 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 --2- 172.21.15.154:0/1909404600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 0x7f81d4102580 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f81cc00b0a0 tx=0x7f81cc00b3b0 comp rx=0 tx=0).stop 2024-05-20T19:54:07.173 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 -- 172.21.15.154:0/1909404600 shutdown_connections 2024-05-20T19:54:07.174 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 --2- 172.21.15.154:0/1909404600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 0x7f81d4102580 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:07.174 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 -- 172.21.15.154:0/1909404600 >> 172.21.15.154:0/1909404600 conn(0x7f81d40fda30 msgr2=0x7f81d40ffe50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:07.174 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 -- 172.21.15.154:0/1909404600 shutdown_connections 2024-05-20T19:54:07.174 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 -- 172.21.15.154:0/1909404600 wait complete. 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.173+0000 7f81d9e87700 1 Processor -- start 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d9e87700 1 -- start start 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d9e87700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 0x7f81d410af00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d9e87700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 0x7f81d410b8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d9e87700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f81d410d520 con 0x7f81d41021a0 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d9e87700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f81d410d3a0 con 0x7f81d410b440 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d2ffd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 0x7f81d410b8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d2ffd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 0x7f81d410b8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:54030/0 (socket says 172.21.15.154:54030) 2024-05-20T19:54:07.175 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d2ffd700 1 -- 172.21.15.154:0/966467732 learned_addr learned my addr 172.21.15.154:0/966467732 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:54:07.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d2ffd700 1 -- 172.21.15.154:0/966467732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 msgr2=0x7f81d410b8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 13 2024-05-20T19:54:07.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d2ffd700 1 -- 172.21.15.154:0/966467732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 msgr2=0x7f81d410b8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-05-20T19:54:07.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d2ffd700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 0x7f81d410b8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-05-20T19:54:07.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d2ffd700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 0x7f81d410b8c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T19:54:07.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.174+0000 7f81d37fe700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 0x7f81d410af00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.175+0000 7f81d37fe700 1 -- 172.21.15.154:0/966467732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 msgr2=0x7f81d410b8c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.175+0000 7f81d37fe700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 0x7f81d410b8c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.175+0000 7f81d37fe700 1 -- 172.21.15.154:0/966467732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f81d410ce20 con 0x7f81d41021a0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.175+0000 7f81d37fe700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 0x7f81d410af00 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f81cc0062a0 tx=0x7f81cc007a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.176+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f81cc017040 con 0x7f81d41021a0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.176+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f81d41a4e40 con 0x7f81d41021a0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.176+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f81d41a5320 con 0x7f81d41021a0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.176+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f81cc0128e0 con 0x7f81d41021a0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.176+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 309+0+0 (secure 0 0 0) 0x7f81cc01d930 con 0x7f81d41021a0 2024-05-20T19:54:07.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.177+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f81c0005330 con 0x7f81d41021a0 2024-05-20T19:54:07.178 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.178+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f81cc024020 con 0x7f81d41021a0 2024-05-20T19:54:07.178 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.178+0000 7f81d0ff9700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81bc03e2b0 0x7f81bc040770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:07.178 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.178+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f81cc00f050 con 0x7f81d41021a0 2024-05-20T19:54:07.179 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.178+0000 7f81d2ffd700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81bc03e2b0 0x7f81bc040770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:07.179 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.179+0000 7f81d2ffd700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81bc03e2b0 0x7f81bc040770 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f81c8009180 tx=0x7f81c8008040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:07.194 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.194+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f81cc02b070 con 0x7f81d41021a0 2024-05-20T19:54:07.270 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.269+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f81cc025b50 con 0x7f81d41021a0 2024-05-20T19:54:07.478 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:07.477+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) -- 0x7f81c00058f0 con 0x7f81d41021a0 2024-05-20T19:54:07.607 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:07 smithi033 systemd[1]: Started Ceph mon.b for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:54:12.289 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.289+0000 7f81d0ff9700 1 -- 172.21.15.154:0/966467732 <== mon.0 v2:172.21.15.28:3300/0 8 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 3 v3) ==== 95+0+1279 (secure 0 0 0) 0x7f81cc033510 con 0x7f81d41021a0 2024-05-20T19:54:12.290 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-05-20T19:54:12.290 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":3,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","modified":"2024-05-20T19:54:07.267421Z","created":"2024-05-20T19:51:39.025045Z","min_mon_release":19,"min_mon_release_name":"squid","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef","squid"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:3300","nonce":0},{"type":"v1","addr":"172.21.15.28:6789","nonce":0}]},"addr":"172.21.15.28:6789/0","public_addr":"172.21.15.28:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-20T19:54:12.292 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81bc03e2b0 msgr2=0x7f81bc040770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81bc03e2b0 0x7f81bc040770 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f81c8009180 tx=0x7f81c8008040 comp rx=0 tx=0).stop 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 msgr2=0x7f81d410af00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 0x7f81d410af00 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f81cc0062a0 tx=0x7f81cc007a50 comp rx=0 tx=0).stop 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 shutdown_connections 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81bc03e2b0 0x7f81bc040770 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f81d41021a0 0x7f81d410af00 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 --2- 172.21.15.154:0/966467732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f81d410b440 0x7f81d410b8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 >> 172.21.15.154:0/966467732 conn(0x7f81d40fda30 msgr2=0x7f81d4103c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 shutdown_connections 2024-05-20T19:54:12.293 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:12.292+0000 7f81d9e87700 1 -- 172.21.15.154:0/966467732 wait complete. 2024-05-20T19:54:12.294 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 3 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: mon.c calling monitor election 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: mon.a calling monitor election 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/966467732' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:12.576 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: monmap epoch 3 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: last_changed 2024-05-20T19:54:07.267421+0000 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: min_mon_release 19 (squid) 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: election_strategy: 1 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:12.577 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: 2: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.b 2024-05-20T19:54:12.578 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: fsmap 2024-05-20T19:54:12.578 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:12.578 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: mgrmap e13: x(active, since 69s) 2024-05-20T19:54:12.578 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: overall HEALTH_OK 2024-05-20T19:54:12.578 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:12.578 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:12.578 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:12.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: mon.c calling monitor election 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: mon.a calling monitor election 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/966467732' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: monmap epoch 3 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: last_changed 2024-05-20T19:54:07.267421+0000 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: min_mon_release 19 (squid) 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: election_strategy: 1 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: 2: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.b 2024-05-20T19:54:12.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: fsmap 2024-05-20T19:54:12.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:12.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: mgrmap e13: x(active, since 69s) 2024-05-20T19:54:12.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: overall HEALTH_OK 2024-05-20T19:54:12.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:12.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:12.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:12.962 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-20T19:54:12.962 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph config generate-minimal-conf 2024-05-20T19:54:13.411 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:54:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:13.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:13.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:13.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:13.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:13.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:13.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:14.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.c calling monitor election 2024-05-20T19:54:14.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:54:14.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.a calling monitor election 2024-05-20T19:54:14.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/966467732' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: monmap epoch 3 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:14.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: last_changed 2024-05-20T19:54:07.267421+0000 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: min_mon_release 19 (squid) 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: election_strategy: 1 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: 2: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.b 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: fsmap 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mgrmap e13: x(active, since 69s) 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: overall HEALTH_OK 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.631 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.631 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:14.631 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:14.631 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: mon.b calling monitor election 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: Updating smithi028:/etc/ceph/ceph.conf 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: mon.b calling monitor election 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: mon.c calling monitor election 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: mon.a calling monitor election 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: monmap epoch 3 2024-05-20T19:54:14.988 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: last_changed 2024-05-20T19:54:07.267421+0000 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: min_mon_release 19 (squid) 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: election_strategy: 1 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: 2: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.b 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: fsmap 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: mgrmap e13: x(active, since 71s) 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: overall HEALTH_OK 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.989 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.990 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.990 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:14.990 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:14 smithi028 ceph-mon[31178]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:15.209 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: mon.b calling monitor election 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: Updating smithi028:/etc/ceph/ceph.conf 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: mon.b calling monitor election 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: mon.c calling monitor election 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: mon.a calling monitor election 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: monmap epoch 3 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:15.210 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: last_changed 2024-05-20T19:54:07.267421+0000 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: min_mon_release 19 (squid) 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: election_strategy: 1 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: 2: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.b 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: fsmap 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: mgrmap e13: x(active, since 71s) 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: overall HEALTH_OK 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.211 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.212 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.212 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:14 smithi154 ceph-mon[36314]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:15.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.b calling monitor election 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: Updating smithi028:/etc/ceph/ceph.conf 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.b calling monitor election 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.c calling monitor election 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.a calling monitor election 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-05-20T19:54:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: monmap epoch 3 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: last_changed 2024-05-20T19:54:07.267421+0000 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: created 2024-05-20T19:51:39.025045+0000 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: min_mon_release 19 (squid) 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: election_strategy: 1 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: 0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: 1: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.c 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: 2: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.b 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: fsmap 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: mgrmap e13: x(active, since 71s) 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: overall HEALTH_OK 2024-05-20T19:54:15.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.380 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.380 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.380 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:15.380 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:14 smithi033 ceph-mon[36179]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T19:54:15.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.484+0000 7fd00b200700 1 -- 172.21.15.28:0/3299887960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 msgr2=0x7fd004103590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:15.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.484+0000 7fd00b200700 1 --2- 172.21.15.28:0/3299887960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd004103590 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fcff4009910 tx=0x7fcff4009c20 comp rx=0 tx=0).stop 2024-05-20T19:54:15.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.484+0000 7fd00b200700 1 -- 172.21.15.28:0/3299887960 shutdown_connections 2024-05-20T19:54:15.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.484+0000 7fd00b200700 1 --2- 172.21.15.28:0/3299887960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd004103590 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.484+0000 7fd00b200700 1 -- 172.21.15.28:0/3299887960 >> 172.21.15.28:0/3299887960 conn(0x7fd0040fea80 msgr2=0x7fd004100ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:15.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.484+0000 7fd00b200700 1 -- 172.21.15.28:0/3299887960 shutdown_connections 2024-05-20T19:54:15.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.484+0000 7fd00b200700 1 -- 172.21.15.28:0/3299887960 wait complete. 2024-05-20T19:54:15.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.485+0000 7fd00b200700 1 Processor -- start 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.485+0000 7fd00b200700 1 -- start start 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd00b200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd00419ae80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd00b200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd00419b3c0 0x7fd00419f750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd00b200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd00419fc90 0x7fd0041a0140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd00b200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd00410d430 con 0x7fd0041031b0 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd00b200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd00410d2b0 con 0x7fd00419fc90 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd00b200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd00410d5b0 con 0x7fd00419b3c0 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd003e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd00419ae80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd003e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd00419ae80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51972/0 (socket says 172.21.15.28:51972) 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd003e00700 1 -- 172.21.15.28:0/1773116956 learned_addr learned my addr 172.21.15.28:0/1773116956 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd009200700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd00419fc90 0x7fd0041a0140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:15.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.486+0000 7fd003400700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd00419b3c0 0x7fd00419f750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:15.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd003e00700 1 -- 172.21.15.28:0/1773116956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd00419b3c0 msgr2=0x7fd00419f750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:15.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd003e00700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd00419b3c0 0x7fd00419f750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd003e00700 1 -- 172.21.15.28:0/1773116956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd00419fc90 msgr2=0x7fd0041a0140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:15.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd003e00700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd00419fc90 0x7fd0041a0140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd003e00700 1 -- 172.21.15.28:0/1773116956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd0041a7a00 con 0x7fd0041031b0 2024-05-20T19:54:15.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd003e00700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd00419ae80 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fcff4004f80 tx=0x7fcff4004fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:15.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd000c00700 1 -- 172.21.15.28:0/1773116956 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcff4026070 con 0x7fd0041031b0 2024-05-20T19:54:15.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd000c00700 1 -- 172.21.15.28:0/1773116956 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcff400b6f0 con 0x7fd0041031b0 2024-05-20T19:54:15.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd000c00700 1 -- 172.21.15.28:0/1773116956 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcff4009e60 con 0x7fd0041031b0 2024-05-20T19:54:15.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd00b200700 1 -- 172.21.15.28:0/1773116956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd0041a7c30 con 0x7fd0041031b0 2024-05-20T19:54:15.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.487+0000 7fd00b200700 1 -- 172.21.15.28:0/1773116956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd0041a8120 con 0x7fd0041031b0 2024-05-20T19:54:15.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.489+0000 7fd000c00700 1 -- 172.21.15.28:0/1773116956 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7fcff4029070 con 0x7fd0041031b0 2024-05-20T19:54:15.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.489+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd004064ba0 con 0x7fd0041031b0 2024-05-20T19:54:15.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.489+0000 7fd000c00700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcfe803e9a0 0x7fcfe8040e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:15.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.490+0000 7fd000c00700 1 -- 172.21.15.28:0/1773116956 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7fcff40531f0 con 0x7fd0041031b0 2024-05-20T19:54:15.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.490+0000 7fd003400700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcfe803e9a0 0x7fcfe8040e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:15.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.490+0000 7fd003400700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcfe803e9a0 0x7fcfe8040e60 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fcff0009ad0 tx=0x7fcff0009210 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:15.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.496+0000 7fd000c00700 1 -- 172.21.15.28:0/1773116956 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fcff4013070 con 0x7fd0041031b0 2024-05-20T19:54:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.738+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) -- 0x7fd004064150 con 0x7fd0041031b0 2024-05-20T19:54:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.738+0000 7fd000c00700 1 -- 172.21.15.28:0/1773116956 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v9) ==== 76+0+273 (secure 0 0 0) 0x7fcff4030130 con 0x7fd0041031b0 2024-05-20T19:54:15.751 INFO:teuthology.orchestra.run.smithi028.stdout:# minimal ceph.conf for 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:15.751 INFO:teuthology.orchestra.run.smithi028.stdout:[global] 2024-05-20T19:54:15.751 INFO:teuthology.orchestra.run.smithi028.stdout: fsid = 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T19:54:15.751 INFO:teuthology.orchestra.run.smithi028.stdout: mon_host = [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcfe803e9a0 msgr2=0x7fcfe8040e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcfe803e9a0 0x7fcfe8040e60 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fcff0009ad0 tx=0x7fcff0009210 comp rx=0 tx=0).stop 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 msgr2=0x7fd00419ae80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd00419ae80 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fcff4004f80 tx=0x7fcff4004fb0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 shutdown_connections 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcfe803e9a0 0x7fcfe8040e60 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd00419fc90 0x7fd0041a0140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0041031b0 0x7fd00419ae80 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 --2- 172.21.15.28:0/1773116956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd00419b3c0 0x7fd00419f750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.741+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 >> 172.21.15.28:0/1773116956 conn(0x7fd0040fea80 msgr2=0x7fd004108d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.742+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 shutdown_connections 2024-05-20T19:54:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:15.742+0000 7fcffa000700 1 -- 172.21.15.28:0/1773116956 wait complete. 2024-05-20T19:54:16.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: Reconfiguring mon.a (unknown last config time)... 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:16.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:16.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: Reconfiguring daemon mon.a on smithi028 2024-05-20T19:54:16.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:16 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1773116956' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:16.512 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-20T19:54:16.513 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:54:16.513 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/ceph.conf 2024-05-20T19:54:16.602 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:54:16.602 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:54:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: Reconfiguring mon.a (unknown last config time)... 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: Reconfiguring daemon mon.a on smithi028 2024-05-20T19:54:16.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:16 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1773116956' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:54:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: Reconfiguring mon.a (unknown last config time)... 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: Reconfiguring daemon mon.a on smithi028 2024-05-20T19:54:16.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:16 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1773116956' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:16.736 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:54:16.736 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.conf 2024-05-20T19:54:16.769 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:54:16.769 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:54:16.838 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:54:16.838 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.conf 2024-05-20T19:54:16.870 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:54:16.870 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T19:54:16.938 INFO:tasks.cephadm:Adding mgr.x on smithi028 2024-05-20T19:54:16.938 INFO:tasks.cephadm:Adding mgr.y on smithi033 2024-05-20T19:54:16.938 INFO:tasks.cephadm:Adding mgr.z on smithi154 2024-05-20T19:54:16.939 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch apply mgr '3;smithi028=x;smithi033=y;smithi154=z' 2024-05-20T19:54:17.215 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:54:18.514 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:18 smithi154 ceph-mon[36314]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:18.514 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:18.514 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:18.515 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:18.515 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:18.515 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:18.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.542+0000 7f19749c3700 1 -- 172.21.15.154:0/4139736553 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1970101ff0 msgr2=0x7f1970102410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:18.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.542+0000 7f19749c3700 1 --2- 172.21.15.154:0/4139736553 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1970101ff0 0x7f1970102410 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f1960009910 tx=0x7f1960009c20 comp rx=0 tx=0).stop 2024-05-20T19:54:18.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.542+0000 7f19749c3700 1 -- 172.21.15.154:0/4139736553 shutdown_connections 2024-05-20T19:54:18.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.542+0000 7f19749c3700 1 --2- 172.21.15.154:0/4139736553 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1970101ff0 0x7f1970102410 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:18.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.542+0000 7f19749c3700 1 -- 172.21.15.154:0/4139736553 >> 172.21.15.154:0/4139736553 conn(0x7f19700fd5b0 msgr2=0x7f19700ff9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:18.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.542+0000 7f19749c3700 1 -- 172.21.15.154:0/4139736553 shutdown_connections 2024-05-20T19:54:18.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.542+0000 7f19749c3700 1 -- 172.21.15.154:0/4139736553 wait complete. 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 Processor -- start 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 -- start start 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1970101ff0 0x7f19700ff500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19700fdb50 0x7f19700fdfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19700fe510 0x7f19701a28f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1970107f30 con 0x7f1970101ff0 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1970107db0 con 0x7f19700fe510 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.543+0000 7f19749c3700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f19701080b0 con 0x7f19700fdb50 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19700fdb50 0x7f19700fdfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19700fdb50 0x7f19700fdfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:34530/0 (socket says 172.21.15.154:34530) 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 -- 172.21.15.154:0/3576215860 learned_addr learned my addr 172.21.15.154:0/3576215860 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f196ed9d700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19700fe510 0x7f19701a28f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:18.547 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 -- 172.21.15.154:0/3576215860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19700fe510 msgr2=0x7f19701a28f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19700fe510 0x7f19701a28f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 -- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1970101ff0 msgr2=0x7f19700ff500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1970101ff0 0x7f19700ff500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 -- 172.21.15.154:0/3576215860 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f19701a2eb0 con 0x7f19700fdb50 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f196ed9d700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19700fe510 0x7f19701a28f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f1967fff700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19700fdb50 0x7f19700fdfd0 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f195800b780 tx=0x7f195800bb40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f19677fe700 1 -- 172.21.15.154:0/3576215860 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1958004230 con 0x7f19700fdb50 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f19677fe700 1 -- 172.21.15.154:0/3576215860 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f19580043d0 con 0x7f19700fdb50 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f19677fe700 1 -- 172.21.15.154:0/3576215860 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1958011bd0 con 0x7f19700fdb50 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f19701a31a0 con 0x7f19700fdb50 2024-05-20T19:54:18.548 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.544+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f19701a3710 con 0x7f19700fdb50 2024-05-20T19:54:18.549 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.545+0000 7f19677fe700 1 -- 172.21.15.154:0/3576215860 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f195801a400 con 0x7f19700fdb50 2024-05-20T19:54:18.549 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.545+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1970064ba0 con 0x7f19700fdb50 2024-05-20T19:54:18.549 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.545+0000 7f19677fe700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f195c03e9a0 0x7f195c040e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:18.549 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.545+0000 7f19677fe700 1 -- 172.21.15.154:0/3576215860 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f19580523c0 con 0x7f19700fdb50 2024-05-20T19:54:18.550 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.546+0000 7f196e59c700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f195c03e9a0 0x7f195c040e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:18.550 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.547+0000 7f196e59c700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f195c03e9a0 0x7f195c040e60 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f1960005e00 tx=0x7f1960005260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:18.552 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.551+0000 7f19677fe700 1 -- 172.21.15.154:0/3576215860 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f1958015df0 con 0x7f19700fdb50 2024-05-20T19:54:18.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:18 smithi033 ceph-mon[36179]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:18.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:18.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:18.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:18.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:18.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:18.767 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.766+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi028=x;smithi033=y;smithi154=z", "target": ["mon-mgr", ""]}) -- 0x7f1970063a10 con 0x7f195c03e9a0 2024-05-20T19:54:18.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.780+0000 7f19677fe700 1 -- 172.21.15.154:0/3576215860 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f1970063a10 con 0x7f195c03e9a0 2024-05-20T19:54:18.782 INFO:teuthology.orchestra.run.smithi154.stdout:Scheduled mgr update... 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f195c03e9a0 msgr2=0x7f195c040e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f195c03e9a0 0x7f195c040e60 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f1960005e00 tx=0x7f1960005260 comp rx=0 tx=0).stop 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19700fdb50 msgr2=0x7f19700fdfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19700fdb50 0x7f19700fdfd0 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f195800b780 tx=0x7f195800bb40 comp rx=0 tx=0).stop 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 shutdown_connections 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f195c03e9a0 0x7f195c040e60 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19700fe510 0x7f19701a28f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1970101ff0 0x7f19700ff500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 --2- 172.21.15.154:0/3576215860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19700fdb50 0x7f19700fdfd0 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 >> 172.21.15.154:0/3576215860 conn(0x7f19700fd5b0 msgr2=0x7f197006bcb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:18.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 shutdown_connections 2024-05-20T19:54:18.785 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:54:18.782+0000 7f19749c3700 1 -- 172.21.15.154:0/3576215860 wait complete. 2024-05-20T19:54:18.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:18 smithi028 ceph-mon[31178]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:18.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:18.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:18.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:18.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:18.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:19.351 DEBUG:teuthology.orchestra.run.smithi033:mgr.y> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.y.service 2024-05-20T19:54:19.354 DEBUG:teuthology.orchestra.run.smithi154:mgr.z> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.z.service 2024-05-20T19:54:19.357 INFO:tasks.cephadm:Deploying OSDs... 2024-05-20T19:54:19.357 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T19:54:19.357 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/scratch_devs of=/dev/stdout 2024-05-20T19:54:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:19 smithi033 ceph-mon[36179]: Reconfiguring mon.b (monmap changed)... 2024-05-20T19:54:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:19 smithi033 ceph-mon[36179]: Reconfiguring daemon mon.b on smithi033 2024-05-20T19:54:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:19.414 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-20T19:54:19.414 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_1 2024-05-20T19:54:19.468 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-20T19:54:19.468 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:19.469 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 6h/6d Inode: 631 Links: 1 2024-05-20T19:54:19.469 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:19.469 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:19.469 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-05-20 19:53:19.694957332 +0000 2024-05-20T19:54:19.469 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-05-20 19:47:17.825308645 +0000 2024-05-20T19:54:19.469 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-05-20 19:47:17.825308645 +0000 2024-05-20T19:54:19.469 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-05-20 19:47:17.825308645 +0000 2024-05-20T19:54:19.469 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-20T19:54:19.550 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-05-20T19:54:19.550 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-05-20T19:54:19.551 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000278902 s, 1.8 MB/s 2024-05-20T19:54:19.553 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-20T19:54:19.607 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_2 2024-05-20T19:54:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:19 smithi154 ceph-mon[36314]: Reconfiguring mon.b (monmap changed)... 2024-05-20T19:54:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:19 smithi154 ceph-mon[36314]: Reconfiguring daemon mon.b on smithi033 2024-05-20T19:54:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:19.659 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-20T19:54:19.659 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:19.659 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 6h/6d Inode: 653 Links: 1 2024-05-20T19:54:19.660 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:19.660 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:19.660 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-05-20 19:53:19.771955232 +0000 2024-05-20T19:54:19.660 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-05-20 19:47:18.474291333 +0000 2024-05-20T19:54:19.660 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-05-20 19:47:18.474291333 +0000 2024-05-20T19:54:19.660 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-05-20 19:47:18.474291333 +0000 2024-05-20T19:54:19.660 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-20T19:54:19.735 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:19 smithi028 ceph-mon[31178]: Reconfiguring mon.b (monmap changed)... 2024-05-20T19:54:19.735 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:19 smithi028 ceph-mon[31178]: Reconfiguring daemon mon.b on smithi033 2024-05-20T19:54:19.735 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:19.745 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-05-20T19:54:19.745 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-05-20T19:54:19.745 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000324428 s, 1.6 MB/s 2024-05-20T19:54:19.749 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-20T19:54:19.808 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_3 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 6h/6d Inode: 671 Links: 1 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-05-20 19:53:19.823953814 +0000 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-05-20 19:47:19.072275381 +0000 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-05-20 19:47:19.072275381 +0000 2024-05-20T19:54:19.860 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-05-20 19:47:19.072275381 +0000 2024-05-20T19:54:19.861 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-20T19:54:19.943 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-05-20T19:54:19.943 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-05-20T19:54:19.944 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000290652 s, 1.8 MB/s 2024-05-20T19:54:19.947 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-20T19:54:20.006 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_4 2024-05-20T19:54:20.060 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 6h/6d Inode: 691 Links: 1 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-05-20 19:53:19.875952396 +0000 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-05-20 19:47:19.655259829 +0000 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-05-20 19:47:19.655259829 +0000 2024-05-20T19:54:20.061 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-05-20 19:47:19.655259829 +0000 2024-05-20T19:54:20.062 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-20T19:54:20.145 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-05-20T19:54:20.145 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-05-20T19:54:20.145 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000278209 s, 1.8 MB/s 2024-05-20T19:54:20.149 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-20T19:54:20.207 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-05-20T19:54:20.207 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/scratch_devs of=/dev/stdout 2024-05-20T19:54:20.228 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-20T19:54:20.228 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_1 2024-05-20T19:54:20.287 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-20T19:54:20.287 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:20.287 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-05-20T19:54:20.287 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:20.287 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:20.287 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-05-20 19:54:06.704652923 +0000 2024-05-20T19:54:20.287 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-05-20 19:46:10.410254149 +0000 2024-05-20T19:54:20.288 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-05-20 19:46:10.410254149 +0000 2024-05-20T19:54:20.288 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-05-20 19:46:10.410254149 +0000 2024-05-20T19:54:20.288 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-20T19:54:20.355 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-05-20T19:54:20.355 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-05-20T19:54:20.355 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000302166 s, 1.7 MB/s 2024-05-20T19:54:20.356 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-20T19:54:20.415 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_2 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-05-20 19:54:06.704652923 +0000 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-05-20 19:46:10.748248750 +0000 2024-05-20T19:54:20.473 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-05-20 19:46:10.748248750 +0000 2024-05-20T19:54:20.474 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-05-20 19:46:10.748248750 +0000 2024-05-20T19:54:20.474 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-20T19:54:20.538 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-05-20T19:54:20.539 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-05-20T19:54:20.539 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000271388 s, 1.9 MB/s 2024-05-20T19:54:20.540 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-20T19:54:20.597 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_3 2024-05-20T19:54:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi028=x;smithi033=y;smithi154=z", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:54:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: Saving service mgr spec with placement smithi028=x;smithi033=y;smithi154=z;count:3 2024-05-20T19:54:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: Reconfiguring mon.c (monmap changed)... 2024-05-20T19:54:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:20 smithi154 ceph-mon[36314]: Reconfiguring daemon mon.c on smithi154 2024-05-20T19:54:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi028=x;smithi033=y;smithi154=z", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:54:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: Saving service mgr spec with placement smithi028=x;smithi033=y;smithi154=z;count:3 2024-05-20T19:54:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:20.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:20.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:20.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: Reconfiguring mon.c (monmap changed)... 2024-05-20T19:54:20.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:20.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:20.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:20.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:20 smithi033 ceph-mon[36179]: Reconfiguring daemon mon.c on smithi154 2024-05-20T19:54:20.643 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-20T19:54:20.643 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:20.644 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-05-20T19:54:20.644 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:20.644 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:20.644 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-05-20 19:54:06.704652923 +0000 2024-05-20T19:54:20.644 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-05-20 19:46:11.130242647 +0000 2024-05-20T19:54:20.644 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-05-20 19:46:11.130242647 +0000 2024-05-20T19:54:20.644 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-05-20 19:46:11.130242647 +0000 2024-05-20T19:54:20.644 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-20T19:54:20.708 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-05-20T19:54:20.708 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-05-20T19:54:20.708 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000195747 s, 2.6 MB/s 2024-05-20T19:54:20.709 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-20T19:54:20.767 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_4 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi028=x;smithi033=y;smithi154=z", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: Saving service mgr spec with placement smithi028=x;smithi033=y;smithi154=z;count:3 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: Reconfiguring mon.c (monmap changed)... 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T19:54:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T19:54:20.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:20.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:20 smithi028 ceph-mon[31178]: Reconfiguring daemon mon.c on smithi154 2024-05-20T19:54:20.823 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-05-20 19:54:06.705652907 +0000 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-05-20 19:46:11.470237215 +0000 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-05-20 19:46:11.470237215 +0000 2024-05-20T19:54:20.824 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-05-20 19:46:11.470237215 +0000 2024-05-20T19:54:20.824 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-20T19:54:20.888 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-05-20T19:54:20.888 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-05-20T19:54:20.888 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000257673 s, 2.0 MB/s 2024-05-20T19:54:20.889 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-20T19:54:20.947 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-05-20T19:54:20.947 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/scratch_devs of=/dev/stdout 2024-05-20T19:54:20.964 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-20T19:54:20.964 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_1 2024-05-20T19:54:21.019 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-20T19:54:21.019 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:21.019 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-05-20T19:54:21.019 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:21.020 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:21.020 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-05-20 19:53:57.299460191 +0000 2024-05-20T19:54:21.020 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-05-20 19:45:57.474779438 +0000 2024-05-20T19:54:21.020 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-05-20 19:45:57.474779438 +0000 2024-05-20T19:54:21.020 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-05-20 19:45:57.474779438 +0000 2024-05-20T19:54:21.020 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-20T19:54:21.082 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-05-20T19:54:21.082 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-05-20T19:54:21.082 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000214682 s, 2.4 MB/s 2024-05-20T19:54:21.083 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-20T19:54:21.139 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_2 2024-05-20T19:54:21.194 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-20T19:54:21.194 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:21.194 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-05-20T19:54:21.195 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:21.195 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:21.195 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-05-20 19:53:57.299460191 +0000 2024-05-20T19:54:21.195 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-05-20 19:45:57.820770372 +0000 2024-05-20T19:54:21.195 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-05-20 19:45:57.820770372 +0000 2024-05-20T19:54:21.195 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-05-20 19:45:57.820770372 +0000 2024-05-20T19:54:21.195 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-20T19:54:21.257 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-05-20T19:54:21.257 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-05-20T19:54:21.257 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000204915 s, 2.5 MB/s 2024-05-20T19:54:21.258 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-20T19:54:21.313 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_3 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-05-20 19:53:57.300460165 +0000 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-05-20 19:45:58.155761596 +0000 2024-05-20T19:54:21.369 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-05-20 19:45:58.155761596 +0000 2024-05-20T19:54:21.370 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-05-20 19:45:58.155761596 +0000 2024-05-20T19:54:21.370 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-20T19:54:21.431 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-05-20T19:54:21.431 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-05-20T19:54:21.432 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000193496 s, 2.6 MB/s 2024-05-20T19:54:21.433 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-20T19:54:21.490 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_4 2024-05-20T19:54:21.545 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-20T19:54:21.545 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T19:54:21.545 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-05-20T19:54:21.545 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T19:54:21.545 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T19:54:21.546 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-05-20 19:53:57.300460165 +0000 2024-05-20T19:54:21.546 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-05-20 19:45:58.486752923 +0000 2024-05-20T19:54:21.546 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-05-20 19:45:58.486752923 +0000 2024-05-20T19:54:21.546 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-05-20 19:45:58.486752923 +0000 2024-05-20T19:54:21.546 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-20T19:54:21.608 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-05-20T19:54:21.608 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-05-20T19:54:21.608 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000206812 s, 2.5 MB/s 2024-05-20T19:54:21.609 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-20T19:54:21.664 INFO:tasks.cephadm:Deploying osd.0 on smithi028 with /dev/vg_nvme/lv_4... 2024-05-20T19:54:21.665 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-20T19:54:22.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: Deploying daemon mgr.z on smithi154 2024-05-20T19:54:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:21 smithi028 ceph-mon[31178]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:22.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: Deploying daemon mgr.z on smithi154 2024-05-20T19:54:22.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:21 smithi154 ceph-mon[36314]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:22.130 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:54:22.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:22.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:22.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:22.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:22.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:22.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T19:54:22.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:22.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:22.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: Deploying daemon mgr.z on smithi154 2024-05-20T19:54:22.380 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:21 smithi033 ceph-mon[36179]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:23.126 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:22 smithi154 systemd[1]: Starting Ceph mgr.z for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T19:54:23.445 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 podman[38149]: 2024-05-20 19:54:23.150554546 +0000 UTC m=+0.095433667 container create 445f1d9e01f49f0f710389465fd7c2fd7370e3c5d42f00566b6abd0986a82887 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0) 2024-05-20T19:54:23.445 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 podman[38149]: 2024-05-20 19:54:23.072163559 +0000 UTC m=+0.017042682 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T19:54:23.445 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 podman[38149]: 2024-05-20 19:54:23.361843714 +0000 UTC m=+0.306722830 container init 445f1d9e01f49f0f710389465fd7c2fd7370e3c5d42f00566b6abd0986a82887 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-05-20T19:54:23.445 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 podman[38149]: 2024-05-20 19:54:23.364403051 +0000 UTC m=+0.309282167 container start 445f1d9e01f49f0f710389465fd7c2fd7370e3c5d42f00566b6abd0986a82887 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T19:54:23.446 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 ceph-mgr[38168]: -- 172.21.15.154:0/1117577915 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e4b281c5a0 con 0x55e4b27f1000 2024-05-20T19:54:23.446 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 bash[38149]: 445f1d9e01f49f0f710389465fd7c2fd7370e3c5d42f00566b6abd0986a82887 2024-05-20T19:54:23.702 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 systemd[1]: Started Ceph mgr.z for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:54:23.703 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:23.516+0000 7f95516a3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T19:54:23.703 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:23.702+0000 7f95516a3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:54:24.113 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:23.804+0000 7f95516a3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T19:54:24.113 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:24.112+0000 7f95516a3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T19:54:24.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:24.198+0000 7f95516a3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T19:54:24.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:24 smithi028 ceph-mon[31178]: Deploying daemon mgr.y on smithi033 2024-05-20T19:54:24.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:24.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:24.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T19:54:24.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:24.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:24.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:24 smithi154 ceph-mon[36314]: Deploying daemon mgr.y on smithi033 2024-05-20T19:54:24.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:24.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:24.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:24.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T19:54:24.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:24.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:24.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:24 smithi033 ceph-mon[36179]: Deploying daemon mgr.y on smithi033 2024-05-20T19:54:25.040 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-20T19:54:25.040 INFO:teuthology.orchestra.run.smithi028.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-20T19:54:25.040 INFO:teuthology.orchestra.run.smithi028.stderr: stderr: 10+0 records in 2024-05-20T19:54:25.040 INFO:teuthology.orchestra.run.smithi028.stderr:10+0 records out 2024-05-20T19:54:25.040 INFO:teuthology.orchestra.run.smithi028.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0393244 s, 267 MB/s 2024-05-20T19:54:25.040 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping successful for: 2024-05-20T19:54:25.627 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:25 smithi033 systemd[1]: Starting Ceph mgr.y for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T19:54:25.727 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:25.389+0000 7f95516a3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T19:54:25.727 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:25.556+0000 7f95516a3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T19:54:25.727 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:25.726+0000 7f95516a3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T19:54:25.789 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_4 2024-05-20T19:54:26.028 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:25 smithi033 podman[37541]: 2024-05-20 19:54:25.758772335 +0000 UTC m=+0.106578974 container create 466c25eca87c4317992dcd6655c7efef088a563e9349162f41d06022bc04bcf1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T19:54:26.028 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:25 smithi033 podman[37541]: 2024-05-20 19:54:25.669493522 +0000 UTC m=+0.017300181 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T19:54:26.028 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:25 smithi033 podman[37541]: 2024-05-20 19:54:25.944239956 +0000 UTC m=+0.292046602 container init 466c25eca87c4317992dcd6655c7efef088a563e9349162f41d06022bc04bcf1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-05-20T19:54:26.028 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:25 smithi033 podman[37541]: 2024-05-20 19:54:25.947627915 +0000 UTC m=+0.295434560 container start 466c25eca87c4317992dcd6655c7efef088a563e9349162f41d06022bc04bcf1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514) 2024-05-20T19:54:26.028 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:25 smithi033 ceph-mgr[37560]: -- 172.21.15.33:0/3342884919 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ca5d8ce5a0 con 0x55ca5d8a2c00 2024-05-20T19:54:26.028 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:26 smithi033 bash[37541]: 466c25eca87c4317992dcd6655c7efef088a563e9349162f41d06022bc04bcf1 2024-05-20T19:54:26.241 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:54:26.304 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:26 smithi033 systemd[1]: Started Ceph mgr.y for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T19:54:26.304 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:26.105+0000 7f8d4042a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T19:54:26.305 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:26.303+0000 7f8d4042a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:54:26.307 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:26.011+0000 7f95516a3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T19:54:26.307 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:26.102+0000 7f95516a3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T19:54:26.307 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:26.307+0000 7f95516a3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T19:54:26.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:26.510+0000 7f95516a3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:54:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:26 smithi154 ceph-mon[36314]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:26.627 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:26.416+0000 7f8d4042a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T19:54:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:26 smithi033 ceph-mon[36179]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:26.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:26.747 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:26 smithi028 ceph-mon[31178]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:26.748 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.748 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.748 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.748 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:26.748 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:27.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:26.760+0000 7f8d4042a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T19:54:27.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:26.852+0000 7f8d4042a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T19:54:27.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:27.140+0000 7f95516a3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T19:54:27.626 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:27.367+0000 7f95516a3200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T19:54:28.125 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:27.793+0000 7f95516a3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T19:54:28.125 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:27.889+0000 7f95516a3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T19:54:28.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 -- 172.21.15.28:0/1495785782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8077570 msgr2=0x7f7ad80759d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:28.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 --2- 172.21.15.28:0/1495785782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8077570 0x7f7ad80759d0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f7ac0009990 tx=0x7f7ac0009ca0 comp rx=0 tx=0).stop 2024-05-20T19:54:28.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 -- 172.21.15.28:0/1495785782 shutdown_connections 2024-05-20T19:54:28.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 --2- 172.21.15.28:0/1495785782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7ad810fb90 0x7f7ad8111f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:28.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 --2- 172.21.15.28:0/1495785782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8077570 0x7f7ad80759d0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:28.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 --2- 172.21.15.28:0/1495785782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7ad8075fa0 0x7f7ad8076420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:28.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 -- 172.21.15.28:0/1495785782 >> 172.21.15.28:0/1495785782 conn(0x7f7ad80feb50 msgr2=0x7f7ad8100f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:28.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 -- 172.21.15.28:0/1495785782 shutdown_connections 2024-05-20T19:54:28.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.145+0000 7f7add200700 1 -- 172.21.15.28:0/1495785782 wait complete. 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.146+0000 7f7add200700 1 Processor -- start 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7add200700 1 -- start start 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7add200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8075fa0 0x7f7ad819fd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7add200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7ad8077570 0x7f7ad81a0270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7add200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7ad810fb90 0x7f7ad81a72b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7add200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7ad8115670 con 0x7f7ad810fb90 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7add200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7ad81154f0 con 0x7f7ad8075fa0 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7add200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7ad81157f0 con 0x7f7ad8077570 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7acd400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7ad8077570 0x7f7ad81a0270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7ad6800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7ad810fb90 0x7f7ad81a72b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7ad5e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8075fa0 0x7f7ad819fd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:28.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7acd400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7ad8077570 0x7f7ad81a0270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53824/0 (socket says 172.21.15.28:53824) 2024-05-20T19:54:28.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.147+0000 7f7acd400700 1 -- 172.21.15.28:0/497462629 learned_addr learned my addr 172.21.15.28:0/497462629 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:54:28.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.148+0000 7f7ad5e00700 1 -- 172.21.15.28:0/497462629 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7ad8077570 msgr2=0x7f7ad81a0270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.148+0000 7f7ad5e00700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7ad8077570 0x7f7ad81a0270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.148+0000 7f7ad5e00700 1 -- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7ad810fb90 msgr2=0x7f7ad81a72b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.148+0000 7f7ad5e00700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7ad810fb90 0x7f7ad81a72b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.148+0000 7f7ad5e00700 1 -- 172.21.15.28:0/497462629 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7ad81a7980 con 0x7f7ad8075fa0 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.148+0000 7f7ad5e00700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8075fa0 0x7f7ad819fd30 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7ac0000c00 tx=0x7f7ac0004790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.148+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ac000fb10 con 0x7f7ad8075fa0 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.149+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7ac000fcb0 con 0x7f7ad8075fa0 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.149+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7ad81a7c10 con 0x7f7ad8075fa0 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.149+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ac00055c0 con 0x7f7ad8075fa0 2024-05-20T19:54:28.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.149+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7ad81a80c0 con 0x7f7ad8075fa0 2024-05-20T19:54:28.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.150+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7ad806a230 con 0x7f7ad8075fa0 2024-05-20T19:54:28.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.150+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 13) ==== 50278+0+0 (secure 0 0 0) 0x7f7ac0009d10 con 0x7f7ad8075fa0 2024-05-20T19:54:28.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.151+0000 7f7acf400700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7abc03e750 0x7f7abc040c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:54:28.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.151+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 1155+0+0 (secure 0 0 0) 0x7f7ac00536e0 con 0x7f7ad8075fa0 2024-05-20T19:54:28.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.151+0000 7f7acd400700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7abc03e750 0x7f7abc040c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:54:28.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.152+0000 7f7acd400700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7abc03e750 0x7f7abc040c10 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f7ac8005950 tx=0x7f7ac8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:54:28.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.156+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7ac0013070 con 0x7f7ad8075fa0 2024-05-20T19:54:28.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:28.406+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7f7ad8063980 con 0x7f7abc03e750 2024-05-20T19:54:28.519 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:28.164+0000 7f8d4042a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T19:54:28.519 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:28.341+0000 7f8d4042a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T19:54:28.519 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:28.518+0000 7f8d4042a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T19:54:28.610 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:28.260+0000 7f95516a3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T19:54:28.610 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:28.609+0000 7f95516a3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T19:54:28.818 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:28.818 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.818 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:54:28.819 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.875 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:28.702+0000 7f95516a3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.877 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:54:28.877 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:54:28.877 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.970 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:28.971 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:54:28.972 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:54:28.972 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:29.123 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:28.817+0000 7f8d4042a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T19:54:29.123 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:28.913+0000 7f8d4042a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T19:54:29.123 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:29.123+0000 7f8d4042a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T19:54:29.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:28.938+0000 7f95516a3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T19:54:29.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:29.067+0000 7f95516a3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T19:54:29.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:29.160+0000 7f95516a3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T19:54:29.377 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:29.323+0000 7f8d4042a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T19:54:29.879 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:29 smithi154 ceph-mon[36314]: Reconfiguring mgr.x (unknown last config time)... 2024-05-20T19:54:29.879 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:29 smithi154 ceph-mon[36314]: Reconfiguring daemon mgr.x on smithi028 2024-05-20T19:54:29.879 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:29 smithi154 ceph-mon[36314]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:54:29.879 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:29.485+0000 7f95516a3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T19:54:29.879 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:29.575+0000 7f95516a3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T19:54:29.910 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:29 smithi028 ceph-mon[31178]: Reconfiguring mgr.x (unknown last config time)... 2024-05-20T19:54:29.910 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:29 smithi028 ceph-mon[31178]: Reconfiguring daemon mgr.x on smithi028 2024-05-20T19:54:29.910 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:29 smithi028 ceph-mon[31178]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:54:29.953 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:29 smithi033 ceph-mon[36179]: Reconfiguring mgr.x (unknown last config time)... 2024-05-20T19:54:29.954 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:29 smithi033 ceph-mon[36179]: Reconfiguring daemon mgr.x on smithi028 2024-05-20T19:54:29.954 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:29 smithi033 ceph-mon[36179]: from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:54:30.377 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:29.953+0000 7f8d4042a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T19:54:30.377 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:30.183+0000 7f8d4042a200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T19:54:30.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:30.168+0000 7f95516a3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T19:54:30.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 19:54:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T19:54:30.383+0000 7f95516a3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T19:54:30.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: Standby manager daemon z started 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-05-20T19:54:30.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:30 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T19:54:30.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:30.623+0000 7f8d4042a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T19:54:30.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:30.723+0000 7f8d4042a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T19:54:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: Standby manager daemon z started 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-05-20T19:54:30.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:30 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T19:54:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: Standby manager daemon z started 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-05-20T19:54:31.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:30 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/1951334103' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T19:54:31.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:31.133+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mgrmap(e 14) ==== 99937+0+0 (secure 0 0 0) 0x7f7ac0004b80 con 0x7f7ad8075fa0 2024-05-20T19:54:31.377 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:31.108+0000 7f8d4042a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T19:54:31.832 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:31.482+0000 7f8d4042a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T19:54:31.833 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:31.578+0000 7f8d4042a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T19:54:32.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:31.832+0000 7f8d4042a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T19:54:32.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:31.970+0000 7f8d4042a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T19:54:32.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:32.069+0000 7f8d4042a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T19:54:32.410 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:32 smithi033 ceph-mon[36179]: mgrmap e14: x(active, since 87s), standbys: z 2024-05-20T19:54:32.410 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:32 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-05-20T19:54:32.410 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:32 smithi033 ceph-mon[36179]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:32.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:32 smithi028 ceph-mon[31178]: mgrmap e14: x(active, since 87s), standbys: z 2024-05-20T19:54:32.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:32 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-05-20T19:54:32.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:32 smithi028 ceph-mon[31178]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:32.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:32 smithi154 ceph-mon[36314]: mgrmap e14: x(active, since 87s), standbys: z 2024-05-20T19:54:32.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:32 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-05-20T19:54:32.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:32 smithi154 ceph-mon[36314]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:32.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:32.409+0000 7f8d4042a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T19:54:32.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:32.505+0000 7f8d4042a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T19:54:33.459 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:33 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2648165153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]: dispatch 2024-05-20T19:54:33.459 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:33 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]: dispatch 2024-05-20T19:54:33.459 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:33 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]': finished 2024-05-20T19:54:33.460 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:33 smithi028 ceph-mon[31178]: osdmap e5: 1 total, 0 up, 1 in 2024-05-20T19:54:33.460 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:33 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:33.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:33 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2648165153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]: dispatch 2024-05-20T19:54:33.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:33 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]: dispatch 2024-05-20T19:54:33.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:33 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]': finished 2024-05-20T19:54:33.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:33 smithi154 ceph-mon[36314]: osdmap e5: 1 total, 0 up, 1 in 2024-05-20T19:54:33.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:33 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:33.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:33 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2648165153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]: dispatch 2024-05-20T19:54:33.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:33 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]: dispatch 2024-05-20T19:54:33.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:33 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ee508d61-6223-4ba1-a7c6-6be05c87bad9"}]': finished 2024-05-20T19:54:33.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:33 smithi033 ceph-mon[36179]: osdmap e5: 1 total, 0 up, 1 in 2024-05-20T19:54:33.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:33 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:33.628 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:33.136+0000 7f8d4042a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T19:54:33.628 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 19:54:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T19:54:33.361+0000 7f8d4042a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T19:54:34.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:34.224+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mon.2 v2:172.21.15.33:3300/0 8 ==== mgrmap(e 15) ==== 149596+0+0 (secure 0 0 0) 0x7f7ac00351f0 con 0x7f7ad8075fa0 2024-05-20T19:54:34.489 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:34 smithi028 ceph-mon[31178]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:34.489 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T19:54:34.490 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:34 smithi028 ceph-mon[31178]: Standby manager daemon y started 2024-05-20T19:54:34.490 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T19:54:34.490 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T19:54:34.490 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T19:54:34.490 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:34 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/745061557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:54:34.562 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:34 smithi154 ceph-mon[36314]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:34.562 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T19:54:34.562 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:34 smithi154 ceph-mon[36314]: Standby manager daemon y started 2024-05-20T19:54:34.562 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T19:54:34.562 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T19:54:34.562 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T19:54:34.563 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:34 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/745061557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:54:34.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:34 smithi033 ceph-mon[36179]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:34.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T19:54:34.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:34 smithi033 ceph-mon[36179]: Standby manager daemon y started 2024-05-20T19:54:34.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T19:54:34.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T19:54:34.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/2035455266' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T19:54:34.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:34 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/745061557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:54:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:35 smithi028 ceph-mon[31178]: mgrmap e15: x(active, since 91s), standbys: z, y 2024-05-20T19:54:35.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:35 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T19:54:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:35 smithi154 ceph-mon[36314]: mgrmap e15: x(active, since 91s), standbys: z, y 2024-05-20T19:54:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:35 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T19:54:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:35 smithi033 ceph-mon[36179]: mgrmap e15: x(active, since 91s), standbys: z, y 2024-05-20T19:54:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:35 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T19:54:36.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:36 smithi028 ceph-mon[31178]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:36.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:36 smithi154 ceph-mon[36314]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:36.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:36 smithi033 ceph-mon[36179]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:38.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:38 smithi028 ceph-mon[31178]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:38.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:38 smithi154 ceph-mon[36314]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:38.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:38 smithi033 ceph-mon[36179]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:40.548 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:40 smithi028 ceph-mon[31178]: pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:40.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:40 smithi154 ceph-mon[36314]: pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:40 smithi033 ceph-mon[36179]: pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:42.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:42 smithi154 ceph-mon[36314]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:42.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T19:54:42.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:42.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:42 smithi033 ceph-mon[36179]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:42.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T19:54:42.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:42 smithi028 ceph-mon[31178]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T19:54:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:43.382 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:43 smithi028 ceph-mon[31178]: Deploying daemon osd.0 on smithi028 2024-05-20T19:54:43.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:43 smithi154 ceph-mon[36314]: Deploying daemon osd.0 on smithi028 2024-05-20T19:54:43.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:43 smithi033 ceph-mon[36179]: Deploying daemon osd.0 on smithi028 2024-05-20T19:54:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:44 smithi028 ceph-mon[31178]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:44 smithi154 ceph-mon[36314]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:44 smithi033 ceph-mon[36179]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:46.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:46 smithi154 ceph-mon[36314]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:46.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:46 smithi033 ceph-mon[36179]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:46.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:46 smithi028 ceph-mon[31178]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:48.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:48 smithi154 ceph-mon[36314]: pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:48.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:48 smithi033 ceph-mon[36179]: pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:48.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:48 smithi028 ceph-mon[31178]: pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:50.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:50 smithi028 ceph-mon[31178]: pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:50.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:50 smithi154 ceph-mon[36314]: pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:50.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:50 smithi033 ceph-mon[36179]: pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:51.664 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:51.664 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:51.664 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:51.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:51.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:52.543 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:52 smithi028 ceph-mon[31178]: pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:52 smithi154 ceph-mon[36314]: pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:52 smithi033 ceph-mon[36179]: pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:54.581 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:54 smithi028 ceph-mon[31178]: pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:54.581 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.581 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.581 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:54.581 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:54.581 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:54 smithi154 ceph-mon[36314]: pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:54.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:54.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:54.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:54 smithi033 ceph-mon[36179]: pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:54:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:54:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:55.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.425+0000 7f7acf400700 1 -- 172.21.15.28:0/497462629 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f7ad8063980 con 0x7f7abc03e750 2024-05-20T19:54:55.425 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 0 on host 'smithi028' 2024-05-20T19:54:55.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7abc03e750 msgr2=0x7f7abc040c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:55.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7abc03e750 0x7f7abc040c10 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f7ac8005950 tx=0x7f7ac8009260 comp rx=0 tx=0).stop 2024-05-20T19:54:55.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8075fa0 msgr2=0x7f7ad819fd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:54:55.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8075fa0 0x7f7ad819fd30 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7ac0000c00 tx=0x7f7ac0004790 comp rx=0 tx=0).stop 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 shutdown_connections 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7abc03e750 0x7f7abc040c10 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7ad810fb90 0x7f7ad81a72b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7ad8075fa0 0x7f7ad819fd30 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 --2- 172.21.15.28:0/497462629 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7ad8077570 0x7f7ad81a0270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 >> 172.21.15.28:0/497462629 conn(0x7f7ad80feb50 msgr2=0x7f7ad8102f30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 shutdown_connections 2024-05-20T19:54:55.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:54:55.428+0000 7f7add200700 1 -- 172.21.15.28:0/497462629 wait complete. 2024-05-20T19:54:55.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:55 smithi028 ceph-mon[31178]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T19:54:55.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:55 smithi154 ceph-mon[36314]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T19:54:55.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:55 smithi033 ceph-mon[36179]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T19:54:56.228 DEBUG:teuthology.orchestra.run.smithi028:osd.0> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.0.service 2024-05-20T19:54:56.233 INFO:tasks.cephadm:Deploying osd.1 on smithi033 with /dev/vg_nvme/lv_4... 2024-05-20T19:54:56.233 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-20T19:54:56.430 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:56.431 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T19:54:56.431 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T19:54:56.431 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:54:56.431 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:56.431 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:56.431 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:56.431 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:56.432 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:54:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0[45678]: 2024-05-20T19:54:56.370+0000 7efe39000700 -1 osd.0 0 waiting for initial osdmap 2024-05-20T19:54:56.432 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:54:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0[45678]: 2024-05-20T19:54:56.377+0000 7efe33e00700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:54:56.478 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:56.600 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T19:54:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T19:54:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:54:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:54:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:54:57.515 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:57 smithi028 ceph-mon[31178]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:54:57.515 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:57 smithi028 ceph-mon[31178]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T19:54:57.516 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:57 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:57.516 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:57 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:57.545 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:57 smithi033 ceph-mon[36179]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:54:57.545 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:57 smithi033 ceph-mon[36179]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T19:54:57.545 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:57 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:57.545 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:57 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:57 smithi154 ceph-mon[36314]: from='osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:54:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:57 smithi154 ceph-mon[36314]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T19:54:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:57 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:57 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:58.408 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:58 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:54:58.408 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:58 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:54:58.408 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:58 smithi033 ceph-mon[36179]: pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:58.408 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:58 smithi033 ceph-mon[36179]: osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] boot 2024-05-20T19:54:58.408 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:58 smithi033 ceph-mon[36179]: osdmap e8: 1 total, 1 up, 1 in 2024-05-20T19:54:58.408 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:58 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:58.409 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-20T19:54:58.409 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-20T19:54:58.409 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-05-20T19:54:58.409 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-05-20T19:54:58.409 INFO:teuthology.orchestra.run.smithi033.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111163 s, 943 MB/s 2024-05-20T19:54:58.409 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-05-20T19:54:58.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:58 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:54:58.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:58 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:54:58.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:58 smithi028 ceph-mon[31178]: pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:58.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:58 smithi028 ceph-mon[31178]: osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] boot 2024-05-20T19:54:58.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:58 smithi028 ceph-mon[31178]: osdmap e8: 1 total, 1 up, 1 in 2024-05-20T19:54:58.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:58 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:58 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:54:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:58 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:54:58.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:58 smithi154 ceph-mon[36314]: pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T19:54:58.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:58 smithi154 ceph-mon[36314]: osd.0 [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] boot 2024-05-20T19:54:58.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:58 smithi154 ceph-mon[36314]: osdmap e8: 1 total, 1 up, 1 in 2024-05-20T19:54:58.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T19:54:58.975 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_4 2024-05-20T19:54:59.220 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:54:59.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:54:59 smithi033 ceph-mon[36179]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T19:54:59.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:54:59 smithi028 ceph-mon[31178]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T19:54:59.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:54:59 smithi154 ceph-mon[36314]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T19:55:00.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:00 smithi028 ceph-mon[31178]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:00.640 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:00 smithi033 ceph-mon[36179]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:00.762 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1899317193 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d0106980 msgr2=0x7fb3d0108d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1899317193 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d0106980 0x7fb3d0108d70 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fb3c0009990 tx=0x7fb3c0009ca0 comp rx=0 tx=0).stop 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1899317193 shutdown_connections 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1899317193 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3d01092b0 0x7fb3d010b720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1899317193 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3d0104050 0x7fb3d0106440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1899317193 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d0106980 0x7fb3d0108d70 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1899317193 >> 172.21.15.33:0/1899317193 conn(0x7fb3d00fda60 msgr2=0x7fb3d00ffea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1899317193 shutdown_connections 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.746+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1899317193 wait complete. 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 Processor -- start 2024-05-20T19:55:00.763 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 -- start start 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3d0104050 0x7fb3d01039b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3d0106980 0x7fb3d0102000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d01092b0 0x7fb3d0102540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d010f0e0 con 0x7fb3d0104050 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d010ef60 con 0x7fb3d01092b0 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d80aa700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb3d010f260 con 0x7fb3d0106980 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d01092b0 0x7fb3d0102540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d01092b0 0x7fb3d0102540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.33:33086/0 (socket says 172.21.15.33:33086) 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 -- 172.21.15.33:0/1743167865 learned_addr learned my addr 172.21.15.33:0/1743167865 (peer_addr_for_me v2:172.21.15.33:0/0) 2024-05-20T19:55:00.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d5645700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3d0106980 0x7fb3d0102000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 -- 172.21.15.33:0/1743167865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3d0106980 msgr2=0x7fb3d0102000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3d0106980 0x7fb3d0102000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 -- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3d0104050 msgr2=0x7fb3d01039b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3d0104050 0x7fb3d01039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.747+0000 7fb3d6647700 1 -- 172.21.15.33:0/1743167865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb3d0102df0 con 0x7fb3d01092b0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.748+0000 7fb3d6647700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d01092b0 0x7fb3d0102540 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fb3c800b9e0 tx=0x7fb3c800bda0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.748+0000 7fb3c6ffd700 1 -- 172.21.15.33:0/1743167865 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3c8019070 con 0x7fb3d01092b0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.748+0000 7fb3c6ffd700 1 -- 172.21.15.33:0/1743167865 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3c8014440 con 0x7fb3d01092b0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.748+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb3d01a38a0 con 0x7fb3d01092b0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.748+0000 7fb3c6ffd700 1 -- 172.21.15.33:0/1743167865 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3c800ce60 con 0x7fb3d01092b0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.748+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb3d01a3e30 con 0x7fb3d01092b0 2024-05-20T19:55:00.765 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.749+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb3d006a230 con 0x7fb3d01092b0 2024-05-20T19:55:00.766 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.750+0000 7fb3c6ffd700 1 -- 172.21.15.33:0/1743167865 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 15) ==== 149596+0+0 (secure 0 0 0) 0x7fb3c801c070 con 0x7fb3d01092b0 2024-05-20T19:55:00.766 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.751+0000 7fb3c6ffd700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3bc0b28f0 0x7fb3bc0b4db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:00.766 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.751+0000 7fb3c6ffd700 1 -- 172.21.15.33:0/1743167865 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(9..9 src has 1..9) ==== 1767+0+0 (secure 0 0 0) 0x7fb3c80dfbb0 con 0x7fb3d01092b0 2024-05-20T19:55:00.766 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.751+0000 7fb3d5e46700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3bc0b28f0 0x7fb3bc0b4db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:00.766 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.752+0000 7fb3d5e46700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3bc0b28f0 0x7fb3bc0b4db0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc005950 tx=0x7fb3cc0058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:55:00.766 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.756+0000 7fb3c6ffd700 1 -- 172.21.15.33:0/1743167865 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb3c80a92a0 con 0x7fb3d01092b0 2024-05-20T19:55:00.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:00 smithi154 ceph-mon[36314]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:00.981 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:00.980+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7fb3d0063980 con 0x7fb3bc0b28f0 2024-05-20T19:55:01.645 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:01.646 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:01.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:01.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:02.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:02 smithi033 ceph-mon[36179]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:02.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:02 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi028 2024-05-20T19:55:02.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:02 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi028 to 17094M 2024-05-20T19:55:02.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:02 smithi033 ceph-mon[36179]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:02.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:02 smithi028 ceph-mon[31178]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:02.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:02 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi028 2024-05-20T19:55:02.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:02 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi028 to 17094M 2024-05-20T19:55:02.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:02 smithi028 ceph-mon[31178]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:02 smithi154 ceph-mon[36314]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:02 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi028 2024-05-20T19:55:02.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:02 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi028 to 17094M 2024-05-20T19:55:02.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:02 smithi154 ceph-mon[36314]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:04 smithi028 ceph-mon[31178]: pgmap v59: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:04 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/1993604503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]: dispatch 2024-05-20T19:55:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:04 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]: dispatch 2024-05-20T19:55:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:04 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]': finished 2024-05-20T19:55:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:04 smithi028 ceph-mon[31178]: osdmap e10: 2 total, 1 up, 2 in 2024-05-20T19:55:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:04 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/32987996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:04.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:04 smithi154 ceph-mon[36314]: pgmap v59: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:04.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:04 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/1993604503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]: dispatch 2024-05-20T19:55:04.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:04 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]: dispatch 2024-05-20T19:55:04.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:04 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]': finished 2024-05-20T19:55:04.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:04 smithi154 ceph-mon[36314]: osdmap e10: 2 total, 1 up, 2 in 2024-05-20T19:55:04.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:04.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:04 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/32987996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:04 smithi033 ceph-mon[36179]: pgmap v59: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:04 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/1993604503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]: dispatch 2024-05-20T19:55:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:04 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]: dispatch 2024-05-20T19:55:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:04 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c93dcd4d-a863-4d99-922d-2ba54fe25033"}]': finished 2024-05-20T19:55:04.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:04 smithi033 ceph-mon[36179]: osdmap e10: 2 total, 1 up, 2 in 2024-05-20T19:55:04.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:04.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:04 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/32987996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:06.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:06 smithi028 ceph-mon[31178]: pgmap v61: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:06.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:06 smithi154 ceph-mon[36314]: pgmap v61: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:06.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:06 smithi033 ceph-mon[36179]: pgmap v61: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:08.594 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:08 smithi033 ceph-mon[36179]: pgmap v62: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:08.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:08 smithi028 ceph-mon[31178]: pgmap v62: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:08.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:08 smithi154 ceph-mon[36314]: pgmap v62: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:10.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:10 smithi028 ceph-mon[31178]: pgmap v63: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:10 smithi154 ceph-mon[36314]: pgmap v63: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:10.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:10 smithi033 ceph-mon[36179]: pgmap v63: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:11.475 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:11 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T19:55:11.475 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:11 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:11.475 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:11 smithi033 ceph-mon[36179]: Deploying daemon osd.1 on smithi033 2024-05-20T19:55:11.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:11 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T19:55:11.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:11 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:11.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:11 smithi028 ceph-mon[31178]: Deploying daemon osd.1 on smithi033 2024-05-20T19:55:11.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:11 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T19:55:11.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:11 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:11.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:11 smithi154 ceph-mon[36314]: Deploying daemon osd.1 on smithi033 2024-05-20T19:55:12.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:12 smithi033 ceph-mon[36179]: pgmap v64: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:12.776 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:12 smithi154 ceph-mon[36314]: pgmap v64: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:12.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:12 smithi028 ceph-mon[31178]: pgmap v64: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:14.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:14 smithi033 ceph-mon[36179]: pgmap v65: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:14.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:14 smithi028 ceph-mon[31178]: pgmap v65: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:14.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:14 smithi154 ceph-mon[36314]: pgmap v65: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:16.587 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:16 smithi033 ceph-mon[36179]: pgmap v66: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:16.588 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:16.588 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:16.588 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:16 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:16 smithi028 ceph-mon[31178]: pgmap v66: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:16 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:16 smithi154 ceph-mon[36314]: pgmap v66: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:16 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:18.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:18 smithi028 ceph-mon[31178]: pgmap v67: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:18.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:18 smithi154 ceph-mon[36314]: pgmap v67: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:18 smithi033 ceph-mon[36179]: pgmap v67: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:19.672 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 1 on host 'smithi033' 2024-05-20T19:55:19.672 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.671+0000 7fb3c6ffd700 1 -- 172.21.15.33:0/1743167865 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fb3d0063980 con 0x7fb3bc0b28f0 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3bc0b28f0 msgr2=0x7fb3bc0b4db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3bc0b28f0 0x7fb3bc0b4db0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fb3cc005950 tx=0x7fb3cc0058e0 comp rx=0 tx=0).stop 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d01092b0 msgr2=0x7fb3d0102540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d01092b0 0x7fb3d0102540 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fb3c800b9e0 tx=0x7fb3c800bda0 comp rx=0 tx=0).stop 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 shutdown_connections 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3bc0b28f0 0x7fb3bc0b4db0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3d01092b0 0x7fb3d0102540 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3d0104050 0x7fb3d01039b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 --2- 172.21.15.33:0/1743167865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3d0106980 0x7fb3d0102000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 >> 172.21.15.33:0/1743167865 conn(0x7fb3d00fda60 msgr2=0x7fb3d0104990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:55:19.675 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 shutdown_connections 2024-05-20T19:55:19.676 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:55:19.674+0000 7fb3d80aa700 1 -- 172.21.15.33:0/1743167865 wait complete. 2024-05-20T19:55:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: pgmap v68: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: pgmap v68: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: pgmap v68: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:20.322 DEBUG:teuthology.orchestra.run.smithi033:osd.1> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.1.service 2024-05-20T19:55:20.325 INFO:tasks.cephadm:Deploying osd.2 on smithi154 with /dev/vg_nvme/lv_4... 2024-05-20T19:55:20.325 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-20T19:55:20.568 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:55:20.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:20 smithi033 ceph-mon[36179]: from='osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T19:55:20.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:20 smithi033 ceph-mon[36179]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T19:55:20.988 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:20 smithi154 ceph-mon[36314]: from='osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T19:55:20.988 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:20 smithi154 ceph-mon[36314]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T19:55:21.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:20 smithi028 ceph-mon[31178]: from='osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T19:55:21.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:20 smithi028 ceph-mon[31178]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T19:55:22.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:21 smithi028 ceph-mon[31178]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T19:55:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:21 smithi028 ceph-mon[31178]: from='osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:55:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:21 smithi028 ceph-mon[31178]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T19:55:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:21 smithi028 ceph-mon[31178]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:55:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:21 smithi028 ceph-mon[31178]: pgmap v70: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:22.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:21 smithi154 ceph-mon[36314]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T19:55:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:21 smithi154 ceph-mon[36314]: from='osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:55:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:21 smithi154 ceph-mon[36314]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T19:55:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:21 smithi154 ceph-mon[36314]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:55:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:21 smithi154 ceph-mon[36314]: pgmap v70: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:22.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:21 smithi033 ceph-mon[36179]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T19:55:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:21 smithi033 ceph-mon[36179]: from='osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:55:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:21 smithi033 ceph-mon[36179]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T19:55:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:21 smithi033 ceph-mon[36179]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:55:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:21 smithi033 ceph-mon[36179]: pgmap v70: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:22.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1[40730]: 2024-05-20T19:55:21.789+0000 7f713fcd4700 -1 osd.1 0 waiting for initial osdmap 2024-05-20T19:55:22.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1[40730]: 2024-05-20T19:55:21.792+0000 7f713bac7700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:55:22.388 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-20T19:55:22.388 INFO:teuthology.orchestra.run.smithi154.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-20T19:55:22.388 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10+0 records in 2024-05-20T19:55:22.388 INFO:teuthology.orchestra.run.smithi154.stderr:10+0 records out 2024-05-20T19:55:22.388 INFO:teuthology.orchestra.run.smithi154.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113174 s, 927 MB/s 2024-05-20T19:55:22.389 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping successful for: 2024-05-20T19:55:22.982 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:22 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:55:22.982 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:22 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:55:22.982 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:22 smithi154 ceph-mon[36314]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:55:22.982 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:22 smithi154 ceph-mon[36314]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T19:55:22.982 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:22.982 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:23.009 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_4 2024-05-20T19:55:23.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:22 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:55:23.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:22 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:55:23.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:22 smithi028 ceph-mon[31178]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:55:23.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:22 smithi028 ceph-mon[31178]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T19:55:23.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:23.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:22 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:55:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:22 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:55:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:22 smithi033 ceph-mon[36179]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:55:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:22 smithi033 ceph-mon[36179]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T19:55:23.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:23.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:23.248 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:55:24.047 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:24.047 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] boot 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: osdmap e13: 2 total, 2 up, 2 in 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi033 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi033 to 17093M 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:24.048 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:23 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:24.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] boot 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: osdmap e13: 2 total, 2 up, 2 in 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi033 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi033 to 17093M 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:24.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:23 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: osd.1 [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] boot 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: osdmap e13: 2 total, 2 up, 2 in 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi033 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi033 to 17093M 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:24.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:24.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:23 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 -- 172.21.15.154:0/3048664427 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281035a0 msgr2=0x7fd8281039c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 --2- 172.21.15.154:0/3048664427 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281035a0 0x7fd8281039c0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fd818009990 tx=0x7fd818009ca0 comp rx=0 tx=0).stop 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 -- 172.21.15.154:0/3048664427 shutdown_connections 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 --2- 172.21.15.154:0/3048664427 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd828105140 0x7fd82810b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 --2- 172.21.15.154:0/3048664427 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281035a0 0x7fd8281039c0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 --2- 172.21.15.154:0/3048664427 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8281047a0 0x7fd828104c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 -- 172.21.15.154:0/3048664427 >> 172.21.15.154:0/3048664427 conn(0x7fd8280feb40 msgr2=0x7fd828100f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 -- 172.21.15.154:0/3048664427 shutdown_connections 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.787+0000 7fd82d450700 1 -- 172.21.15.154:0/3048664427 wait complete. 2024-05-20T19:55:24.788 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 Processor -- start 2024-05-20T19:55:24.790 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 -- start start 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8281035a0 0x7fd828199830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281047a0 0x7fd828199d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd828105140 0x7fd8281a0da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd82810f110 con 0x7fd8281035a0 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd82810ef90 con 0x7fd828105140 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd82d450700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd82810f290 con 0x7fd8281047a0 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281047a0 0x7fd828199d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281047a0 0x7fd828199d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:34970/0 (socket says 172.21.15.154:34970) 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 -- 172.21.15.154:0/106078922 learned_addr learned my addr 172.21.15.154:0/106078922 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd8277fe700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd828105140 0x7fd8281a0da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 -- 172.21.15.154:0/106078922 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd828105140 msgr2=0x7fd8281a0da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:24.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd828105140 0x7fd8281a0da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 -- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8281035a0 msgr2=0x7fd828199830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8281035a0 0x7fd828199830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.788+0000 7fd81e7fc700 1 -- 172.21.15.154:0/106078922 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd8281a1440 con 0x7fd8281047a0 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.789+0000 7fd81e7fc700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281047a0 0x7fd828199d70 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fd81000b560 tx=0x7fd81000b920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.789+0000 7fd824ff9700 1 -- 172.21.15.154:0/106078922 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd810015020 con 0x7fd8281047a0 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.789+0000 7fd824ff9700 1 -- 172.21.15.154:0/106078922 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd810011ce0 con 0x7fd8281047a0 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.789+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd8281a1730 con 0x7fd8281047a0 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.789+0000 7fd824ff9700 1 -- 172.21.15.154:0/106078922 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd810010da0 con 0x7fd8281047a0 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.789+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd8281a1ca0 con 0x7fd8281047a0 2024-05-20T19:55:24.792 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.790+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd82806a230 con 0x7fd8281047a0 2024-05-20T19:55:24.793 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.791+0000 7fd824ff9700 1 -- 172.21.15.154:0/106078922 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 15) ==== 149596+0+0 (secure 0 0 0) 0x7fd810018020 con 0x7fd8281047a0 2024-05-20T19:55:24.793 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.792+0000 7fd824ff9700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8140b27f0 0x7fd8140b4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:24.793 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.792+0000 7fd824ff9700 1 -- 172.21.15.154:0/106078922 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(14..14 src has 1..14) ==== 2309+0+0 (secure 0 0 0) 0x7fd8100e07a0 con 0x7fd8281047a0 2024-05-20T19:55:24.793 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.792+0000 7fd826ffd700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8140b27f0 0x7fd8140b4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:24.793 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.793+0000 7fd826ffd700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8140b27f0 0x7fd8140b4cb0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fd81800b5c0 tx=0x7fd818003860 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:55:24.796 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:24.795+0000 7fd824ff9700 1 -- 172.21.15.154:0/106078922 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fd81001f020 con 0x7fd8281047a0 2024-05-20T19:55:25.008 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:25.008+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) -- 0x7fd828063980 con 0x7fd8140b27f0 2024-05-20T19:55:25.419 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:25 smithi033 ceph-mon[36179]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T19:55:25.419 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:25.419 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:25.419 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:25 smithi154 ceph-mon[36314]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T19:55:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:25.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:25 smithi028 ceph-mon[31178]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T19:55:25.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:25.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:26.582 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:26 smithi154 ceph-mon[36314]: from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:26.583 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:26 smithi154 ceph-mon[36314]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:26 smithi033 ceph-mon[36179]: from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:26 smithi033 ceph-mon[36179]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:26.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:26 smithi028 ceph-mon[31178]: from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:26.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:26 smithi028 ceph-mon[31178]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:28.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:28 smithi028 ceph-mon[31178]: pgmap v76: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:28 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/1290275677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]: dispatch 2024-05-20T19:55:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:28 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]: dispatch 2024-05-20T19:55:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:28 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]': finished 2024-05-20T19:55:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:28 smithi028 ceph-mon[31178]: osdmap e15: 3 total, 2 up, 3 in 2024-05-20T19:55:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:28.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:28 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/1149085663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:28 smithi154 ceph-mon[36314]: pgmap v76: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:28 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/1290275677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]: dispatch 2024-05-20T19:55:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:28 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]: dispatch 2024-05-20T19:55:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:28 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]': finished 2024-05-20T19:55:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:28 smithi154 ceph-mon[36314]: osdmap e15: 3 total, 2 up, 3 in 2024-05-20T19:55:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:28 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/1149085663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:28 smithi033 ceph-mon[36179]: pgmap v76: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:28 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/1290275677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]: dispatch 2024-05-20T19:55:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:28 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]: dispatch 2024-05-20T19:55:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:28 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a98e172e-f3be-463f-a25d-bc9fc778ce48"}]': finished 2024-05-20T19:55:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:28 smithi033 ceph-mon[36179]: osdmap e15: 3 total, 2 up, 3 in 2024-05-20T19:55:28.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:28.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:28 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/1149085663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:30.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:30 smithi028 ceph-mon[31178]: pgmap v78: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:30.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:30 smithi154 ceph-mon[36314]: pgmap v78: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:30 smithi033 ceph-mon[36179]: pgmap v78: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:32.674 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:32 smithi154 ceph-mon[36314]: pgmap v79: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:32.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:32 smithi028 ceph-mon[31178]: pgmap v79: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:32 smithi033 ceph-mon[36179]: pgmap v79: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:34.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:34 smithi028 ceph-mon[31178]: pgmap v80: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:34.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:34 smithi154 ceph-mon[36314]: pgmap v80: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:34.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:34 smithi033 ceph-mon[36179]: pgmap v80: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:35.582 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:35 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T19:55:35.582 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:35 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:35.582 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:35 smithi154 ceph-mon[36314]: Deploying daemon osd.2 on smithi154 2024-05-20T19:55:35.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:35 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T19:55:35.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:35 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:35.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:35 smithi028 ceph-mon[31178]: Deploying daemon osd.2 on smithi154 2024-05-20T19:55:35.867 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:35 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T19:55:35.867 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:35 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:35.867 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:35 smithi033 ceph-mon[36179]: Deploying daemon osd.2 on smithi154 2024-05-20T19:55:36.735 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:36 smithi154 ceph-mon[36314]: pgmap v81: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:36.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:36 smithi028 ceph-mon[31178]: pgmap v81: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:36.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:36 smithi033 ceph-mon[36179]: pgmap v81: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:38.683 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:38 smithi154 ceph-mon[36314]: pgmap v82: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:38 smithi028 ceph-mon[31178]: pgmap v82: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:38.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:38 smithi033 ceph-mon[36179]: pgmap v82: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:40.479 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:40 smithi154 ceph-mon[36314]: pgmap v83: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:40.479 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:40.479 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:40.479 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:40 smithi028 ceph-mon[31178]: pgmap v83: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:40.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:40.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:40 smithi033 ceph-mon[36179]: pgmap v83: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:42.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:42 smithi154 ceph-mon[36314]: pgmap v84: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:42.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:42.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:42.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:42.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:42.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:42 smithi033 ceph-mon[36179]: pgmap v84: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:42.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:42.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:43.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:42 smithi028 ceph-mon[31178]: pgmap v84: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:43.542 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.542+0000 7fd824ff9700 1 -- 172.21.15.154:0/106078922 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fd828063980 con 0x7fd8140b27f0 2024-05-20T19:55:43.542 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 2 on host 'smithi154' 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8140b27f0 msgr2=0x7fd8140b4cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8140b27f0 0x7fd8140b4cb0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fd81800b5c0 tx=0x7fd818003860 comp rx=0 tx=0).stop 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281047a0 msgr2=0x7fd828199d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281047a0 0x7fd828199d70 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fd81000b560 tx=0x7fd81000b920 comp rx=0 tx=0).stop 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 shutdown_connections 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8140b27f0 0x7fd8140b4cb0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd828105140 0x7fd8281a0da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8281035a0 0x7fd828199830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 --2- 172.21.15.154:0/106078922 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8281047a0 0x7fd828199d70 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 >> 172.21.15.154:0/106078922 conn(0x7fd8280feb40 msgr2=0x7fd828107bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:55:43.545 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 shutdown_connections 2024-05-20T19:55:43.546 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:55:43.544+0000 7fd82d450700 1 -- 172.21.15.154:0/106078922 wait complete. 2024-05-20T19:55:44.169 DEBUG:teuthology.orchestra.run.smithi154:osd.2> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.2.service 2024-05-20T19:55:44.172 INFO:tasks.cephadm:Deploying osd.3 on smithi028 with /dev/vg_nvme/lv_3... 2024-05-20T19:55:44.172 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-20T19:55:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:44 smithi028 ceph-mon[31178]: pgmap v85: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:44 smithi028 ceph-mon[31178]: from='osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T19:55:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:44 smithi028 ceph-mon[31178]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T19:55:44.627 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:55:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:44 smithi154 ceph-mon[36314]: pgmap v85: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:44 smithi154 ceph-mon[36314]: from='osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T19:55:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:44 smithi154 ceph-mon[36314]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T19:55:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:44 smithi033 ceph-mon[36179]: pgmap v85: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:55:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:44 smithi033 ceph-mon[36179]: from='osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T19:55:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:44 smithi033 ceph-mon[36179]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T19:55:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:45 smithi154 ceph-mon[36314]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T19:55:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:45 smithi154 ceph-mon[36314]: from='osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:55:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:45 smithi154 ceph-mon[36314]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T19:55:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:45 smithi154 ceph-mon[36314]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:55:45.876 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2[41409]: 2024-05-20T19:55:45.635+0000 7efe53e71700 -1 osd.2 0 waiting for initial osdmap 2024-05-20T19:55:45.876 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2[41409]: 2024-05-20T19:55:45.639+0000 7efe4fc64700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:55:45.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:45 smithi028 ceph-mon[31178]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T19:55:45.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:45 smithi028 ceph-mon[31178]: from='osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:55:45.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:45 smithi028 ceph-mon[31178]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T19:55:45.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:45.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:45 smithi028 ceph-mon[31178]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:55:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:45 smithi033 ceph-mon[36179]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T19:55:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:45 smithi033 ceph-mon[36179]: from='osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:55:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:45 smithi033 ceph-mon[36179]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T19:55:46.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:46.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:45 smithi033 ceph-mon[36179]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:55:46.831 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:46 smithi154 ceph-mon[36314]: pgmap v87: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:46.831 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:46 smithi154 ceph-mon[36314]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:55:46.831 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:46 smithi154 ceph-mon[36314]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T19:55:46.831 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:46 smithi028 ceph-mon[31178]: pgmap v87: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:46 smithi028 ceph-mon[31178]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:55:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:46 smithi028 ceph-mon[31178]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T19:55:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:46 smithi033 ceph-mon[36179]: pgmap v87: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:46 smithi033 ceph-mon[36179]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:55:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:46 smithi033 ceph-mon[36179]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T19:55:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:47.536 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-20T19:55:47.536 INFO:teuthology.orchestra.run.smithi028.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-20T19:55:47.536 INFO:teuthology.orchestra.run.smithi028.stderr: stderr: 10+0 records in 2024-05-20T19:55:47.536 INFO:teuthology.orchestra.run.smithi028.stderr:10+0 records out 2024-05-20T19:55:47.536 INFO:teuthology.orchestra.run.smithi028.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0325268 s, 322 MB/s 2024-05-20T19:55:47.536 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping successful for: 2024-05-20T19:55:47.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:55:47.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] boot 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: osdmap e18: 3 total, 3 up, 3 in 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:47.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:47.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-05-20T19:55:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] boot 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: osdmap e18: 3 total, 3 up, 3 in 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:48.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:48.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-05-20T19:55:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:55:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:55:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: osd.2 [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] boot 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: osdmap e18: 3 total, 3 up, 3 in 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-05-20T19:55:48.359 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_3 2024-05-20T19:55:48.807 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:55:48.929 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi154 2024-05-20T19:55:48.929 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi154 to 17088M 2024-05-20T19:55:48.929 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: pgmap v90: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:48.929 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-05-20T19:55:48.929 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T19:55:48.930 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T19:55:48.930 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T19:55:48.930 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:48 smithi154 ceph-mon[36314]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T19:55:48.959 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi154 2024-05-20T19:55:48.959 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi154 to 17088M 2024-05-20T19:55:48.959 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: pgmap v90: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:48.959 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-05-20T19:55:48.960 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T19:55:48.960 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T19:55:48.960 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T19:55:48.960 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:48 smithi028 ceph-mon[31178]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T19:55:48.960 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50233]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T19:55:48.960 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50233]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:48.960 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50233]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:48.960 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50233]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:48.960 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50250]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T19:55:48.960 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50250]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:48.961 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50250]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:48.961 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 19:55:48 smithi028 sudo[50250]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi154 2024-05-20T19:55:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi154 to 17088M 2024-05-20T19:55:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: pgmap v90: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-05-20T19:55:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-05-20T19:55:49.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T19:55:49.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T19:55:49.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T19:55:49.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:48 smithi033 ceph-mon[36179]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T19:55:49.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44573]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T19:55:49.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44573]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44573]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44573]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44576]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T19:55:49.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44576]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.128 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44576]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.129 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 19:55:48 smithi033 sudo[44576]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50269]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T19:55:49.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50269]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50269]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50269]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50272]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T19:55:49.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50272]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50272]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 sudo[50272]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.375 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45472]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T19:55:49.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45472]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45472]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45472]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45475]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T19:55:49.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45475]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45475]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 19:55:48 smithi154 sudo[45475]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45478]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T19:55:49.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45478]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45478]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45478]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45481]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T19:55:49.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45481]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45481]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 sudo[45481]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44579]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T19:55:49.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44579]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44579]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44579]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:49.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44582]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T19:55:49.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44582]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-05-20T19:55:49.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44582]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-05-20T19:55:49.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 sudo[44582]: pam_unix(sudo:session): session closed for user root 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: pgmap v93: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:50.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:49 smithi154 ceph-mon[36314]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: pgmap v93: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:50.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:49 smithi033 ceph-mon[36179]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T19:55:50.258 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.258 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.258 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:55:50.258 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.258 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.258 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T19:55:50.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T19:55:50.260 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: pgmap v93: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:50.260 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:49 smithi028 ceph-mon[31178]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T19:55:51.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 -- 172.21.15.28:0/728873216 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 msgr2=0x7f7e280759d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:51.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/728873216 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e280759d0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f7e1800b0a0 tx=0x7f7e1800b3b0 comp rx=0 tx=0).stop 2024-05-20T19:55:51.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 -- 172.21.15.28:0/728873216 shutdown_connections 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/728873216 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7e2810fb90 0x7f7e28111f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/728873216 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e280759d0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/728873216 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7e28075fa0 0x7f7e28076420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 -- 172.21.15.28:0/728873216 >> 172.21.15.28:0/728873216 conn(0x7f7e280feb10 msgr2=0x7f7e28100f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 -- 172.21.15.28:0/728873216 shutdown_connections 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.050+0000 7f7e2cc00700 1 -- 172.21.15.28:0/728873216 wait complete. 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.051+0000 7f7e2cc00700 1 Processor -- start 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e2cc00700 1 -- start start 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e2cc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7e28075fa0 0x7f7e2819fd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e2cc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e281a0280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:51.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e2cc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7e2810fb90 0x7f7e281a72c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e2cc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7e28115620 con 0x7f7e28077570 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e2cc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7e281154a0 con 0x7f7e28075fa0 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e2cc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7e281157a0 con 0x7f7e2810fb90 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e1d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e281a0280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e1d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e281a0280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37392/0 (socket says 172.21.15.28:37392) 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e1d400700 1 -- 172.21.15.28:0/2543260268 learned_addr learned my addr 172.21.15.28:0/2543260268 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.052+0000 7f7e25e00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7e28075fa0 0x7f7e2819fd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e26800700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7e2810fb90 0x7f7e281a72c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1d400700 1 -- 172.21.15.28:0/2543260268 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7e2810fb90 msgr2=0x7f7e281a72c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1d400700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7e2810fb90 0x7f7e281a72c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1d400700 1 -- 172.21.15.28:0/2543260268 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7e28075fa0 msgr2=0x7f7e2819fd40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:55:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1d400700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7e28075fa0 0x7f7e2819fd40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1d400700 1 -- 172.21.15.28:0/2543260268 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7e281a7990 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e25e00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7e28075fa0 0x7f7e2819fd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1d400700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e281a0280 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f7e1000ea50 tx=0x7f7e1000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1f400700 1 -- 172.21.15.28:0/2543260268 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e1000cbd0 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1f400700 1 -- 172.21.15.28:0/2543260268 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7e1000cd70 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e1f400700 1 -- 172.21.15.28:0/2543260268 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7e10018680 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.053+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7e281a7c80 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.054+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7e281a8190 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.055+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7e2806a230 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.057+0000 7f7e1f400700 1 -- 172.21.15.28:0/2543260268 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f7e1001f020 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.058+0000 7f7e1f400700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7e0c0b2900 0x7f7e0c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.059+0000 7f7e1f400700 1 -- 172.21.15.28:0/2543260268 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3237+0+0 (secure 0 0 0) 0x7f7e100e1680 con 0x7f7e28077570 2024-05-20T19:55:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.059+0000 7f7e25e00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7e0c0b2900 0x7f7e0c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:55:51.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.059+0000 7f7e25e00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7e0c0b2900 0x7f7e0c0b4dc0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f7e18006130 tx=0x7f7e180060c0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:55:51.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.062+0000 7f7e1f400700 1 -- 172.21.15.28:0/2543260268 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7e10026070 con 0x7f7e28077570 2024-05-20T19:55:51.127 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:51 smithi028 ceph-mon[31178]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-05-20T19:55:51.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:55:51.312+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7f7e28063980 con 0x7f7e0c0b2900 2024-05-20T19:55:51.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:51 smithi154 ceph-mon[36314]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-05-20T19:55:51.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:51 smithi033 ceph-mon[36179]: mgrmap e16: x(active, since 2m), standbys: z, y 2024-05-20T19:55:52.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:52 smithi028 ceph-mon[31178]: pgmap v95: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:52.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:52 smithi028 ceph-mon[31178]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:52.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:52 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:52.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:52 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:52.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:52 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:52.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:52 smithi154 ceph-mon[36314]: pgmap v95: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:52.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:52 smithi154 ceph-mon[36314]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:52.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:52 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:52.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:52 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:52.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:52 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:52.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:52 smithi033 ceph-mon[36179]: pgmap v95: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:52.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:52 smithi033 ceph-mon[36179]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:55:52.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:52 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:55:52.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:52 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:55:52.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:52 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:55:54.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:54 smithi028 ceph-mon[31178]: pgmap v96: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:54.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:54 smithi154 ceph-mon[36314]: pgmap v96: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:54 smithi033 ceph-mon[36179]: pgmap v96: 1 pgs: 1 active+clean; 320 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:55:56.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:56 smithi028 ceph-mon[31178]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 6.7 KiB/s rd, 305 KiB/s wr, 20 op/s 2024-05-20T19:55:56.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:56 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/251399420' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]: dispatch 2024-05-20T19:55:56.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:56 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]: dispatch 2024-05-20T19:55:56.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:56 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]': finished 2024-05-20T19:55:56.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:56 smithi028 ceph-mon[31178]: osdmap e22: 4 total, 3 up, 4 in 2024-05-20T19:55:56.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:55:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:56 smithi154 ceph-mon[36314]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 6.7 KiB/s rd, 305 KiB/s wr, 20 op/s 2024-05-20T19:55:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:56 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/251399420' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]: dispatch 2024-05-20T19:55:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:56 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]: dispatch 2024-05-20T19:55:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:56 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]': finished 2024-05-20T19:55:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:56 smithi154 ceph-mon[36314]: osdmap e22: 4 total, 3 up, 4 in 2024-05-20T19:55:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:55:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:56 smithi033 ceph-mon[36179]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 6.7 KiB/s rd, 305 KiB/s wr, 20 op/s 2024-05-20T19:55:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:56 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/251399420' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]: dispatch 2024-05-20T19:55:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:56 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]: dispatch 2024-05-20T19:55:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:56 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "43c813b2-4e9a-4c76-9e2e-6c19825adf7a"}]': finished 2024-05-20T19:55:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:56 smithi033 ceph-mon[36179]: osdmap e22: 4 total, 3 up, 4 in 2024-05-20T19:55:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:55:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:57 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2738978150' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:57.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:57 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2738978150' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:57.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:57 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2738978150' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:55:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:55:58 smithi154 ceph-mon[36314]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 5.0 KiB/s rd, 228 KiB/s wr, 15 op/s 2024-05-20T19:55:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:55:58 smithi033 ceph-mon[36179]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 5.0 KiB/s rd, 228 KiB/s wr, 15 op/s 2024-05-20T19:55:58.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:55:58 smithi028 ceph-mon[31178]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 5.0 KiB/s rd, 228 KiB/s wr, 15 op/s 2024-05-20T19:56:00.508 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:00 smithi028 ceph-mon[31178]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.2 KiB/s rd, 190 KiB/s wr, 13 op/s 2024-05-20T19:56:00.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:00 smithi154 ceph-mon[36314]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.2 KiB/s rd, 190 KiB/s wr, 13 op/s 2024-05-20T19:56:00.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:00 smithi033 ceph-mon[36179]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.2 KiB/s rd, 190 KiB/s wr, 13 op/s 2024-05-20T19:56:02.598 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:02 smithi028 ceph-mon[31178]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 183 KiB/s wr, 12 op/s 2024-05-20T19:56:02.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:02 smithi154 ceph-mon[36314]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 183 KiB/s wr, 12 op/s 2024-05-20T19:56:02.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:02 smithi033 ceph-mon[36179]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 183 KiB/s wr, 12 op/s 2024-05-20T19:56:04.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:04 smithi154 ceph-mon[36314]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 183 KiB/s wr, 12 op/s 2024-05-20T19:56:04.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:04 smithi033 ceph-mon[36179]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 183 KiB/s wr, 12 op/s 2024-05-20T19:56:04.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:04 smithi028 ceph-mon[31178]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 183 KiB/s wr, 12 op/s 2024-05-20T19:56:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:05 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T19:56:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:05 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:05 smithi154 ceph-mon[36314]: Deploying daemon osd.3 on smithi028 2024-05-20T19:56:05.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:05 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T19:56:05.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:05 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:05.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:05 smithi033 ceph-mon[36179]: Deploying daemon osd.3 on smithi028 2024-05-20T19:56:05.712 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:05 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T19:56:05.712 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:05 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:05.712 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:05 smithi028 ceph-mon[31178]: Deploying daemon osd.3 on smithi028 2024-05-20T19:56:06.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:06 smithi028 ceph-mon[31178]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:06 smithi154 ceph-mon[36314]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:06.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:06 smithi033 ceph-mon[36179]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:08.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:08 smithi028 ceph-mon[31178]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:08.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:08 smithi154 ceph-mon[36314]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:08.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:08 smithi033 ceph-mon[36179]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:10.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:10 smithi033 ceph-mon[36179]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:10.697 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:10 smithi028 ceph-mon[31178]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:10 smithi154 ceph-mon[36314]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:12.677 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:12 smithi154 ceph-mon[36314]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:12.754 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:12 smithi028 ceph-mon[31178]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:12.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:12 smithi033 ceph-mon[36179]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:14.548 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:14.549 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:14 smithi028 ceph-mon[31178]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:14.549 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:14.549 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:14.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:14.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:14 smithi154 ceph-mon[36314]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:14.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:14.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:14.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:14.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:14 smithi033 ceph-mon[36179]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:14.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:14.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:16 smithi154 ceph-mon[36314]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:16 smithi033 ceph-mon[36179]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:16.634 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:16 smithi028 ceph-mon[31178]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:18.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:18.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:18.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:17 smithi028 ceph-mon[31178]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T19:56:18.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:17 smithi028 ceph-mon[31178]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:18.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.274+0000 7f7e1f400700 1 -- 172.21.15.28:0/2543260268 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f7e28063980 con 0x7f7e0c0b2900 2024-05-20T19:56:18.275 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 3 on host 'smithi028' 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7e0c0b2900 msgr2=0x7f7e0c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7e0c0b2900 0x7f7e0c0b4dc0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f7e18006130 tx=0x7f7e180060c0 comp rx=0 tx=0).stop 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 msgr2=0x7f7e281a0280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e281a0280 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f7e1000ea50 tx=0x7f7e1000ed60 comp rx=0 tx=0).stop 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 shutdown_connections 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7e0c0b2900 0x7f7e0c0b4dc0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7e2810fb90 0x7f7e281a72c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7e28075fa0 0x7f7e2819fd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:18.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 --2- 172.21.15.28:0/2543260268 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7e28077570 0x7f7e281a0280 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:18.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 >> 172.21.15.28:0/2543260268 conn(0x7f7e280feb10 msgr2=0x7f7e28102f30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:56:18.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.278+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 shutdown_connections 2024-05-20T19:56:18.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:56:18.279+0000 7f7e2cc00700 1 -- 172.21.15.28:0/2543260268 wait complete. 2024-05-20T19:56:18.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:18.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:18.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:17 smithi154 ceph-mon[36314]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T19:56:18.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:17 smithi154 ceph-mon[36314]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:18.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:18.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:18.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:18.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:17 smithi033 ceph-mon[36179]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T19:56:18.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:17 smithi033 ceph-mon[36179]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:19.067 DEBUG:teuthology.orchestra.run.smithi028:osd.3> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.3.service 2024-05-20T19:56:19.073 INFO:tasks.cephadm:Deploying osd.4 on smithi033 with /dev/vg_nvme/lv_3... 2024-05-20T19:56:19.073 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-20T19:56:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:19 smithi028 ceph-mon[31178]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T19:56:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:19 smithi028 ceph-mon[31178]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T19:56:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:19 smithi028 ceph-mon[31178]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:56:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:19.326 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:56:19.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:19 smithi154 ceph-mon[36314]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T19:56:19.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:19 smithi154 ceph-mon[36314]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T19:56:19.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:19 smithi154 ceph-mon[36314]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:56:19.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:19.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:19.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:19.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:19 smithi033 ceph-mon[36179]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T19:56:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:19 smithi033 ceph-mon[36179]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T19:56:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:19 smithi033 ceph-mon[36179]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:56:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:19.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:20.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:20 smithi028 ceph-mon[31178]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:56:20.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:20 smithi028 ceph-mon[31178]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T19:56:20.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:20.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:20.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:20 smithi028 ceph-mon[31178]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:20.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:20 smithi154 ceph-mon[36314]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:56:20.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:20 smithi154 ceph-mon[36314]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T19:56:20.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:20.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:20.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:20 smithi154 ceph-mon[36314]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:20.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:20 smithi033 ceph-mon[36179]: from='osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:56:20.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:20 smithi033 ceph-mon[36179]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T19:56:20.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:20.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:20.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:20 smithi033 ceph-mon[36179]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-05-20T19:56:21.206 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-20T19:56:21.206 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-20T19:56:21.206 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-05-20T19:56:21.206 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-05-20T19:56:21.206 INFO:teuthology.orchestra.run.smithi033.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111943 s, 937 MB/s 2024-05-20T19:56:21.206 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-05-20T19:56:21.255 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:21 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:56:21.256 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:21 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:56:21.256 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:21 smithi028 ceph-mon[31178]: osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] boot 2024-05-20T19:56:21.256 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:21 smithi028 ceph-mon[31178]: osdmap e25: 4 total, 4 up, 4 in 2024-05-20T19:56:21.256 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:21.256 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:21 smithi028 ceph-mon[31178]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T19:56:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:21 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:56:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:21 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:56:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:21 smithi154 ceph-mon[36314]: osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] boot 2024-05-20T19:56:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:21 smithi154 ceph-mon[36314]: osdmap e25: 4 total, 4 up, 4 in 2024-05-20T19:56:21.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:21.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:21 smithi154 ceph-mon[36314]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T19:56:21.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:21 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:56:21.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:21 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:56:21.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:21 smithi033 ceph-mon[36179]: osd.3 [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] boot 2024-05-20T19:56:21.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:21 smithi033 ceph-mon[36179]: osdmap e25: 4 total, 4 up, 4 in 2024-05-20T19:56:21.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T19:56:21.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:21 smithi033 ceph-mon[36179]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T19:56:21.911 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_3 2024-05-20T19:56:22.157 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:56:22.280 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:22 smithi033 ceph-mon[36179]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T19:56:22.280 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:22 smithi033 ceph-mon[36179]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 106 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:22.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:22 smithi028 ceph-mon[31178]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T19:56:22.302 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:22 smithi028 ceph-mon[31178]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 106 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:22.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:22 smithi154 ceph-mon[36314]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T19:56:22.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:22 smithi154 ceph-mon[36314]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 106 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:23.821 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.818+0000 7fa590673700 1 -- 172.21.15.33:0/587484970 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588106980 msgr2=0x7fa588108d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:23.821 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.818+0000 7fa590673700 1 --2- 172.21.15.33:0/587484970 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588106980 0x7fa588108d70 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fa584009990 tx=0x7fa584009ca0 comp rx=0 tx=0).stop 2024-05-20T19:56:23.821 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 -- 172.21.15.33:0/587484970 shutdown_connections 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 --2- 172.21.15.33:0/587484970 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5881092b0 0x7fa58810b720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 --2- 172.21.15.33:0/587484970 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa588104050 0x7fa588106440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 --2- 172.21.15.33:0/587484970 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588106980 0x7fa588108d70 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 -- 172.21.15.33:0/587484970 >> 172.21.15.33:0/587484970 conn(0x7fa5880fda60 msgr2=0x7fa5880ffea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 -- 172.21.15.33:0/587484970 shutdown_connections 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 -- 172.21.15.33:0/587484970 wait complete. 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 Processor -- start 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.819+0000 7fa590673700 1 -- start start 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa590673700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588104050 0x7fa5881039b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:23.822 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa590673700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa588106980 0x7fa588102000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa590673700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5881092b0 0x7fa588102540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa590673700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa58810f130 con 0x7fa588106980 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa590673700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa58810efb0 con 0x7fa588104050 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa590673700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa58810f2b0 con 0x7fa5881092b0 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588104050 0x7fa5881039b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588104050 0x7fa5881039b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.33:49456/0 (socket says 172.21.15.33:49456) 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 -- 172.21.15.33:0/1042681068 learned_addr learned my addr 172.21.15.33:0/1042681068 (peer_addr_for_me v2:172.21.15.33:0/0) 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 -- 172.21.15.33:0/1042681068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5881092b0 msgr2=0x7fa588102540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5881092b0 0x7fa588102540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 -- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa588106980 msgr2=0x7fa588102000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa588106980 0x7fa588102000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 -- 172.21.15.33:0/1042681068 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa588102df0 con 0x7fa588104050 2024-05-20T19:56:23.823 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa58e40f700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588104050 0x7fa5881039b0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fa57c00b530 tx=0x7fa57c00b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:56:23.824 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa57b7fe700 1 -- 172.21.15.33:0/1042681068 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa57c015020 con 0x7fa588104050 2024-05-20T19:56:23.824 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa57b7fe700 1 -- 172.21.15.33:0/1042681068 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa57c011ce0 con 0x7fa588104050 2024-05-20T19:56:23.824 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa5881a3920 con 0x7fa588104050 2024-05-20T19:56:23.824 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.820+0000 7fa57b7fe700 1 -- 172.21.15.33:0/1042681068 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa57c010d20 con 0x7fa588104050 2024-05-20T19:56:23.824 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.821+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa5881a3eb0 con 0x7fa588104050 2024-05-20T19:56:23.825 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.821+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa58806a230 con 0x7fa588104050 2024-05-20T19:56:23.825 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.823+0000 7fa57b7fe700 1 -- 172.21.15.33:0/1042681068 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fa57c018020 con 0x7fa588104050 2024-05-20T19:56:23.825 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.824+0000 7fa57b7fe700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa5740b2880 0x7fa5740b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:23.825 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.824+0000 7fa57b7fe700 1 -- 172.21.15.33:0/1042681068 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(27..27 src has 1..27) ==== 3669+0+0 (secure 0 0 0) 0x7fa57c0e0e40 con 0x7fa588104050 2024-05-20T19:56:23.825 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.824+0000 7fa58dc0e700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa5740b2880 0x7fa5740b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:56:23.826 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.825+0000 7fa58dc0e700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa5740b2880 0x7fa5740b4d40 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa584003790 tx=0x7fa584003860 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:56:23.828 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:23.827+0000 7fa57b7fe700 1 -- 172.21.15.33:0/1042681068 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa57c01f020 con 0x7fa588104050 2024-05-20T19:56:24.051 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:24.050+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7fa588063980 con 0x7fa5740b2880 2024-05-20T19:56:24.542 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:24 smithi028 ceph-mon[31178]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:24.543 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:56:24.543 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:56:24.543 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:24 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:24.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:24 smithi154 ceph-mon[36314]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:24.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:56:24.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:56:24.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:24 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:24.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:24 smithi033 ceph-mon[36179]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:24.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:56:24.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:56:24.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:24 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:25.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:25 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:56:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:25.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:25.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:25 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='client.24233 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:25.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:25 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:26.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:26 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi028 2024-05-20T19:56:26.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:26 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi028 to 8547M 2024-05-20T19:56:26.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:26 smithi154 ceph-mon[36314]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T19:56:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:26 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi028 2024-05-20T19:56:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:26 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi028 to 8547M 2024-05-20T19:56:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:26 smithi033 ceph-mon[36179]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T19:56:26.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:26 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi028 2024-05-20T19:56:26.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:26 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi028 to 8547M 2024-05-20T19:56:26.665 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:26 smithi028 ceph-mon[31178]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T19:56:27.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:27 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/1873489632' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]: dispatch 2024-05-20T19:56:27.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:27 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]: dispatch 2024-05-20T19:56:27.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:27 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]': finished 2024-05-20T19:56:27.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:27 smithi154 ceph-mon[36314]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T19:56:27.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:27 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:27.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:27 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/1873489632' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]: dispatch 2024-05-20T19:56:27.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:27 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]: dispatch 2024-05-20T19:56:27.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:27 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]': finished 2024-05-20T19:56:27.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:27 smithi033 ceph-mon[36179]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T19:56:27.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:27 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:27.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:27 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/1873489632' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]: dispatch 2024-05-20T19:56:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:27 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]: dispatch 2024-05-20T19:56:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:27 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57abdf0f-f317-465a-99d5-1d761d481da1"}]': finished 2024-05-20T19:56:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:27 smithi028 ceph-mon[31178]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T19:56:27.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:27 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:28.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:28 smithi154 ceph-mon[36314]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-05-20T19:56:28.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:28 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/2685290087' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:56:28.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:28 smithi033 ceph-mon[36179]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-05-20T19:56:28.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:28 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/2685290087' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:56:28.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:28 smithi028 ceph-mon[31178]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-05-20T19:56:28.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:28 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/2685290087' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:56:30.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:30 smithi154 ceph-mon[36314]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T19:56:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:30 smithi033 ceph-mon[36179]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T19:56:30.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:30 smithi028 ceph-mon[31178]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T19:56:32.532 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:32 smithi033 ceph-mon[36179]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:56:32.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:32 smithi154 ceph-mon[36314]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:56:32.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:32 smithi028 ceph-mon[31178]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:56:34.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:34 smithi154 ceph-mon[36314]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:56:34.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:34 smithi033 ceph-mon[36179]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:56:34.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:34 smithi028 ceph-mon[31178]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:56:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:35 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T19:56:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:35 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:35 smithi154 ceph-mon[36314]: Deploying daemon osd.4 on smithi033 2024-05-20T19:56:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:35 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T19:56:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:35 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:35 smithi033 ceph-mon[36179]: Deploying daemon osd.4 on smithi033 2024-05-20T19:56:35.638 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:35 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T19:56:35.638 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:35 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:35.638 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:35 smithi028 ceph-mon[31178]: Deploying daemon osd.4 on smithi033 2024-05-20T19:56:36.539 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:36 smithi033 ceph-mon[36179]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:36.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:36 smithi154 ceph-mon[36314]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:36.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:36 smithi028 ceph-mon[31178]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:38.587 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:38 smithi033 ceph-mon[36179]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:38 smithi028 ceph-mon[31178]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:38 smithi154 ceph-mon[36314]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:40 smithi028 ceph-mon[31178]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:40.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:40.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:40 smithi154 ceph-mon[36314]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:40 smithi033 ceph-mon[36179]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:42.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:42 smithi033 ceph-mon[36179]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:42.677 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:42 smithi154 ceph-mon[36314]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:42 smithi028 ceph-mon[31178]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:43.582 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 4 on host 'smithi033' 2024-05-20T19:56:43.583 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.582+0000 7fa57b7fe700 1 -- 172.21.15.33:0/1042681068 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fa588063980 con 0x7fa5740b2880 2024-05-20T19:56:43.585 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa5740b2880 msgr2=0x7fa5740b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:43.585 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa5740b2880 0x7fa5740b4d40 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa584003790 tx=0x7fa584003860 comp rx=0 tx=0).stop 2024-05-20T19:56:43.585 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588104050 msgr2=0x7fa5881039b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:43.585 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588104050 0x7fa5881039b0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fa57c00b530 tx=0x7fa57c00b8f0 comp rx=0 tx=0).stop 2024-05-20T19:56:43.585 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 shutdown_connections 2024-05-20T19:56:43.586 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa5740b2880 0x7fa5740b4d40 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:43.586 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5881092b0 0x7fa588102540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:43.586 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa588104050 0x7fa5881039b0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:43.586 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 --2- 172.21.15.33:0/1042681068 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa588106980 0x7fa588102000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:43.586 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 >> 172.21.15.33:0/1042681068 conn(0x7fa5880fda60 msgr2=0x7fa588104990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:56:43.586 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 shutdown_connections 2024-05-20T19:56:43.586 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:56:43.584+0000 7fa590673700 1 -- 172.21.15.33:0/1042681068 wait complete. 2024-05-20T19:56:44.422 DEBUG:teuthology.orchestra.run.smithi033:osd.4> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.4.service 2024-05-20T19:56:44.425 INFO:tasks.cephadm:Deploying osd.5 on smithi154 with /dev/vg_nvme/lv_3... 2024-05-20T19:56:44.425 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-20T19:56:44.673 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:56:44.794 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:44 smithi154 ceph-mon[36314]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:44.794 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:44.795 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:44.795 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:44 smithi154 ceph-mon[36314]: from='osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T19:56:44.795 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:44 smithi154 ceph-mon[36314]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T19:56:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:44 smithi028 ceph-mon[31178]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:44 smithi028 ceph-mon[31178]: from='osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T19:56:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:44 smithi028 ceph-mon[31178]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T19:56:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:44 smithi033 ceph-mon[36179]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:44 smithi033 ceph-mon[36179]: from='osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T19:56:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:44 smithi033 ceph-mon[36179]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T19:56:45.868 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:45 smithi154 ceph-mon[36314]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T19:56:45.868 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:45 smithi154 ceph-mon[36314]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T19:56:45.868 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:45.868 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:45 smithi154 ceph-mon[36314]: from='osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:56:45.868 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:45 smithi154 ceph-mon[36314]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:56:45.868 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:45.868 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:45 smithi033 ceph-mon[36179]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T19:56:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:45 smithi033 ceph-mon[36179]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T19:56:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:45 smithi033 ceph-mon[36179]: from='osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:56:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:45 smithi033 ceph-mon[36179]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:56:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:45.878 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 19:56:45 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T19:56:45.562+0000 7f63e0a01700 -1 osd.4 0 waiting for initial osdmap 2024-05-20T19:56:45.878 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 19:56:45 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T19:56:45.569+0000 7f63dc7f4700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:56:46.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:45 smithi028 ceph-mon[31178]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T19:56:46.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:45 smithi028 ceph-mon[31178]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T19:56:46.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:46.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:45 smithi028 ceph-mon[31178]: from='osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:56:46.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:45 smithi028 ceph-mon[31178]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:56:46.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:46.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:46.724 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:46 smithi154 ceph-mon[36314]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:46.724 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:46 smithi154 ceph-mon[36314]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:56:46.724 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:46 smithi154 ceph-mon[36314]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T19:56:46.725 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:46.725 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:46.725 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-20T19:56:46.725 INFO:teuthology.orchestra.run.smithi154.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-20T19:56:46.725 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10+0 records in 2024-05-20T19:56:46.725 INFO:teuthology.orchestra.run.smithi154.stderr:10+0 records out 2024-05-20T19:56:46.725 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113363 s, 925 MB/s 2024-05-20T19:56:46.725 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping successful for: 2024-05-20T19:56:46.854 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:46 smithi033 ceph-mon[36179]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:46.854 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:46 smithi033 ceph-mon[36179]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:56:46.854 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:46 smithi033 ceph-mon[36179]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T19:56:46.854 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:46.854 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:46 smithi028 ceph-mon[31178]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:46 smithi028 ceph-mon[31178]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:56:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:46 smithi028 ceph-mon[31178]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T19:56:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:47.295 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_3 2024-05-20T19:56:47.536 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:56:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:47 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:56:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:47 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:56:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:47 smithi028 ceph-mon[31178]: osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] boot 2024-05-20T19:56:47.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:47 smithi028 ceph-mon[31178]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T19:56:47.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:47.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:47 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:56:47.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:47 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:56:47.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:47 smithi154 ceph-mon[36314]: osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] boot 2024-05-20T19:56:47.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:47 smithi154 ceph-mon[36314]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T19:56:47.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:47 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:56:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:47 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:56:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:47 smithi033 ceph-mon[36179]: osd.4 [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] boot 2024-05-20T19:56:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:47 smithi033 ceph-mon[36179]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T19:56:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:48.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:48.790 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:48.790 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:48.791 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:49.042 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 -- 172.21.15.154:0/4188536274 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 msgr2=0x7f40cc1028d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:49.042 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 --2- 172.21.15.154:0/4188536274 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1028d0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f40c0009990 tx=0x7f40c0009ca0 comp rx=0 tx=0).stop 2024-05-20T19:56:49.042 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 -- 172.21.15.154:0/4188536274 shutdown_connections 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 --2- 172.21.15.154:0/4188536274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f40cc104050 0x7f40cc10a900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 --2- 172.21.15.154:0/4188536274 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1028d0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 --2- 172.21.15.154:0/4188536274 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f40cc1036b0 0x7f40cc103b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 -- 172.21.15.154:0/4188536274 >> 172.21.15.154:0/4188536274 conn(0x7f40cc0fda70 msgr2=0x7f40cc0ffe90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 -- 172.21.15.154:0/4188536274 shutdown_connections 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 -- 172.21.15.154:0/4188536274 wait complete. 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 Processor -- start 2024-05-20T19:56:49.043 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.039+0000 7f40d322c700 1 -- start start 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d322c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1063f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d322c700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f40cc1036b0 0x7f40cc104a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d322c700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f40cc104050 0x7f40cc104fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d322c700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f40cc10e020 con 0x7f40cc104050 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d322c700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f40cc10dea0 con 0x7f40cc1036b0 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d322c700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f40cc10e1a0 con 0x7f40cc1024b0 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1063f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1063f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:35646/0 (socket says 172.21.15.154:35646) 2024-05-20T19:56:49.044 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 -- 172.21.15.154:0/3809829512 learned_addr learned my addr 172.21.15.154:0/3809829512 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40cbfff700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f40cc1036b0 0x7f40cc104a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 -- 172.21.15.154:0/3809829512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f40cc1036b0 msgr2=0x7f40cc104a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f40cc1036b0 0x7f40cc104a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 -- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f40cc104050 msgr2=0x7f40cc104fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f40cc104050 0x7f40cc104fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 -- 172.21.15.154:0/3809829512 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f40cc1058a0 con 0x7f40cc1024b0 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40cbfff700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f40cc1036b0 0x7f40cc104a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d0fc8700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1063f0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f40c0000c00 tx=0x7f40c0009f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40c9ffb700 1 -- 172.21.15.154:0/3809829512 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f40c0027070 con 0x7f40cc1024b0 2024-05-20T19:56:49.045 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40c9ffb700 1 -- 172.21.15.154:0/3809829512 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f40c0004080 con 0x7f40cc1024b0 2024-05-20T19:56:49.046 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f40cc1a3990 con 0x7f40cc1024b0 2024-05-20T19:56:49.046 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.040+0000 7f40c9ffb700 1 -- 172.21.15.154:0/3809829512 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f40c0022d40 con 0x7f40cc1024b0 2024-05-20T19:56:49.046 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.041+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f40cc1a3e40 con 0x7f40cc1024b0 2024-05-20T19:56:49.046 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.042+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f40cc06a230 con 0x7f40cc1024b0 2024-05-20T19:56:49.047 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.043+0000 7f40c9ffb700 1 -- 172.21.15.154:0/3809829512 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f40c002a070 con 0x7f40cc1024b0 2024-05-20T19:56:49.047 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.043+0000 7f40c9ffb700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f40bc0b2880 0x7f40bc0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:56:49.047 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.044+0000 7f40c9ffb700 1 -- 172.21.15.154:0/3809829512 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 4101+0+0 (secure 0 0 0) 0x7f40c00e2230 con 0x7f40cc1024b0 2024-05-20T19:56:49.047 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.044+0000 7f40cbfff700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f40bc0b2880 0x7f40bc0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:56:49.047 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.045+0000 7f40cbfff700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f40bc0b2880 0x7f40bc0b4d40 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f40b800bcf0 tx=0x7f40b800b2c0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:56:49.048 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.048+0000 7f40c9ffb700 1 -- 172.21.15.154:0/3809829512 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f40c0013070 con 0x7f40cc1024b0 2024-05-20T19:56:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 107 MiB used, 358 GiB / 358 GiB avail 2024-05-20T19:56:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T19:56:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:56:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:49.262 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:56:49.262+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) -- 0x7f40cc06c410 con 0x7f40bc0b2880 2024-05-20T19:56:49.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi033 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi033 to 8546M 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:56:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi033 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi033 to 8546M 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:56:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:56:49.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:50.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi033 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi033 to 8546M 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:56:50.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:56:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:50 smithi154 ceph-mon[36314]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:56:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:50 smithi154 ceph-mon[36314]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:50 smithi033 ceph-mon[36179]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:56:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:50 smithi033 ceph-mon[36179]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:51.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:50 smithi028 ceph-mon[31178]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:56:51.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:50 smithi028 ceph-mon[31178]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:52 smithi154 ceph-mon[36314]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:52.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:52 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/1655048082' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]: dispatch 2024-05-20T19:56:52.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:52 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]: dispatch 2024-05-20T19:56:52.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:52 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]': finished 2024-05-20T19:56:52.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:52 smithi154 ceph-mon[36314]: osdmap e33: 6 total, 5 up, 6 in 2024-05-20T19:56:52.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:52 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:56:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:52 smithi033 ceph-mon[36179]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:52 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/1655048082' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]: dispatch 2024-05-20T19:56:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:52 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]: dispatch 2024-05-20T19:56:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:52 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]': finished 2024-05-20T19:56:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:52 smithi033 ceph-mon[36179]: osdmap e33: 6 total, 5 up, 6 in 2024-05-20T19:56:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:52 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:56:53.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:52 smithi028 ceph-mon[31178]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:53.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:52 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/1655048082' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]: dispatch 2024-05-20T19:56:53.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:52 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]: dispatch 2024-05-20T19:56:53.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:52 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1"}]': finished 2024-05-20T19:56:53.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:52 smithi028 ceph-mon[31178]: osdmap e33: 6 total, 5 up, 6 in 2024-05-20T19:56:53.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:52 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:56:53.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:53 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/2692864331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:56:53.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:53 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/2692864331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:56:54.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:53 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/2692864331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:56:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:54 smithi154 ceph-mon[36314]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:54 smithi033 ceph-mon[36179]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:55.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:54 smithi028 ceph-mon[31178]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:56.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:56 smithi154 ceph-mon[36314]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:56.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:56 smithi033 ceph-mon[36179]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:57.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:56 smithi028 ceph-mon[31178]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:58.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:58 smithi154 ceph-mon[36314]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:58 smithi033 ceph-mon[36179]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:59.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:58 smithi028 ceph-mon[31178]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:56:59.633 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:59 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T19:56:59.633 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:56:59 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:00.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:59 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T19:57:00.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:56:59 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:00.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:59 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T19:57:00.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:56:59 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:01.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:00 smithi028 ceph-mon[31178]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:01.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:00 smithi028 ceph-mon[31178]: Deploying daemon osd.5 on smithi154 2024-05-20T19:57:01.065 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:00 smithi154 ceph-mon[36314]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:01.065 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:00 smithi154 ceph-mon[36314]: Deploying daemon osd.5 on smithi154 2024-05-20T19:57:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:00 smithi033 ceph-mon[36179]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:00 smithi033 ceph-mon[36179]: Deploying daemon osd.5 on smithi154 2024-05-20T19:57:02.770 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:02 smithi154 ceph-mon[36314]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:03.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:02 smithi028 ceph-mon[31178]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:02 smithi033 ceph-mon[36179]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:04.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:04 smithi154 ceph-mon[36314]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:05.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:04 smithi028 ceph-mon[31178]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:04 smithi033 ceph-mon[36179]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:06.514 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:06.514 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:06.514 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:06.515 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:06 smithi033 ceph-mon[36179]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:06.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:06.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:06.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:06.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:06 smithi028 ceph-mon[31178]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:06.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:06.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:06.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:06 smithi154 ceph-mon[36314]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:08.590 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:08 smithi154 ceph-mon[36314]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:08.590 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.590 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.590 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:08.590 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:08.590 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:08 smithi033 ceph-mon[36179]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:08.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:08.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:08.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:08 smithi028 ceph-mon[31178]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:08.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:08.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:08.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:08.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:09.004 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.003+0000 7f40c9ffb700 1 -- 172.21.15.154:0/3809829512 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f40cc06c410 con 0x7f40bc0b2880 2024-05-20T19:57:09.004 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 5 on host 'smithi154' 2024-05-20T19:57:09.006 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.005+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f40bc0b2880 msgr2=0x7f40bc0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:09.006 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.005+0000 7f40d322c700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f40bc0b2880 0x7f40bc0b4d40 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f40b800bcf0 tx=0x7f40b800b2c0 comp rx=0 tx=0).stop 2024-05-20T19:57:09.006 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.005+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 msgr2=0x7f40cc1063f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:09.006 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.005+0000 7f40d322c700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1063f0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f40c0000c00 tx=0x7f40c0009f90 comp rx=0 tx=0).stop 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 shutdown_connections 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f40bc0b2880 0x7f40bc0b4d40 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f40cc104050 0x7f40cc104fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40cc1024b0 0x7f40cc1063f0 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 --2- 172.21.15.154:0/3809829512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f40cc1036b0 0x7f40cc104a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 >> 172.21.15.154:0/3809829512 conn(0x7f40cc0fda70 msgr2=0x7f40cc06bd00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 shutdown_connections 2024-05-20T19:57:09.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:57:09.006+0000 7f40d322c700 1 -- 172.21.15.154:0/3809829512 wait complete. 2024-05-20T19:57:09.704 DEBUG:teuthology.orchestra.run.smithi154:osd.5> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.5.service 2024-05-20T19:57:09.706 INFO:tasks.cephadm:Deploying osd.6 on smithi028 with /dev/vg_nvme/lv_2... 2024-05-20T19:57:09.706 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-20T19:57:10.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:09 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:10.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:09 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:10.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:09 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:10.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:09 smithi028 ceph-mon[31178]: from='osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T19:57:10.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:09 smithi028 ceph-mon[31178]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T19:57:10.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:09 smithi028 ceph-mon[31178]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:10.172 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:57:10.273 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:09 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:10.273 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:09 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:10.273 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:09 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:10.273 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:09 smithi154 ceph-mon[36314]: from='osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T19:57:10.273 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:09 smithi154 ceph-mon[36314]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T19:57:10.273 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:09 smithi154 ceph-mon[36314]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:10.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:09 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:10.382 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:09 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:10.382 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:09 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:10.383 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:09 smithi033 ceph-mon[36179]: from='osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T19:57:10.383 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:09 smithi033 ceph-mon[36179]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T19:57:10.383 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:09 smithi033 ceph-mon[36179]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:11.196 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:10 smithi154 ceph-mon[36314]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T19:57:11.196 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:10 smithi154 ceph-mon[36314]: from='osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:57:11.196 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:10 smithi154 ceph-mon[36314]: osdmap e34: 6 total, 5 up, 6 in 2024-05-20T19:57:11.196 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:10 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:11.196 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:10 smithi154 ceph-mon[36314]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:57:11.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:10 smithi028 ceph-mon[31178]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T19:57:11.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:10 smithi028 ceph-mon[31178]: from='osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:57:11.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:10 smithi028 ceph-mon[31178]: osdmap e34: 6 total, 5 up, 6 in 2024-05-20T19:57:11.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:10 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:11.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:10 smithi028 ceph-mon[31178]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:57:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:10 smithi033 ceph-mon[36179]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T19:57:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:10 smithi033 ceph-mon[36179]: from='osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:57:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:10 smithi033 ceph-mon[36179]: osdmap e34: 6 total, 5 up, 6 in 2024-05-20T19:57:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:10 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:10 smithi033 ceph-mon[36179]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:57:11.448 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 19:57:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T19:57:11.408+0000 7feb48a0c700 -1 osd.5 0 waiting for initial osdmap 2024-05-20T19:57:11.448 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 19:57:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T19:57:11.416+0000 7feb407f7700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:57:12.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:11 smithi028 ceph-mon[31178]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:57:12.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:11 smithi028 ceph-mon[31178]: osdmap e35: 6 total, 5 up, 6 in 2024-05-20T19:57:12.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:11 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:12.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:11 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:12.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:11 smithi028 ceph-mon[31178]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:12.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:11 smithi028 ceph-mon[31178]: from='osd.5 ' entity='osd.5' 2024-05-20T19:57:12.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:11 smithi154 ceph-mon[36314]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:57:12.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:11 smithi154 ceph-mon[36314]: osdmap e35: 6 total, 5 up, 6 in 2024-05-20T19:57:12.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:11 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:12.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:11 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:12.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:11 smithi154 ceph-mon[36314]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:12.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:11 smithi154 ceph-mon[36314]: from='osd.5 ' entity='osd.5' 2024-05-20T19:57:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:11 smithi033 ceph-mon[36179]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:57:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:11 smithi033 ceph-mon[36179]: osdmap e35: 6 total, 5 up, 6 in 2024-05-20T19:57:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:11 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:11 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:11 smithi033 ceph-mon[36179]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 133 MiB used, 447 GiB / 447 GiB avail 2024-05-20T19:57:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:11 smithi033 ceph-mon[36179]: from='osd.5 ' entity='osd.5' 2024-05-20T19:57:13.232 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-20T19:57:13.232 INFO:teuthology.orchestra.run.smithi028.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-20T19:57:13.232 INFO:teuthology.orchestra.run.smithi028.stderr: stderr: 10+0 records in 2024-05-20T19:57:13.233 INFO:teuthology.orchestra.run.smithi028.stderr:10+0 records out 2024-05-20T19:57:13.233 INFO:teuthology.orchestra.run.smithi028.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0373866 s, 280 MB/s 2024-05-20T19:57:13.233 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping successful for: 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] boot 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: osdmap e36: 6 total, 6 up, 6 in 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:13.273 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:13.274 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.274 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:13.274 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:13.274 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.274 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:12 smithi028 ceph-mon[31178]: osdmap e37: 6 total, 6 up, 6 in 2024-05-20T19:57:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:57:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:57:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] boot 2024-05-20T19:57:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: osdmap e36: 6 total, 6 up, 6 in 2024-05-20T19:57:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:12 smithi154 ceph-mon[36314]: osdmap e37: 6 total, 6 up, 6 in 2024-05-20T19:57:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:57:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:57:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: osd.5 [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] boot 2024-05-20T19:57:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: osdmap e36: 6 total, 6 up, 6 in 2024-05-20T19:57:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:13.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:12 smithi033 ceph-mon[36179]: osdmap e37: 6 total, 6 up, 6 in 2024-05-20T19:57:14.129 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_2 2024-05-20T19:57:14.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:14 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi154 2024-05-20T19:57:14.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:14 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi154 to 8544M 2024-05-20T19:57:14.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:14 smithi028 ceph-mon[31178]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:14.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:14 smithi028 ceph-mon[31178]: osdmap e38: 6 total, 6 up, 6 in 2024-05-20T19:57:14.354 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:14 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi154 2024-05-20T19:57:14.355 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:14 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi154 to 8544M 2024-05-20T19:57:14.355 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:14 smithi154 ceph-mon[36314]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:14.355 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:14 smithi154 ceph-mon[36314]: osdmap e38: 6 total, 6 up, 6 in 2024-05-20T19:57:14.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:14 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi154 2024-05-20T19:57:14.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:14 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi154 to 8544M 2024-05-20T19:57:14.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:14 smithi033 ceph-mon[36179]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:14.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:14 smithi033 ceph-mon[36179]: osdmap e38: 6 total, 6 up, 6 in 2024-05-20T19:57:14.618 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:57:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:16 smithi154 ceph-mon[36314]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:16 smithi033 ceph-mon[36179]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:16.684 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:16 smithi028 ceph-mon[31178]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:16.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 -- 172.21.15.28:0/2434673385 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d41091c0 msgr2=0x7f24d410b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:16.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2434673385 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d41091c0 0x7f24d410b630 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f24cc009f90 tx=0x7f24cc00b330 comp rx=0 tx=0).stop 2024-05-20T19:57:16.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 -- 172.21.15.28:0/2434673385 shutdown_connections 2024-05-20T19:57:16.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2434673385 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d41091c0 0x7f24d410b630 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:16.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2434673385 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24d4103f60 0x7f24d4106350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:16.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2434673385 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24d4106890 0x7f24d4108c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:16.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 -- 172.21.15.28:0/2434673385 >> 172.21.15.28:0/2434673385 conn(0x7f24d40fd950 msgr2=0x7f24d40ffdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:57:16.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 -- 172.21.15.28:0/2434673385 shutdown_connections 2024-05-20T19:57:16.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.860+0000 7f24d3e00700 1 -- 172.21.15.28:0/2434673385 wait complete. 2024-05-20T19:57:16.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.861+0000 7f24d3e00700 1 Processor -- start 2024-05-20T19:57:16.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.861+0000 7f24d3e00700 1 -- start start 2024-05-20T19:57:16.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d4103f60 0x7f24d4103870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:16.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24d4106890 0x7f24d4101f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:16.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24d41091c0 0x7f24d4102450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d3e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f24d410ef80 con 0x7f24d4106890 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d3e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f24d410ee00 con 0x7f24d4103f60 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d3e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f24d410f100 con 0x7f24d41091c0 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d2000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24d4106890 0x7f24d4101f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d4103f60 0x7f24d4103870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d2000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24d4106890 0x7f24d4101f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36672/0 (socket says 172.21.15.28:36672) 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24d41091c0 0x7f24d4102450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d4103f60 0x7f24d4103870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:35828/0 (socket says 172.21.15.28:35828) 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.862+0000 7f24d2000700 1 -- 172.21.15.28:0/2578996091 learned_addr learned my addr 172.21.15.28:0/2578996091 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d2a00700 1 -- 172.21.15.28:0/2578996091 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24d41091c0 msgr2=0x7f24d4102450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:16.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d2a00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24d41091c0 0x7f24d4102450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d2a00700 1 -- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24d4106890 msgr2=0x7f24d4101f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d2a00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24d4106890 0x7f24d4101f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d2a00700 1 -- 172.21.15.28:0/2578996091 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f24d4102d20 con 0x7f24d4103f60 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d3400700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24d41091c0 0x7f24d4102450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d2a00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d4103f60 0x7f24d4103870 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f24c400e9c0 tx=0x7f24c400ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24cb400700 1 -- 172.21.15.28:0/2578996091 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f24c400c9c0 con 0x7f24d4103f60 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f24d41a36f0 con 0x7f24d4103f60 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f24d41a3c50 con 0x7f24d4103f60 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24cb400700 1 -- 172.21.15.28:0/2578996091 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f24c400cb60 con 0x7f24d4103f60 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.863+0000 7f24cb400700 1 -- 172.21.15.28:0/2578996091 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f24c4018420 con 0x7f24d4103f60 2024-05-20T19:57:16.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.865+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f24d406a230 con 0x7f24d4103f60 2024-05-20T19:57:16.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.868+0000 7f24cb400700 1 -- 172.21.15.28:0/2578996091 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f24c4010430 con 0x7f24d4103f60 2024-05-20T19:57:16.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.869+0000 7f24cb400700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f24b80b2900 0x7f24b80b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:16.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.869+0000 7f24cb400700 1 -- 172.21.15.28:0/2578996091 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(38..38 src has 1..38) ==== 4533+0+0 (secure 0 0 0) 0x7f24c40e1510 con 0x7f24d4103f60 2024-05-20T19:57:16.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.869+0000 7f24d2000700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f24b80b2900 0x7f24b80b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:57:16.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.869+0000 7f24d2000700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f24b80b2900 0x7f24b80b4dc0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f24c0009d20 tx=0x7f24c0009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:57:16.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:16.872+0000 7f24cb400700 1 -- 172.21.15.28:0/2578996091 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f24c40aa050 con 0x7f24d4103f60 2024-05-20T19:57:17.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:17.123+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7f24d402e990 con 0x7f24b80b2900 2024-05-20T19:57:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:57:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:57:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:57:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:57:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:17.645 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:57:17.646 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:57:17.646 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:18.452 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:18 smithi028 ceph-mon[31178]: from='client.24281 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:57:18.452 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:18 smithi028 ceph-mon[31178]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 164 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:18.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:18 smithi154 ceph-mon[36314]: from='client.24281 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:57:18.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:18 smithi154 ceph-mon[36314]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 164 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:18.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:18 smithi033 ceph-mon[36179]: from='client.24281 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:57:18.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:18 smithi033 ceph-mon[36179]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 164 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:20 smithi154 ceph-mon[36314]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-20T19:57:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:20 smithi033 ceph-mon[36179]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-20T19:57:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:20 smithi028 ceph-mon[31178]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-05-20T19:57:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:21 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2962111998' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb2acac-f855-46cf-8255-833f111bdddd"}]: dispatch 2024-05-20T19:57:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:21 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2962111998' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb2acac-f855-46cf-8255-833f111bdddd"}]': finished 2024-05-20T19:57:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:21 smithi154 ceph-mon[36314]: osdmap e39: 7 total, 6 up, 7 in 2024-05-20T19:57:21.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:21 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2962111998' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb2acac-f855-46cf-8255-833f111bdddd"}]: dispatch 2024-05-20T19:57:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:21 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2962111998' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb2acac-f855-46cf-8255-833f111bdddd"}]': finished 2024-05-20T19:57:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:21 smithi033 ceph-mon[36179]: osdmap e39: 7 total, 6 up, 7 in 2024-05-20T19:57:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:21.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:21 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2962111998' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1fb2acac-f855-46cf-8255-833f111bdddd"}]: dispatch 2024-05-20T19:57:21.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:21 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2962111998' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1fb2acac-f855-46cf-8255-833f111bdddd"}]': finished 2024-05-20T19:57:21.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:21 smithi028 ceph-mon[31178]: osdmap e39: 7 total, 6 up, 7 in 2024-05-20T19:57:21.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:22 smithi154 ceph-mon[36314]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:57:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:22 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/844852192' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:57:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:22 smithi033 ceph-mon[36179]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:57:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:22 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/844852192' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:57:22.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:22 smithi028 ceph-mon[31178]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:57:22.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:22 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/844852192' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:57:24.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:24 smithi154 ceph-mon[36314]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-05-20T19:57:24.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:24 smithi033 ceph-mon[36179]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-05-20T19:57:24.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:24 smithi028 ceph-mon[31178]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-05-20T19:57:26.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:26 smithi028 ceph-mon[31178]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:57:26.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:26 smithi154 ceph-mon[36314]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:57:26.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:26 smithi033 ceph-mon[36179]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:57:28.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:28 smithi028 ceph-mon[31178]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:57:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:28 smithi154 ceph-mon[36314]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:57:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:28 smithi033 ceph-mon[36179]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:57:30.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:30 smithi028 ceph-mon[31178]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:30.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:30 smithi154 ceph-mon[36314]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:30 smithi033 ceph-mon[36179]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:31.727 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T19:57:31.727 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:31.727 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:31 smithi028 ceph-mon[31178]: Deploying daemon osd.6 on smithi028 2024-05-20T19:57:31.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T19:57:31.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:31.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:31 smithi154 ceph-mon[36314]: Deploying daemon osd.6 on smithi028 2024-05-20T19:57:31.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T19:57:31.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:31.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:31 smithi033 ceph-mon[36179]: Deploying daemon osd.6 on smithi028 2024-05-20T19:57:32.770 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:32 smithi028 ceph-mon[31178]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:32.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:32 smithi154 ceph-mon[36314]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:32 smithi033 ceph-mon[36179]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:34.773 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:34 smithi028 ceph-mon[31178]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:34.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:34 smithi154 ceph-mon[36314]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:34.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:34 smithi033 ceph-mon[36179]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:36.693 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:36 smithi028 ceph-mon[31178]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:36.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:36 smithi154 ceph-mon[36314]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:36.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:36 smithi033 ceph-mon[36179]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:38.722 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:38 smithi028 ceph-mon[31178]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:38 smithi154 ceph-mon[36314]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:38.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:38 smithi033 ceph-mon[36179]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:40.913 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:40 smithi028 ceph-mon[31178]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:40.914 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:40.914 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:40.914 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:41.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:40 smithi154 ceph-mon[36314]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:41.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:41.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:41.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:40 smithi033 ceph-mon[36179]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:42.928 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:42 smithi154 ceph-mon[36314]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:42.954 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:42 smithi028 ceph-mon[31178]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:42 smithi033 ceph-mon[36179]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:44 smithi028 ceph-mon[31178]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:44 smithi028 ceph-mon[31178]: from='osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T19:57:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:44 smithi028 ceph-mon[31178]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T19:57:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:44 smithi154 ceph-mon[36314]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:44 smithi154 ceph-mon[36314]: from='osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T19:57:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:44 smithi154 ceph-mon[36314]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T19:57:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:44 smithi033 ceph-mon[36179]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:44 smithi033 ceph-mon[36179]: from='osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T19:57:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:44 smithi033 ceph-mon[36179]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T19:57:44.991 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 6 on host 'smithi028' 2024-05-20T19:57:44.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.989+0000 7f24cb400700 1 -- 172.21.15.28:0/2578996091 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f24d402e990 con 0x7f24b80b2900 2024-05-20T19:57:44.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f24b80b2900 msgr2=0x7f24b80b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:44.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f24b80b2900 0x7f24b80b4dc0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f24c0009d20 tx=0x7f24c0009260 comp rx=0 tx=0).stop 2024-05-20T19:57:44.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d4103f60 msgr2=0x7f24d4103870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:44.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d4103f60 0x7f24d4103870 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f24c400e9c0 tx=0x7f24c400ecd0 comp rx=0 tx=0).stop 2024-05-20T19:57:44.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 shutdown_connections 2024-05-20T19:57:44.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f24b80b2900 0x7f24b80b4dc0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:44.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24d41091c0 0x7f24d4102450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:44.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24d4103f60 0x7f24d4103870 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:44.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 --2- 172.21.15.28:0/2578996091 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24d4106890 0x7f24d4101f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:44.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 >> 172.21.15.28:0/2578996091 conn(0x7f24d40fd950 msgr2=0x7f24d4104860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:57:44.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.993+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 shutdown_connections 2024-05-20T19:57:44.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:57:44.994+0000 7f24d3e00700 1 -- 172.21.15.28:0/2578996091 wait complete. 2024-05-20T19:57:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T19:57:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:57:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: osdmap e40: 7 total, 6 up, 7 in 2024-05-20T19:57:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:57:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: osdmap e40: 7 total, 6 up, 7 in 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T19:57:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:57:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: osdmap e40: 7 total, 6 up, 7 in 2024-05-20T19:57:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:57:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:57:45.917 DEBUG:teuthology.orchestra.run.smithi028:osd.6> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.6.service 2024-05-20T19:57:45.922 INFO:tasks.cephadm:Deploying osd.7 on smithi033 with /dev/vg_nvme/lv_2... 2024-05-20T19:57:45.923 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-20T19:57:46.178 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:57:46.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:46 smithi028 ceph-mon[31178]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:46.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:46 smithi028 ceph-mon[31178]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:57:46.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:46 smithi028 ceph-mon[31178]: osdmap e41: 7 total, 6 up, 7 in 2024-05-20T19:57:46.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:46.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:46 smithi154 ceph-mon[36314]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:46 smithi154 ceph-mon[36314]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:57:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:46 smithi154 ceph-mon[36314]: osdmap e41: 7 total, 6 up, 7 in 2024-05-20T19:57:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:46.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:46 smithi033 ceph-mon[36179]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:46 smithi033 ceph-mon[36179]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:57:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:46 smithi033 ceph-mon[36179]: osdmap e41: 7 total, 6 up, 7 in 2024-05-20T19:57:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:47.731 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:47 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:57:47.731 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:47 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:57:47.731 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:47 smithi028 ceph-mon[31178]: osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] boot 2024-05-20T19:57:47.731 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:47 smithi028 ceph-mon[31178]: osdmap e42: 7 total, 7 up, 7 in 2024-05-20T19:57:47.731 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:47.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:47 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:57:47.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:47 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:57:47.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:47 smithi154 ceph-mon[36314]: osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] boot 2024-05-20T19:57:47.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:47 smithi154 ceph-mon[36314]: osdmap e42: 7 total, 7 up, 7 in 2024-05-20T19:57:47.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:47 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:57:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:47 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:57:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:47 smithi033 ceph-mon[36179]: osd.6 [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] boot 2024-05-20T19:57:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:47 smithi033 ceph-mon[36179]: osdmap e42: 7 total, 7 up, 7 in 2024-05-20T19:57:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T19:57:48.142 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-20T19:57:48.142 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-20T19:57:48.142 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-05-20T19:57:48.142 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-05-20T19:57:48.143 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129071 s, 812 MB/s 2024-05-20T19:57:48.143 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-05-20T19:57:48.770 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:48 smithi033 ceph-mon[36179]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:48.770 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:48 smithi033 ceph-mon[36179]: osdmap e43: 7 total, 7 up, 7 in 2024-05-20T19:57:48.799 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_2 2024-05-20T19:57:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:48 smithi154 ceph-mon[36314]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:48 smithi154 ceph-mon[36314]: osdmap e43: 7 total, 7 up, 7 in 2024-05-20T19:57:49.048 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:57:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:48 smithi028 ceph-mon[31178]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-05-20T19:57:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:48 smithi028 ceph-mon[31178]: osdmap e43: 7 total, 7 up, 7 in 2024-05-20T19:57:50.701 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 -- 172.21.15.33:0/1853672838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a410b4a0 msgr2=0x7fd7a410d910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1853672838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a410b4a0 0x7fd7a410d910 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fd79c00b0a0 tx=0x7fd79c00b3b0 comp rx=0 tx=0).stop 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 -- 172.21.15.33:0/1853672838 shutdown_connections 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1853672838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a410b4a0 0x7fd7a410d910 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1853672838 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd7a40774f0 0x7fd7a4075950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1853672838 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd7a4075f20 0x7fd7a40763a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 -- 172.21.15.33:0/1853672838 >> 172.21.15.33:0/1853672838 conn(0x7fd7a40feb40 msgr2=0x7fd7a4100f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 -- 172.21.15.33:0/1853672838 shutdown_connections 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.699+0000 7fd7a8959700 1 -- 172.21.15.33:0/1853672838 wait complete. 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 Processor -- start 2024-05-20T19:57:50.702 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 -- start start 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a4075f20 0x7fd7a419b8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd7a40774f0 0x7fd7a419bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd7a410b4a0 0x7fd7a41a2e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd7a4111270 con 0x7fd7a40774f0 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd7a41110f0 con 0x7fd7a4075f20 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a8959700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd7a41113f0 con 0x7fd7a410b4a0 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a4075f20 0x7fd7a419b8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a4075f20 0x7fd7a419b8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.33:54194/0 (socket says 172.21.15.33:54194) 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 -- 172.21.15.33:0/1052183219 learned_addr learned my addr 172.21.15.33:0/1052183219 (peer_addr_for_me v2:172.21.15.33:0/0) 2024-05-20T19:57:50.703 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 -- 172.21.15.33:0/1052183219 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd7a410b4a0 msgr2=0x7fd7a41a2e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd7a410b4a0 0x7fd7a41a2e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 -- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd7a40774f0 msgr2=0x7fd7a419bdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd7a40774f0 0x7fd7a419bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 -- 172.21.15.33:0/1052183219 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd7a41a3510 con 0x7fd7a4075f20 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.700+0000 7fd7a259c700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a4075f20 0x7fd7a419b8b0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fd79400b530 tx=0x7fd79400b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.701+0000 7fd7937fe700 1 -- 172.21.15.33:0/1052183219 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd794015020 con 0x7fd7a4075f20 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.701+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd7a41a3800 con 0x7fd7a4075f20 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.701+0000 7fd7937fe700 1 -- 172.21.15.33:0/1052183219 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd794011ce0 con 0x7fd7a4075f20 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.701+0000 7fd7937fe700 1 -- 172.21.15.33:0/1052183219 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd794010d50 con 0x7fd7a4075f20 2024-05-20T19:57:50.704 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.701+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd7a41a3dc0 con 0x7fd7a4075f20 2024-05-20T19:57:50.705 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.702+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd7a406a230 con 0x7fd7a4075f20 2024-05-20T19:57:50.705 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.703+0000 7fd7937fe700 1 -- 172.21.15.33:0/1052183219 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fd794018020 con 0x7fd7a4075f20 2024-05-20T19:57:50.705 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.703+0000 7fd7937fe700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd78c0b2900 0x7fd78c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:57:50.705 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.704+0000 7fd7937fe700 1 -- 172.21.15.33:0/1052183219 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(43..43 src has 1..43) ==== 4965+0+0 (secure 0 0 0) 0x7fd7940e1330 con 0x7fd7a4075f20 2024-05-20T19:57:50.705 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.704+0000 7fd7a1d9b700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd78c0b2900 0x7fd78c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:57:50.706 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.705+0000 7fd7a1d9b700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd78c0b2900 0x7fd78c0b4dc0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fd798005950 tx=0x7fd7980058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:57:50.708 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.707+0000 7fd7937fe700 1 -- 172.21.15.33:0/1052183219 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fd79401f020 con 0x7fd7a4075f20 2024-05-20T19:57:50.708 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:50 smithi033 ceph-mon[36179]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:50.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:50 smithi028 ceph-mon[31178]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:50 smithi154 ceph-mon[36314]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:50.929 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:57:50.928+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7fd7a4063980 con 0x7fd78c0b2900 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:51.836 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:51.837 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:51.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:57:51.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:57:52.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:57:52.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:57:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:52 smithi154 ceph-mon[36314]: from='client.24308 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:57:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:52 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi028 2024-05-20T19:57:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:52 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi028 to 5698M 2024-05-20T19:57:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:52 smithi154 ceph-mon[36314]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:52 smithi033 ceph-mon[36179]: from='client.24308 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:57:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:52 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi028 2024-05-20T19:57:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:52 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi028 to 5698M 2024-05-20T19:57:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:52 smithi033 ceph-mon[36179]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:53.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:52 smithi028 ceph-mon[31178]: from='client.24308 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:57:53.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:52 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi028 2024-05-20T19:57:53.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:52 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi028 to 5698M 2024-05-20T19:57:53.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:52 smithi028 ceph-mon[31178]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:54 smithi154 ceph-mon[36314]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:54 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/392300085' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]: dispatch 2024-05-20T19:57:54.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:54 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]: dispatch 2024-05-20T19:57:54.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:54 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]': finished 2024-05-20T19:57:54.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:54 smithi154 ceph-mon[36314]: osdmap e44: 8 total, 7 up, 8 in 2024-05-20T19:57:54.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:57:54.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:54 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/4126155686' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:57:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:54 smithi033 ceph-mon[36179]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:54 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/392300085' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]: dispatch 2024-05-20T19:57:54.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:54 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]: dispatch 2024-05-20T19:57:54.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:54 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]': finished 2024-05-20T19:57:54.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:54 smithi033 ceph-mon[36179]: osdmap e44: 8 total, 7 up, 8 in 2024-05-20T19:57:54.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:57:54.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:54 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/4126155686' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:57:55.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:54 smithi028 ceph-mon[31178]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:55.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:54 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/392300085' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]: dispatch 2024-05-20T19:57:55.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:54 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]: dispatch 2024-05-20T19:57:55.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:54 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a66dec84-09f9-4179-9130-3eb1e6cc5044"}]': finished 2024-05-20T19:57:55.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:54 smithi028 ceph-mon[31178]: osdmap e44: 8 total, 7 up, 8 in 2024-05-20T19:57:55.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:57:55.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:54 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/4126155686' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:57:56.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:56 smithi154 ceph-mon[36314]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:56.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:56 smithi033 ceph-mon[36179]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:56 smithi028 ceph-mon[31178]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:57:58 smithi154 ceph-mon[36314]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:57:58 smithi033 ceph-mon[36179]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:57:59.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:57:58 smithi028 ceph-mon[31178]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:00.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:00 smithi154 ceph-mon[36314]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:00.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:00 smithi033 ceph-mon[36179]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:01.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:00 smithi028 ceph-mon[31178]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:01.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T19:58:01.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:01.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T19:58:01.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:02.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T19:58:02.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:02 smithi154 ceph-mon[36314]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:02 smithi154 ceph-mon[36314]: Deploying daemon osd.7 on smithi033 2024-05-20T19:58:02.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:02 smithi033 ceph-mon[36179]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:02.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:02 smithi033 ceph-mon[36179]: Deploying daemon osd.7 on smithi033 2024-05-20T19:58:03.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:02 smithi028 ceph-mon[31178]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:03.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:02 smithi028 ceph-mon[31178]: Deploying daemon osd.7 on smithi033 2024-05-20T19:58:04.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:04 smithi154 ceph-mon[36314]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:04 smithi033 ceph-mon[36179]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:05.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:04 smithi028 ceph-mon[31178]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:06.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:06 smithi154 ceph-mon[36314]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:06.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:06 smithi033 ceph-mon[36179]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:07.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:06 smithi028 ceph-mon[31178]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:07.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:07.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:07.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:07.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:07.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:07.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:08.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:08.128 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:08.128 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:08.902 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:08 smithi033 ceph-mon[36179]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:08 smithi028 ceph-mon[31178]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:08 smithi154 ceph-mon[36314]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:10.745 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:10 smithi033 ceph-mon[36179]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:10.745 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:10 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:10.746 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:10 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:10.746 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:10 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:10.746 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:10 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:10.746 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:10 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:11.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:10 smithi028 ceph-mon[31178]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:11.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:10 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:11.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:10 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:11.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:10 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:11.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:10 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:11.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:10 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:11.122 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 7 on host 'smithi033' 2024-05-20T19:58:11.122 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.120+0000 7fd7937fe700 1 -- 172.21.15.33:0/1052183219 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fd7a4063980 con 0x7fd78c0b2900 2024-05-20T19:58:11.124 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd78c0b2900 msgr2=0x7fd78c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:11.124 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd78c0b2900 0x7fd78c0b4dc0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fd798005950 tx=0x7fd7980058e0 comp rx=0 tx=0).stop 2024-05-20T19:58:11.124 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a4075f20 msgr2=0x7fd7a419b8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:11.124 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a4075f20 0x7fd7a419b8b0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fd79400b530 tx=0x7fd79400b8f0 comp rx=0 tx=0).stop 2024-05-20T19:58:11.124 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 shutdown_connections 2024-05-20T19:58:11.125 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd78c0b2900 0x7fd78c0b4dc0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:11.125 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd7a410b4a0 0x7fd7a41a2e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:11.125 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd7a4075f20 0x7fd7a419b8b0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:11.125 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 --2- 172.21.15.33:0/1052183219 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd7a40774f0 0x7fd7a419bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:11.125 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 >> 172.21.15.33:0/1052183219 conn(0x7fd7a40feb40 msgr2=0x7fd7a41073f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:58:11.125 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 shutdown_connections 2024-05-20T19:58:11.125 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:58:11.123+0000 7fd7a8959700 1 -- 172.21.15.33:0/1052183219 wait complete. 2024-05-20T19:58:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:10 smithi154 ceph-mon[36314]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:11.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:10 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:11.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:10 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:11.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:10 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:11.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:10 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:11.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:10 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:11.818 DEBUG:teuthology.orchestra.run.smithi033:osd.7> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.7.service 2024-05-20T19:58:11.820 INFO:tasks.cephadm:Deploying osd.8 on smithi154 with /dev/vg_nvme/lv_2... 2024-05-20T19:58:11.821 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-20T19:58:12.066 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:58:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:12 smithi154 ceph-mon[36314]: from='osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T19:58:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:12 smithi154 ceph-mon[36314]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T19:58:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:12.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:12 smithi154 ceph-mon[36314]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:12 smithi033 ceph-mon[36179]: from='osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T19:58:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:12 smithi033 ceph-mon[36179]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T19:58:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:12 smithi033 ceph-mon[36179]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:12.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:12.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:12.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:12 smithi028 ceph-mon[31178]: from='osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T19:58:12.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:12 smithi028 ceph-mon[31178]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T19:58:12.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:12.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:12 smithi028 ceph-mon[31178]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:13.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:13 smithi033 ceph-mon[36179]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T19:58:13.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:13 smithi033 ceph-mon[36179]: osdmap e45: 8 total, 7 up, 8 in 2024-05-20T19:58:13.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:13.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:13 smithi033 ceph-mon[36179]: from='osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:58:13.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:13 smithi033 ceph-mon[36179]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:58:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:13 smithi154 ceph-mon[36314]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T19:58:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:13 smithi154 ceph-mon[36314]: osdmap e45: 8 total, 7 up, 8 in 2024-05-20T19:58:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:13 smithi154 ceph-mon[36314]: from='osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:58:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:13 smithi154 ceph-mon[36314]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:58:13.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:13 smithi028 ceph-mon[31178]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T19:58:13.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:13 smithi028 ceph-mon[31178]: osdmap e45: 8 total, 7 up, 8 in 2024-05-20T19:58:13.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:13.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:13 smithi028 ceph-mon[31178]: from='osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:58:13.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:13 smithi028 ceph-mon[31178]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:58:14.021 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 19:58:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T19:58:13.641+0000 7ff8b0ccd700 -1 osd.7 0 waiting for initial osdmap 2024-05-20T19:58:14.021 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 19:58:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T19:58:13.651+0000 7ff8a8ab8700 -1 osd.7 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:58:14.036 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-20T19:58:14.036 INFO:teuthology.orchestra.run.smithi154.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-20T19:58:14.036 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10+0 records in 2024-05-20T19:58:14.036 INFO:teuthology.orchestra.run.smithi154.stderr:10+0 records out 2024-05-20T19:58:14.036 INFO:teuthology.orchestra.run.smithi154.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114323 s, 917 MB/s 2024-05-20T19:58:14.036 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping successful for: 2024-05-20T19:58:14.343 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:14 smithi154 ceph-mon[36314]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:58:14.343 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:14 smithi154 ceph-mon[36314]: osdmap e46: 8 total, 7 up, 8 in 2024-05-20T19:58:14.343 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:14.343 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:14.343 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:14 smithi154 ceph-mon[36314]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:14.344 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:14 smithi154 ceph-mon[36314]: from='osd.7 ' entity='osd.7' 2024-05-20T19:58:14.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:14 smithi033 ceph-mon[36179]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:58:14.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:14 smithi033 ceph-mon[36179]: osdmap e46: 8 total, 7 up, 8 in 2024-05-20T19:58:14.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:14.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:14.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:14 smithi033 ceph-mon[36179]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:14.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:14 smithi033 ceph-mon[36179]: from='osd.7 ' entity='osd.7' 2024-05-20T19:58:14.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:14 smithi028 ceph-mon[31178]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:58:14.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:14 smithi028 ceph-mon[31178]: osdmap e46: 8 total, 7 up, 8 in 2024-05-20T19:58:14.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:14.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:14.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:14 smithi028 ceph-mon[31178]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 192 MiB used, 626 GiB / 626 GiB avail 2024-05-20T19:58:14.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:14 smithi028 ceph-mon[31178]: from='osd.7 ' entity='osd.7' 2024-05-20T19:58:14.735 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_2 2024-05-20T19:58:14.977 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:58:15.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] boot 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: osdmap e47: 8 total, 8 up, 8 in 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:15.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:15.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:15 smithi154 ceph-mon[36314]: osdmap e48: 8 total, 8 up, 8 in 2024-05-20T19:58:15.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:58:15.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:58:15.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] boot 2024-05-20T19:58:15.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: osdmap e47: 8 total, 8 up, 8 in 2024-05-20T19:58:15.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:15 smithi033 ceph-mon[36179]: osdmap e48: 8 total, 8 up, 8 in 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: osd.7 [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] boot 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: osdmap e47: 8 total, 8 up, 8 in 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:15.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:15.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:15.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:15 smithi028 ceph-mon[31178]: osdmap e48: 8 total, 8 up, 8 in 2024-05-20T19:58:16.357 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:16 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi033 2024-05-20T19:58:16.357 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:16 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi033 to 5697M 2024-05-20T19:58:16.358 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:16 smithi154 ceph-mon[36314]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:16.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:16 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi033 2024-05-20T19:58:16.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:16 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi033 to 5697M 2024-05-20T19:58:16.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:16 smithi028 ceph-mon[31178]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:16.598 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2125072009 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 msgr2=0x7fdb0410a8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:16.598 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2125072009 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0410a8f0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7fdafc00b0a0 tx=0x7fdafc00b3b0 comp rx=0 tx=0).stop 2024-05-20T19:58:16.598 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2125072009 shutdown_connections 2024-05-20T19:58:16.598 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2125072009 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0410a8f0 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:16.598 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2125072009 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdb041024a0 0x7fdb041028c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:16.598 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2125072009 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdb041036a0 0x7fdb04103b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2125072009 >> 172.21.15.154:0/2125072009 conn(0x7fdb040fda60 msgr2=0x7fdb040ffe80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2125072009 shutdown_connections 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.596+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2125072009 wait complete. 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 Processor -- start 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 -- start start 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdb041024a0 0x7fdb04198720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdb041036a0 0x7fdb04198c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0419fce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdb0410e030 con 0x7fdb041024a0 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdb0410deb0 con 0x7fdb041036a0 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0b1be700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdb0410e1b0 con 0x7fdb04104040 2024-05-20T19:58:16.599 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdafbfff700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdb041036a0 0x7fdb04198c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0419fce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdafbfff700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdb041036a0 0x7fdb04198c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.154:41550/0 (socket says 172.21.15.154:41550) 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdafbfff700 1 -- 172.21.15.154:0/2596707245 learned_addr learned my addr 172.21.15.154:0/2596707245 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0419fce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:43754/0 (socket says 172.21.15.154:43754) 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 -- 172.21.15.154:0/2596707245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdb041036a0 msgr2=0x7fdb04198c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdb041036a0 0x7fdb04198c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 -- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdb041024a0 msgr2=0x7fdb04198720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdb041024a0 0x7fdb04198720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 -- 172.21.15.154:0/2596707245 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdb041a0380 con 0x7fdb04104040 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.597+0000 7fdb0975b700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0419fce0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fdafc00b0a0 tx=0x7fdafc004e40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.598+0000 7fdaf9ffb700 1 -- 172.21.15.154:0/2596707245 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdafc021070 con 0x7fdb04104040 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.598+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdb041a0610 con 0x7fdb04104040 2024-05-20T19:58:16.600 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.598+0000 7fdaf9ffb700 1 -- 172.21.15.154:0/2596707245 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdafc007c00 con 0x7fdb04104040 2024-05-20T19:58:16.601 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.598+0000 7fdaf9ffb700 1 -- 172.21.15.154:0/2596707245 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdafc0052a0 con 0x7fdb04104040 2024-05-20T19:58:16.601 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.598+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdb041a0b90 con 0x7fdb04104040 2024-05-20T19:58:16.601 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.599+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdb0406a230 con 0x7fdb04104040 2024-05-20T19:58:16.601 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.600+0000 7fdaf9ffb700 1 -- 172.21.15.154:0/2596707245 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fdafc024070 con 0x7fdb04104040 2024-05-20T19:58:16.601 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.601+0000 7fdaf9ffb700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fdaec0b2880 0x7fdaec0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:16.602 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.601+0000 7fdaf9ffb700 1 -- 172.21.15.154:0/2596707245 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(49..49 src has 1..49) ==== 5397+0+0 (secure 0 0 0) 0x7fdafc00f020 con 0x7fdb04104040 2024-05-20T19:58:16.602 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.602+0000 7fdb08f5a700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fdaec0b2880 0x7fdaec0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:58:16.603 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.603+0000 7fdb08f5a700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fdaec0b2880 0x7fdaec0b4d40 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fdb04103500 tx=0x7fdb00006d20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:58:16.605 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.605+0000 7fdaf9ffb700 1 -- 172.21.15.154:0/2596707245 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fdafc02b020 con 0x7fdb04104040 2024-05-20T19:58:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:16 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi033 2024-05-20T19:58:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:16 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi033 to 5697M 2024-05-20T19:58:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:16 smithi033 ceph-mon[36179]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:16.818 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:16.818+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) -- 0x7fdb04063980 con 0x7fdaec0b2880 2024-05-20T19:58:17.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:17 smithi028 ceph-mon[31178]: osdmap e49: 8 total, 8 up, 8 in 2024-05-20T19:58:17.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:58:17.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:58:17.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:17 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:17.460 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:17 smithi154 ceph-mon[36314]: osdmap e49: 8 total, 8 up, 8 in 2024-05-20T19:58:17.460 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:58:17.460 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:58:17.460 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:17 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:17 smithi033 ceph-mon[36179]: osdmap e49: 8 total, 8 up, 8 in 2024-05-20T19:58:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:58:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:58:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:17 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:18.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:18 smithi028 ceph-mon[31178]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:58:18.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:18 smithi028 ceph-mon[31178]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:18.560 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:18 smithi154 ceph-mon[36314]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:58:18.560 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:18 smithi154 ceph-mon[36314]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:18.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:18 smithi033 ceph-mon[36179]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:58:18.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:18 smithi033 ceph-mon[36179]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:20 smithi154 ceph-mon[36314]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T19:58:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:20 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/2641237071' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]: dispatch 2024-05-20T19:58:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:20 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]: dispatch 2024-05-20T19:58:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:20 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]': finished 2024-05-20T19:58:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:20 smithi154 ceph-mon[36314]: osdmap e50: 9 total, 8 up, 9 in 2024-05-20T19:58:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:20.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:20 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/2224049013' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:58:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:20 smithi033 ceph-mon[36179]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T19:58:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:20 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/2641237071' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]: dispatch 2024-05-20T19:58:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:20 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]: dispatch 2024-05-20T19:58:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:20 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]': finished 2024-05-20T19:58:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:20 smithi033 ceph-mon[36179]: osdmap e50: 9 total, 8 up, 9 in 2024-05-20T19:58:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:20.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:20 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/2224049013' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:58:20.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:20 smithi028 ceph-mon[31178]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T19:58:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:20 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/2641237071' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]: dispatch 2024-05-20T19:58:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:20 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]: dispatch 2024-05-20T19:58:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:20 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8eb55633-1f82-4288-bf5c-e1e988c1d9fc"}]': finished 2024-05-20T19:58:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:20 smithi028 ceph-mon[31178]: osdmap e50: 9 total, 8 up, 9 in 2024-05-20T19:58:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:20.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:20 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/2224049013' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:58:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:22 smithi154 ceph-mon[36314]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T19:58:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:22 smithi033 ceph-mon[36179]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T19:58:22.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:22 smithi028 ceph-mon[31178]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T19:58:24.680 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:24 smithi154 ceph-mon[36314]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:58:24.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:24 smithi028 ceph-mon[31178]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:58:24.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:24 smithi033 ceph-mon[36179]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:58:26.688 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:26 smithi033 ceph-mon[36179]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-05-20T19:58:26.742 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:26 smithi154 ceph-mon[36314]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-05-20T19:58:26.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:26 smithi028 ceph-mon[31178]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-05-20T19:58:27.737 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:27 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-05-20T19:58:27.737 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:27 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:27.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:27 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-05-20T19:58:27.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:27 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:27.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:27 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-05-20T19:58:27.879 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:27 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:28.593 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:28 smithi154 ceph-mon[36314]: Deploying daemon osd.8 on smithi154 2024-05-20T19:58:28.594 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:28 smithi154 ceph-mon[36314]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:28.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:28 smithi028 ceph-mon[31178]: Deploying daemon osd.8 on smithi154 2024-05-20T19:58:28.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:28 smithi028 ceph-mon[31178]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:28 smithi033 ceph-mon[36179]: Deploying daemon osd.8 on smithi154 2024-05-20T19:58:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:28 smithi033 ceph-mon[36179]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:30.621 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:30 smithi154 ceph-mon[36314]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:30.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:30 smithi028 ceph-mon[31178]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:30 smithi033 ceph-mon[36179]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:32.662 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:32 smithi154 ceph-mon[36314]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:32.662 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:32 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:32.662 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:32 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:32.662 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:32 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:32.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:32 smithi028 ceph-mon[31178]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:32.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:32 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:32.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:32 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:32.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:32 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:32 smithi033 ceph-mon[36179]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:32 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:32 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:32 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:34.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:34 smithi154 ceph-mon[36314]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:34.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:34 smithi028 ceph-mon[31178]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:34.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:34 smithi033 ceph-mon[36179]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:36.156 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 8 on host 'smithi154' 2024-05-20T19:58:36.156 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.155+0000 7fdaf9ffb700 1 -- 172.21.15.154:0/2596707245 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7fdb04063980 con 0x7fdaec0b2880 2024-05-20T19:58:36.159 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.157+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fdaec0b2880 msgr2=0x7fdaec0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:36.159 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.157+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fdaec0b2880 0x7fdaec0b4d40 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fdb04103500 tx=0x7fdb00006d20 comp rx=0 tx=0).stop 2024-05-20T19:58:36.159 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 msgr2=0x7fdb0419fce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:36.159 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0419fce0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fdafc00b0a0 tx=0x7fdafc004e40 comp rx=0 tx=0).stop 2024-05-20T19:58:36.159 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 shutdown_connections 2024-05-20T19:58:36.159 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fdaec0b2880 0x7fdaec0b4d40 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:36.159 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdb04104040 0x7fdb0419fce0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:36.160 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdb041024a0 0x7fdb04198720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:36.160 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 --2- 172.21.15.154:0/2596707245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdb041036a0 0x7fdb04198c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:36.160 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 >> 172.21.15.154:0/2596707245 conn(0x7fdb040fda60 msgr2=0x7fdb04106ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:58:36.160 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 shutdown_connections 2024-05-20T19:58:36.160 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:58:36.158+0000 7fdb0b1be700 1 -- 172.21.15.154:0/2596707245 wait complete. 2024-05-20T19:58:36.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:36.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:36.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:36 smithi154 ceph-mon[36314]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:36.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:36.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:36.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:36 smithi033 ceph-mon[36179]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:36.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:36.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:36.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:36.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:36 smithi028 ceph-mon[31178]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:36.772 DEBUG:teuthology.orchestra.run.smithi154:osd.8> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.8.service 2024-05-20T19:58:36.776 INFO:tasks.cephadm:Deploying osd.9 on smithi028 with /dev/vg_nvme/lv_1... 2024-05-20T19:58:36.777 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-20T19:58:37.267 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:58:37.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:37 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:37.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:37 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:37.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:37 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:37.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:37 smithi028 ceph-mon[31178]: from='osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-05-20T19:58:37.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:37 smithi028 ceph-mon[31178]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-05-20T19:58:37.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:37 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:37.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:37 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:37.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:37 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:37.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:37 smithi154 ceph-mon[36314]: from='osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-05-20T19:58:37.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:37 smithi154 ceph-mon[36314]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-05-20T19:58:37.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:37 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:37.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:37 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:37.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:37 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:58:37.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:37 smithi033 ceph-mon[36179]: from='osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-05-20T19:58:37.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:37 smithi033 ceph-mon[36179]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-05-20T19:58:38.235 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:38 smithi154 ceph-mon[36314]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-05-20T19:58:38.236 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:38 smithi154 ceph-mon[36314]: from='osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:58:38.236 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:38 smithi154 ceph-mon[36314]: osdmap e51: 9 total, 8 up, 9 in 2024-05-20T19:58:38.236 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:38 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:38.236 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:38 smithi154 ceph-mon[36314]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:58:38.236 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:38 smithi154 ceph-mon[36314]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:38.236 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 19:58:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T19:58:38.128+0000 7ffaafd3c700 -1 osd.8 0 waiting for initial osdmap 2024-05-20T19:58:38.236 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 19:58:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T19:58:38.137+0000 7ffaabb2f700 -1 osd.8 52 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:58:38.494 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:38 smithi028 ceph-mon[31178]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-05-20T19:58:38.494 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:38 smithi028 ceph-mon[31178]: from='osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:58:38.495 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:38 smithi028 ceph-mon[31178]: osdmap e51: 9 total, 8 up, 9 in 2024-05-20T19:58:38.495 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:38.495 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:38 smithi028 ceph-mon[31178]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:58:38.495 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:38 smithi028 ceph-mon[31178]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:38.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:38 smithi033 ceph-mon[36179]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-05-20T19:58:38.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:38 smithi033 ceph-mon[36179]: from='osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:58:38.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:38 smithi033 ceph-mon[36179]: osdmap e51: 9 total, 8 up, 9 in 2024-05-20T19:58:38.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:38 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:38.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:38 smithi033 ceph-mon[36179]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T19:58:38.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:38 smithi033 ceph-mon[36179]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-05-20T19:58:39.309 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:39 smithi154 ceph-mon[36314]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:58:39.310 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:39 smithi154 ceph-mon[36314]: osdmap e52: 9 total, 8 up, 9 in 2024-05-20T19:58:39.310 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:39.419 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:39 smithi028 ceph-mon[31178]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:58:39.420 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:39 smithi028 ceph-mon[31178]: osdmap e52: 9 total, 8 up, 9 in 2024-05-20T19:58:39.420 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:39 smithi033 ceph-mon[36179]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T19:58:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:39 smithi033 ceph-mon[36179]: osdmap e52: 9 total, 8 up, 9 in 2024-05-20T19:58:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] boot 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: osdmap e53: 9 total, 9 up, 9 in 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: pgmap v211: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi154 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:40.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:40.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:40 smithi028 ceph-mon[31178]: osdmap e54: 9 total, 9 up, 9 in 2024-05-20T19:58:40.312 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-20T19:58:40.312 INFO:teuthology.orchestra.run.smithi028.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-20T19:58:40.313 INFO:teuthology.orchestra.run.smithi028.stderr: stderr: 10+0 records in 2024-05-20T19:58:40.313 INFO:teuthology.orchestra.run.smithi028.stderr:10+0 records out 2024-05-20T19:58:40.313 INFO:teuthology.orchestra.run.smithi028.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0393095 s, 267 MB/s 2024-05-20T19:58:40.313 INFO:teuthology.orchestra.run.smithi028.stderr:--> Zapping successful for: 2024-05-20T19:58:40.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] boot 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: osdmap e53: 9 total, 9 up, 9 in 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: pgmap v211: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi154 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:40 smithi154 ceph-mon[36314]: osdmap e54: 9 total, 9 up, 9 in 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: osd.8 [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] boot 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: osdmap e53: 9 total, 9 up, 9 in 2024-05-20T19:58:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: pgmap v211: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi154 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:58:40.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:40 smithi033 ceph-mon[36179]: osdmap e54: 9 total, 9 up, 9 in 2024-05-20T19:58:41.278 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi028:vg_nvme/lv_1 2024-05-20T19:58:41.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:41 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi154 to 5696M 2024-05-20T19:58:41.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:41 smithi028 ceph-mon[31178]: osdmap e55: 9 total, 9 up, 9 in 2024-05-20T19:58:41.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:41 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi154 to 5696M 2024-05-20T19:58:41.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:41 smithi154 ceph-mon[36314]: osdmap e55: 9 total, 9 up, 9 in 2024-05-20T19:58:41.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:41 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi154 to 5696M 2024-05-20T19:58:41.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:41 smithi033 ceph-mon[36179]: osdmap e55: 9 total, 9 up, 9 in 2024-05-20T19:58:41.724 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T19:58:42.527 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:42 smithi154 ceph-mon[36314]: pgmap v214: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:42.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:42 smithi028 ceph-mon[31178]: pgmap v214: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:42.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:42 smithi033 ceph-mon[36179]: pgmap v214: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:43.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.822+0000 7f2c6c800700 1 -- 172.21.15.28:0/1927487547 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 msgr2=0x7f2c64112030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:43.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.822+0000 7f2c6c800700 1 --2- 172.21.15.28:0/1927487547 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 0x7f2c64112030 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f2c6000b0d0 tx=0x7f2c6000b3e0 comp rx=0 tx=0).stop 2024-05-20T19:58:43.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.823+0000 7f2c6c800700 1 -- 172.21.15.28:0/1927487547 shutdown_connections 2024-05-20T19:58:43.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.823+0000 7f2c6c800700 1 --2- 172.21.15.28:0/1927487547 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 0x7f2c64112030 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:43.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.823+0000 7f2c6c800700 1 --2- 172.21.15.28:0/1927487547 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c64109be0 0x7f2c6410a000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:43.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.823+0000 7f2c6c800700 1 --2- 172.21.15.28:0/1927487547 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 0x7f2c6410b240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:43.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.823+0000 7f2c6c800700 1 -- 172.21.15.28:0/1927487547 >> 172.21.15.28:0/1927487547 conn(0x7f2c64078bb0 msgr2=0x7f2c6407aff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:58:43.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.823+0000 7f2c6c800700 1 -- 172.21.15.28:0/1927487547 shutdown_connections 2024-05-20T19:58:43.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.823+0000 7f2c6c800700 1 -- 172.21.15.28:0/1927487547 wait complete. 2024-05-20T19:58:43.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 Processor -- start 2024-05-20T19:58:43.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 -- start start 2024-05-20T19:58:43.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c64109be0 0x7f2c6419fc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:43.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 0x7f2c641a01a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:43.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 0x7f2c641a7210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:43.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2c641156b0 con 0x7f2c6410b780 2024-05-20T19:58:43.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2c64115530 con 0x7f2c6410ade0 2024-05-20T19:58:43.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.824+0000 7f2c6c800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2c64115830 con 0x7f2c64109be0 2024-05-20T19:58:43.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c6a800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 0x7f2c641a7210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 0x7f2c641a01a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 0x7f2c641a01a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36428/0 (socket says 172.21.15.28:36428) 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 -- 172.21.15.28:0/3053283736 learned_addr learned my addr 172.21.15.28:0/3053283736 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69e00700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c64109be0 0x7f2c6419fc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 -- 172.21.15.28:0/3053283736 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c64109be0 msgr2=0x7f2c6419fc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c64109be0 0x7f2c6419fc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 -- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 msgr2=0x7f2c641a7210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 0x7f2c641a7210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.825+0000 7f2c69400700 1 -- 172.21.15.28:0/3053283736 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2c641a78e0 con 0x7f2c6410ade0 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.826+0000 7f2c69400700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 0x7f2c641a01a0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2c5400e9a0 tx=0x7f2c5400ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:58:43.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.826+0000 7f2c5ea00700 1 -- 172.21.15.28:0/3053283736 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2c5400ca10 con 0x7f2c6410ade0 2024-05-20T19:58:43.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.826+0000 7f2c5ea00700 1 -- 172.21.15.28:0/3053283736 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2c5400cbb0 con 0x7f2c6410ade0 2024-05-20T19:58:43.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.826+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2c641a7bd0 con 0x7f2c6410ade0 2024-05-20T19:58:43.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.826+0000 7f2c5ea00700 1 -- 172.21.15.28:0/3053283736 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2c54018680 con 0x7f2c6410ade0 2024-05-20T19:58:43.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.826+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2c641a8160 con 0x7f2c6410ade0 2024-05-20T19:58:43.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.827+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2c6406a230 con 0x7f2c6410ade0 2024-05-20T19:58:43.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.830+0000 7f2c5ea00700 1 -- 172.21.15.28:0/3053283736 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f2c54010430 con 0x7f2c6410ade0 2024-05-20T19:58:43.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.832+0000 7f2c5ea00700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2c4c0b2900 0x7f2c4c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:58:43.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.832+0000 7f2c5ea00700 1 -- 172.21.15.28:0/3053283736 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(55..55 src has 1..55) ==== 5829+0+0 (secure 0 0 0) 0x7f2c540e1d30 con 0x7f2c6410ade0 2024-05-20T19:58:43.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.832+0000 7f2c69e00700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2c4c0b2900 0x7f2c4c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:58:43.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.833+0000 7f2c69e00700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2c4c0b2900 0x7f2c4c0b4dc0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f2c641a0b40 tx=0x7f2c58009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:58:43.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:43.833+0000 7f2c5ea00700 1 -- 172.21.15.28:0/3053283736 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f2c540aa360 con 0x7f2c6410ade0 2024-05-20T19:58:44.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:58:44.083+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7f2c64063980 con 0x7f2c4c0b2900 2024-05-20T19:58:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:44 smithi033 ceph-mon[36179]: pgmap v215: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:58:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:58:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:44 smithi028 ceph-mon[31178]: pgmap v215: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:44.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:58:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:58:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:44 smithi154 ceph-mon[36314]: pgmap v215: 1 pgs: 1 remapped+peering; 577 KiB data, 250 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:58:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:58:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:45.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:45 smithi028 ceph-mon[31178]: from='client.24356 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:58:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:45 smithi154 ceph-mon[36314]: from='client.24356 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:58:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:45 smithi033 ceph-mon[36179]: from='client.24356 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi028:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:58:46.800 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:46 smithi028 ceph-mon[31178]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T19:58:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:46 smithi154 ceph-mon[36314]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T19:58:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:46 smithi033 ceph-mon[36179]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-05-20T19:58:48.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:48 smithi028 ceph-mon[31178]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:58:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:48 smithi154 ceph-mon[36314]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:58:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:48 smithi033 ceph-mon[36179]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:58:49.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:49 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2046321986' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]: dispatch 2024-05-20T19:58:49.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:49 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]: dispatch 2024-05-20T19:58:49.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:49 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]': finished 2024-05-20T19:58:49.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:49 smithi028 ceph-mon[31178]: osdmap e56: 10 total, 9 up, 10 in 2024-05-20T19:58:49.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:49 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:58:49.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:49 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2952314592' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:58:49.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:49 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2046321986' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]: dispatch 2024-05-20T19:58:49.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:49 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]: dispatch 2024-05-20T19:58:49.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:49 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]': finished 2024-05-20T19:58:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:49 smithi154 ceph-mon[36314]: osdmap e56: 10 total, 9 up, 10 in 2024-05-20T19:58:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:49 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:58:49.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:49 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2952314592' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:58:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:49 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2046321986' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]: dispatch 2024-05-20T19:58:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:49 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]: dispatch 2024-05-20T19:58:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:49 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4de26914-f99d-42d0-a3f9-9050e7bf73b0"}]': finished 2024-05-20T19:58:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:49 smithi033 ceph-mon[36179]: osdmap e56: 10 total, 9 up, 10 in 2024-05-20T19:58:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:49 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:58:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:49 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2952314592' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:58:50.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:50 smithi028 ceph-mon[31178]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T19:58:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:50 smithi154 ceph-mon[36314]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T19:58:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:50 smithi033 ceph-mon[36179]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T19:58:52.731 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:52 smithi028 ceph-mon[31178]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:52.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:52 smithi154 ceph-mon[36314]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:52 smithi033 ceph-mon[36179]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:54.767 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:54 smithi028 ceph-mon[31178]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:54 smithi154 ceph-mon[36314]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:54 smithi033 ceph-mon[36179]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:58:56.668 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:56 smithi028 ceph-mon[31178]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:56.877 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:56 smithi154 ceph-mon[36314]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:56.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:56 smithi033 ceph-mon[36179]: pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:58.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:58 smithi028 ceph-mon[31178]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:58.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:58 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-05-20T19:58:58.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:58 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:58 smithi154 ceph-mon[36314]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-05-20T19:58:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:58 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:58 smithi033 ceph-mon[36179]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:58:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:58 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-05-20T19:58:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:58 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:58:59.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:58:59 smithi028 ceph-mon[31178]: Deploying daemon osd.9 on smithi028 2024-05-20T19:58:59.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:58:59 smithi154 ceph-mon[36314]: Deploying daemon osd.9 on smithi028 2024-05-20T19:58:59.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:58:59 smithi033 ceph-mon[36179]: Deploying daemon osd.9 on smithi028 2024-05-20T19:59:00.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:00 smithi028 ceph-mon[31178]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:00.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:00 smithi154 ceph-mon[36314]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:00.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:00 smithi033 ceph-mon[36179]: pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:02.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:02 smithi028 ceph-mon[31178]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:02 smithi154 ceph-mon[36314]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:02.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:02 smithi033 ceph-mon[36179]: pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:04 smithi028 ceph-mon[31178]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:04.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:04 smithi154 ceph-mon[36314]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:04 smithi033 ceph-mon[36179]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:06.532 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:06 smithi028 ceph-mon[31178]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:06.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:06 smithi154 ceph-mon[36314]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:06.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:06 smithi033 ceph-mon[36179]: pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:08.037 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:08.037 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:08.037 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:08.037 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:07 smithi028 ceph-mon[31178]: pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:08.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:08.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:08.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:08.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:07 smithi154 ceph-mon[36314]: pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:08.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:08.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:08.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:08.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:07 smithi033 ceph-mon[36179]: pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:10.669 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:10 smithi028 ceph-mon[31178]: pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:10 smithi154 ceph-mon[36314]: pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:10.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:10 smithi033 ceph-mon[36179]: pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:11.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.733+0000 7f2c5ea00700 1 -- 172.21.15.28:0/3053283736 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+37 (secure 0 0 0) 0x7f2c64063980 con 0x7f2c4c0b2900 2024-05-20T19:59:11.734 INFO:teuthology.orchestra.run.smithi028.stdout:Created osd(s) 9 on host 'smithi028' 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2c4c0b2900 msgr2=0x7f2c4c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2c4c0b2900 0x7f2c4c0b4dc0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f2c641a0b40 tx=0x7f2c58009260 comp rx=0 tx=0).stop 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 msgr2=0x7f2c641a01a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 0x7f2c641a01a0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2c5400e9a0 tx=0x7f2c5400ecb0 comp rx=0 tx=0).stop 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 shutdown_connections 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2c4c0b2900 0x7f2c4c0b4dc0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2c6410b780 0x7f2c641a7210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c64109be0 0x7f2c6419fc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 --2- 172.21.15.28:0/3053283736 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2c6410ade0 0x7f2c641a01a0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:11.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.737+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 >> 172.21.15.28:0/3053283736 conn(0x7f2c64078bb0 msgr2=0x7f2c6410e0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:59:11.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.738+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 shutdown_connections 2024-05-20T19:59:11.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T19:59:11.738+0000 7f2c6c800700 1 -- 172.21.15.28:0/3053283736 wait complete. 2024-05-20T19:59:12.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:11 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:11 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:11 smithi028 ceph-mon[31178]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-05-20T19:59:12.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:11 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:11 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:11 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:11 smithi154 ceph-mon[36314]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-05-20T19:59:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:11 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:11 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:11 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:11 smithi033 ceph-mon[36179]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-05-20T19:59:12.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:11 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:12.636 DEBUG:teuthology.orchestra.run.smithi028:osd.9> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.9.service 2024-05-20T19:59:12.641 INFO:tasks.cephadm:Deploying osd.10 on smithi033 with /dev/vg_nvme/lv_1... 2024-05-20T19:59:12.642 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-20T19:59:12.898 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:59:13.020 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:12 smithi033 ceph-mon[36179]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:13.020 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:13.020 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:12 smithi033 ceph-mon[36179]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-05-20T19:59:13.020 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:12 smithi033 ceph-mon[36179]: osdmap e57: 10 total, 9 up, 10 in 2024-05-20T19:59:13.021 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:12 smithi033 ceph-mon[36179]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:59:13.021 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:13.026 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:12 smithi154 ceph-mon[36314]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:13.026 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:13.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:12 smithi154 ceph-mon[36314]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-05-20T19:59:13.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:12 smithi154 ceph-mon[36314]: osdmap e57: 10 total, 9 up, 10 in 2024-05-20T19:59:13.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:12 smithi154 ceph-mon[36314]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:59:13.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:13.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:12 smithi028 ceph-mon[31178]: pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:13.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:13.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:12 smithi028 ceph-mon[31178]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-05-20T19:59:13.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:12 smithi028 ceph-mon[31178]: osdmap e57: 10 total, 9 up, 10 in 2024-05-20T19:59:13.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:12 smithi028 ceph-mon[31178]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]: dispatch 2024-05-20T19:59:13.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:13.661 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 19:59:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T19:59:13.387+0000 7fbceaa00700 -1 osd.9 0 waiting for initial osdmap 2024-05-20T19:59:13.661 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 19:59:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T19:59:13.434+0000 7fbcdfe00700 -1 osd.9 58 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:59:13.927 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:13 smithi028 ceph-mon[31178]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:59:13.928 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:13 smithi028 ceph-mon[31178]: osdmap e58: 10 total, 9 up, 10 in 2024-05-20T19:59:13.928 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:13.928 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:13.928 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:13 smithi028 ceph-mon[31178]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' 2024-05-20T19:59:14.058 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:13 smithi033 ceph-mon[36179]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:59:14.058 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:13 smithi033 ceph-mon[36179]: osdmap e58: 10 total, 9 up, 10 in 2024-05-20T19:59:14.058 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.059 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.059 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:13 smithi033 ceph-mon[36179]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' 2024-05-20T19:59:14.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:13 smithi154 ceph-mon[36314]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi028", "root=default"]}]': finished 2024-05-20T19:59:14.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:13 smithi154 ceph-mon[36314]: osdmap e58: 10 total, 9 up, 10 in 2024-05-20T19:59:14.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:13 smithi154 ceph-mon[36314]: from='osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961]' entity='osd.9' 2024-05-20T19:59:14.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:14 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:59:14.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:14 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:59:14.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:14 smithi028 ceph-mon[31178]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:14.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:14 smithi028 ceph-mon[31178]: osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] boot 2024-05-20T19:59:14.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:14 smithi028 ceph-mon[31178]: osdmap e59: 10 total, 10 up, 10 in 2024-05-20T19:59:14.709 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.996 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:14 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:59:14.996 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:14 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:59:14.997 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:14 smithi033 ceph-mon[36179]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:14.997 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.997 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:14 smithi033 ceph-mon[36179]: osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] boot 2024-05-20T19:59:14.997 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:14 smithi033 ceph-mon[36179]: osdmap e59: 10 total, 10 up, 10 in 2024-05-20T19:59:14.997 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:14.998 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-20T19:59:14.998 INFO:teuthology.orchestra.run.smithi033.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-20T19:59:14.998 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10+0 records in 2024-05-20T19:59:14.998 INFO:teuthology.orchestra.run.smithi033.stderr:10+0 records out 2024-05-20T19:59:14.998 INFO:teuthology.orchestra.run.smithi033.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127591 s, 822 MB/s 2024-05-20T19:59:14.998 INFO:teuthology.orchestra.run.smithi033.stderr:--> Zapping successful for: 2024-05-20T19:59:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:14 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:59:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:14 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:59:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:14 smithi154 ceph-mon[36314]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 251 MiB used, 804 GiB / 805 GiB avail 2024-05-20T19:59:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:15.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:14 smithi154 ceph-mon[36314]: osd.9 [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] boot 2024-05-20T19:59:15.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:14 smithi154 ceph-mon[36314]: osdmap e59: 10 total, 10 up, 10 in 2024-05-20T19:59:15.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T19:59:15.716 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_1 2024-05-20T19:59:15.963 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T19:59:17.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:16 smithi028 ceph-mon[31178]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:17.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:16 smithi028 ceph-mon[31178]: osdmap e60: 10 total, 10 up, 10 in 2024-05-20T19:59:17.092 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:16 smithi033 ceph-mon[36179]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:17.092 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:16 smithi033 ceph-mon[36179]: osdmap e60: 10 total, 10 up, 10 in 2024-05-20T19:59:17.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:16 smithi154 ceph-mon[36314]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:17.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:16 smithi154 ceph-mon[36314]: osdmap e60: 10 total, 10 up, 10 in 2024-05-20T19:59:17.645 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 -- 172.21.15.33:0/3049443703 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 msgr2=0x7f379c103840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 --2- 172.21.15.33:0/3049443703 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c103840 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f379400b0a0 tx=0x7f379400b3b0 comp rx=0 tx=0).stop 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 -- 172.21.15.33:0/3049443703 shutdown_connections 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 --2- 172.21.15.33:0/3049443703 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c103840 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 --2- 172.21.15.33:0/3049443703 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f379c101890 0x7f379c101cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 --2- 172.21.15.33:0/3049443703 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f379c102600 0x7f379c102a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 -- 172.21.15.33:0/3049443703 >> 172.21.15.33:0/3049443703 conn(0x7f379c0fd1f0 msgr2=0x7f379c0ff630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 -- 172.21.15.33:0/3049443703 shutdown_connections 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.643+0000 7f37a37bd700 1 -- 172.21.15.33:0/3049443703 wait complete. 2024-05-20T19:59:17.646 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 Processor -- start 2024-05-20T19:59:17.647 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 -- start start 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f379c101890 0x7f379c19b8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f379c102600 0x7f379c19be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c1a2e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f379c1112c0 con 0x7f379c101890 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f379c111140 con 0x7f379c102fc0 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a37bd700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f379c111440 con 0x7f379c102600 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a1d5a700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c1a2e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:59:17.649 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a1d5a700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c1a2e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.33:59860/0 (socket says 172.21.15.33:59860) 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a1d5a700 1 -- 172.21.15.33:0/878904220 learned_addr learned my addr 172.21.15.33:0/878904220 (peer_addr_for_me v2:172.21.15.33:0/0) 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.644+0000 7f37a0d58700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f379c102600 0x7f379c19be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a1d5a700 1 -- 172.21.15.33:0/878904220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f379c102600 msgr2=0x7f379c19be10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a1d5a700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f379c102600 0x7f379c19be10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a1d5a700 1 -- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f379c101890 msgr2=0x7f379c19b8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a1d5a700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f379c101890 0x7f379c19b8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a1d5a700 1 -- 172.21.15.33:0/878904220 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f379c1a3530 con 0x7f379c102fc0 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a0d58700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f379c102600 0x7f379c19be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T19:59:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a1d5a700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c1a2e90 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f379400d010 tx=0x7f3794004e20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:59:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37927fc700 1 -- 172.21.15.33:0/878904220 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3794021070 con 0x7f379c102fc0 2024-05-20T19:59:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f379c1a37c0 con 0x7f379c102fc0 2024-05-20T19:59:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37927fc700 1 -- 172.21.15.33:0/878904220 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3794007c80 con 0x7f379c102fc0 2024-05-20T19:59:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37927fc700 1 -- 172.21.15.33:0/878904220 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3794005330 con 0x7f379c102fc0 2024-05-20T19:59:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.645+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f379c1a3dd0 con 0x7f379c102fc0 2024-05-20T19:59:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.646+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f379c06a230 con 0x7f379c102fc0 2024-05-20T19:59:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.648+0000 7f37927fc700 1 -- 172.21.15.33:0/878904220 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f3794024070 con 0x7f379c102fc0 2024-05-20T19:59:17.652 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.648+0000 7f37927fc700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37880b2880 0x7f37880b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:17.652 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.648+0000 7f37927fc700 1 -- 172.21.15.33:0/878904220 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(60..60 src has 1..60) ==== 6261+0+0 (secure 0 0 0) 0x7f379400f020 con 0x7f379c102fc0 2024-05-20T19:59:17.652 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.649+0000 7f37a1559700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37880b2880 0x7f37880b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:59:17.652 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.650+0000 7f37a1559700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37880b2880 0x7f37880b4d40 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f37980097b0 tx=0x7f3798006d20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:59:17.654 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.652+0000 7f37927fc700 1 -- 172.21.15.33:0/878904220 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f379402b020 con 0x7f379c102fc0 2024-05-20T19:59:17.874 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:17.873+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7f379c063a40 con 0x7f37880b2880 2024-05-20T19:59:18.832 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:18.833 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:18.834 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:18.834 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:18.834 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:19.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:19.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:59:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:19.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:19.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:19.873 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:19 smithi033 ceph-mon[36179]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:59:19.873 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:19 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi028 2024-05-20T19:59:19.873 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:19 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi028 to 4273M 2024-05-20T19:59:19.873 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:19.873 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:19.873 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:19.873 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:20.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:19 smithi028 ceph-mon[31178]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:59:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:19 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi028 2024-05-20T19:59:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:19 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi028 to 4273M 2024-05-20T19:59:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:20.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:20.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:19 smithi154 ceph-mon[36314]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:59:20.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:19 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi028 2024-05-20T19:59:20.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:19 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi028 to 4273M 2024-05-20T19:59:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:20.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:21.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:20 smithi028 ceph-mon[31178]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:20 smithi154 ceph-mon[36314]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:20 smithi033 ceph-mon[36179]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:21 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/4052937081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]: dispatch 2024-05-20T19:59:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:21 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]: dispatch 2024-05-20T19:59:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:21 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]': finished 2024-05-20T19:59:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:21 smithi028 ceph-mon[31178]: osdmap e61: 11 total, 10 up, 11 in 2024-05-20T19:59:22.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:21 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:22.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:21 smithi028 ceph-mon[31178]: from='client.? 172.21.15.33:0/1285064007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:59:22.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:21 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/4052937081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]: dispatch 2024-05-20T19:59:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:21 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]: dispatch 2024-05-20T19:59:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:21 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]': finished 2024-05-20T19:59:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:21 smithi154 ceph-mon[36314]: osdmap e61: 11 total, 10 up, 11 in 2024-05-20T19:59:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:21 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:22.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:21 smithi154 ceph-mon[36314]: from='client.? 172.21.15.33:0/1285064007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:59:22.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:21 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/4052937081' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]: dispatch 2024-05-20T19:59:22.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:21 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]: dispatch 2024-05-20T19:59:22.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:21 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "28571f48-e611-45cd-9835-72989012a981"}]': finished 2024-05-20T19:59:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:21 smithi033 ceph-mon[36179]: osdmap e61: 11 total, 10 up, 11 in 2024-05-20T19:59:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:21 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:22.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:21 smithi033 ceph-mon[36179]: from='client.? 172.21.15.33:0/1285064007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:59:23.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:22 smithi028 ceph-mon[31178]: pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:23.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:22 smithi154 ceph-mon[36314]: pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:22 smithi033 ceph-mon[36179]: pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:24.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:24 smithi033 ceph-mon[36179]: pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:25.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:24 smithi028 ceph-mon[31178]: pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:25.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:24 smithi154 ceph-mon[36314]: pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:27.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:26 smithi028 ceph-mon[31178]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:27.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:26 smithi154 ceph-mon[36314]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:26 smithi033 ceph-mon[36179]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:28 smithi033 ceph-mon[36179]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-05-20T19:59:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:28 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:29.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:28 smithi028 ceph-mon[31178]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:29.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-05-20T19:59:29.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:28 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:28 smithi154 ceph-mon[36314]: pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-05-20T19:59:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:28 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:29.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:29 smithi033 ceph-mon[36179]: Deploying daemon osd.10 on smithi033 2024-05-20T19:59:30.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:29 smithi028 ceph-mon[31178]: Deploying daemon osd.10 on smithi033 2024-05-20T19:59:30.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:29 smithi154 ceph-mon[36314]: Deploying daemon osd.10 on smithi033 2024-05-20T19:59:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:30 smithi033 ceph-mon[36179]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:31.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:30 smithi028 ceph-mon[31178]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:30 smithi154 ceph-mon[36314]: pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:33.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:32 smithi028 ceph-mon[31178]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:33.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:32 smithi154 ceph-mon[36314]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:33.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:32 smithi033 ceph-mon[36179]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:35.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:34 smithi028 ceph-mon[31178]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:35.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:34 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:35.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:34 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:35.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:34 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:34 smithi154 ceph-mon[36314]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:34 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:34 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:34 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:34 smithi033 ceph-mon[36179]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:34 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:34 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:34 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:36.038 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:35 smithi033 ceph-mon[36179]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:36.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:35 smithi154 ceph-mon[36314]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:36.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:35 smithi028 ceph-mon[31178]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:38.221 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.220+0000 7f37927fc700 1 -- 172.21.15.33:0/878904220 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+38 (secure 0 0 0) 0x7f379c063a40 con 0x7f37880b2880 2024-05-20T19:59:38.221 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 10 on host 'smithi033' 2024-05-20T19:59:38.223 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37880b2880 msgr2=0x7f37880b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:38.223 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37880b2880 0x7f37880b4d40 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f37980097b0 tx=0x7f3798006d20 comp rx=0 tx=0).stop 2024-05-20T19:59:38.223 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 msgr2=0x7f379c1a2e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c1a2e90 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f379400d010 tx=0x7f3794004e20 comp rx=0 tx=0).stop 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 shutdown_connections 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37880b2880 0x7f37880b4d40 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f379c102fc0 0x7f379c1a2e90 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f379c101890 0x7f379c19b8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 --2- 172.21.15.33:0/878904220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f379c102600 0x7f379c19be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 >> 172.21.15.33:0/878904220 conn(0x7f379c0fd1f0 msgr2=0x7f379c0fecf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 shutdown_connections 2024-05-20T19:59:38.224 INFO:teuthology.orchestra.run.smithi033.stderr:2024-05-20T19:59:38.223+0000 7f37a37bd700 1 -- 172.21.15.33:0/878904220 wait complete. 2024-05-20T19:59:38.541 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:38 smithi033 ceph-mon[36179]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:38.542 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:38 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:38.542 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:38 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:38 smithi028 ceph-mon[31178]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:38 smithi154 ceph-mon[36314]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:38 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:38 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:38.878 DEBUG:teuthology.orchestra.run.smithi033:osd.10> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.10.service 2024-05-20T19:59:38.880 INFO:tasks.cephadm:Deploying osd.11 on smithi154 with /dev/vg_nvme/lv_1... 2024-05-20T19:59:38.880 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-20T19:59:39.126 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:59:39.542 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:39 smithi154 ceph-mon[36314]: from='osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-05-20T19:59:39.542 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:39 smithi154 ceph-mon[36314]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-05-20T19:59:39.542 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:39.542 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:39.733 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:39 smithi033 ceph-mon[36179]: from='osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-05-20T19:59:39.733 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:39 smithi033 ceph-mon[36179]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-05-20T19:59:39.734 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:39.734 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:39.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:39 smithi028 ceph-mon[31178]: from='osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-05-20T19:59:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:39 smithi028 ceph-mon[31178]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-05-20T19:59:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:39.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:40.508 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:40 smithi154 ceph-mon[36314]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:40.508 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:40 smithi154 ceph-mon[36314]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-05-20T19:59:40.508 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:40 smithi154 ceph-mon[36314]: osdmap e62: 11 total, 10 up, 11 in 2024-05-20T19:59:40.508 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:40 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:40.509 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:40 smithi154 ceph-mon[36314]: from='osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:59:40.509 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:40 smithi154 ceph-mon[36314]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:59:40.705 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:40 smithi033 ceph-mon[36179]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:40.705 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:40 smithi033 ceph-mon[36179]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-05-20T19:59:40.705 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:40 smithi033 ceph-mon[36179]: osdmap e62: 11 total, 10 up, 11 in 2024-05-20T19:59:40.705 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:40 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:40.706 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:40 smithi033 ceph-mon[36179]: from='osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:59:40.706 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:40 smithi033 ceph-mon[36179]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:59:40.706 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 19:59:40 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T19:59:40.415+0000 7f2b32e64700 -1 osd.10 0 waiting for initial osdmap 2024-05-20T19:59:40.706 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 19:59:40 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T19:59:40.429+0000 7f2b2ec57700 -1 osd.10 63 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T19:59:40.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:40 smithi028 ceph-mon[31178]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:40.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:40 smithi028 ceph-mon[31178]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-05-20T19:59:40.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:40 smithi028 ceph-mon[31178]: osdmap e62: 11 total, 10 up, 11 in 2024-05-20T19:59:40.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:40 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:40.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:40 smithi028 ceph-mon[31178]: from='osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:59:40.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:40 smithi028 ceph-mon[31178]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-05-20T19:59:41.233 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-20T19:59:41.233 INFO:teuthology.orchestra.run.smithi154.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-20T19:59:41.234 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10+0 records in 2024-05-20T19:59:41.234 INFO:teuthology.orchestra.run.smithi154.stderr:10+0 records out 2024-05-20T19:59:41.234 INFO:teuthology.orchestra.run.smithi154.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.011408 s, 919 MB/s 2024-05-20T19:59:41.234 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping successful for: 2024-05-20T19:59:41.541 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:41 smithi154 ceph-mon[36314]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:59:41.541 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:41 smithi154 ceph-mon[36314]: osdmap e63: 11 total, 10 up, 11 in 2024-05-20T19:59:41.541 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:41 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:41.541 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:41 smithi154 ceph-mon[36314]: osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] boot 2024-05-20T19:59:41.541 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:41 smithi154 ceph-mon[36314]: osdmap e64: 11 total, 11 up, 11 in 2024-05-20T19:59:41.541 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:41 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:41.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:41 smithi028 ceph-mon[31178]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:59:41.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:41 smithi028 ceph-mon[31178]: osdmap e63: 11 total, 10 up, 11 in 2024-05-20T19:59:41.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:41 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:41.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:41 smithi028 ceph-mon[31178]: osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] boot 2024-05-20T19:59:41.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:41 smithi028 ceph-mon[31178]: osdmap e64: 11 total, 11 up, 11 in 2024-05-20T19:59:41.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:41 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:41 smithi033 ceph-mon[36179]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-05-20T19:59:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:41 smithi033 ceph-mon[36179]: osdmap e63: 11 total, 10 up, 11 in 2024-05-20T19:59:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:41 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:41 smithi033 ceph-mon[36179]: osd.10 [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] boot 2024-05-20T19:59:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:41 smithi033 ceph-mon[36179]: osdmap e64: 11 total, 11 up, 11 in 2024-05-20T19:59:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:41 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:41.967 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_1 2024-05-20T19:59:42.215 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T19:59:42.523 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:42 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T19:59:42.523 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:42 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T19:59:42.523 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:42 smithi033 ceph-mon[36179]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:42.523 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:42.523 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:42 smithi033 ceph-mon[36179]: osdmap e65: 11 total, 11 up, 11 in 2024-05-20T19:59:42.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:42 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T19:59:42.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:42 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T19:59:42.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:42 smithi154 ceph-mon[36314]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:42.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:42.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:42 smithi154 ceph-mon[36314]: osdmap e65: 11 total, 11 up, 11 in 2024-05-20T19:59:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:42 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T19:59:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:42 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T19:59:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:42 smithi028 ceph-mon[31178]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 282 MiB used, 894 GiB / 894 GiB avail 2024-05-20T19:59:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T19:59:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:42 smithi028 ceph-mon[31178]: osdmap e65: 11 total, 11 up, 11 in 2024-05-20T19:59:43.761 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi033 2024-05-20T19:59:43.761 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.761 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.761 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi033 to 4273M 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:43.762 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:43.763 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.763 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:43 smithi154 ceph-mon[36314]: osdmap e66: 11 total, 11 up, 11 in 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 -- 172.21.15.154:0/2855463030 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138104030 msgr2=0x7fb13810a8e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/2855463030 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138104030 0x7fb13810a8e0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fb13000b0a0 tx=0x7fb13000b3b0 comp rx=0 tx=0).stop 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 -- 172.21.15.154:0/2855463030 shutdown_connections 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/2855463030 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138104030 0x7fb13810a8e0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/2855463030 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb138102490 0x7fb1381028b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/2855463030 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb138103690 0x7fb138103af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 -- 172.21.15.154:0/2855463030 >> 172.21.15.154:0/2855463030 conn(0x7fb1380fda30 msgr2=0x7fb1380ffe70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 -- 172.21.15.154:0/2855463030 shutdown_connections 2024-05-20T19:59:43.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.864+0000 7fb13e8fa700 1 -- 172.21.15.154:0/2855463030 wait complete. 2024-05-20T19:59:43.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 Processor -- start 2024-05-20T19:59:43.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 -- start start 2024-05-20T19:59:43.867 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138102490 0x7fb138199800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb138103690 0x7fb138199d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb138104030 0x7fb1381a0dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb13810e020 con 0x7fb138103690 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb13810dea0 con 0x7fb138104030 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13e8fa700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb13810e1a0 con 0x7fb138102490 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb137fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138102490 0x7fb138199800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb137fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138102490 0x7fb138199800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:56146/0 (socket says 172.21.15.154:56146) 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb137fff700 1 -- 172.21.15.154:0/354855511 learned_addr learned my addr 172.21.15.154:0/354855511 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb13ce97700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb138104030 0x7fb1381a0dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:59:43.868 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb137fff700 1 -- 172.21.15.154:0/354855511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb138104030 msgr2=0x7fb1381a0dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb137fff700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb138104030 0x7fb1381a0dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb137fff700 1 -- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb138103690 msgr2=0x7fb138199d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.865+0000 7fb137fff700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb138103690 0x7fb138199d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb137fff700 1 -- 172.21.15.154:0/354855511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1381a1460 con 0x7fb138102490 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb13ce97700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb138104030 0x7fb1381a0dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb137fff700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138102490 0x7fb138199800 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fb12800b530 tx=0x7fb12800b8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb1357fa700 1 -- 172.21.15.154:0/354855511 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb128015020 con 0x7fb138102490 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb1357fa700 1 -- 172.21.15.154:0/354855511 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb128011ce0 con 0x7fb138102490 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb1381a1750 con 0x7fb138102490 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb1357fa700 1 -- 172.21.15.154:0/354855511 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb128010d60 con 0x7fb138102490 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.866+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb1381a1d10 con 0x7fb138102490 2024-05-20T19:59:43.869 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.867+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb13806a230 con 0x7fb138102490 2024-05-20T19:59:43.870 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.868+0000 7fb1357fa700 1 -- 172.21.15.154:0/354855511 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fb128018020 con 0x7fb138102490 2024-05-20T19:59:43.870 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.869+0000 7fb1357fa700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1200b2880 0x7fb1200b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T19:59:43.870 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.869+0000 7fb1357fa700 1 -- 172.21.15.154:0/354855511 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(66..66 src has 1..66) ==== 6694+0+0 (secure 0 0 0) 0x7fb1280e1980 con 0x7fb138102490 2024-05-20T19:59:43.871 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.869+0000 7fb1377fe700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1200b2880 0x7fb1200b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T19:59:43.871 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.870+0000 7fb1377fe700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1200b2880 0x7fb1200b4d40 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fb12c0097b0 tx=0x7fb12c006d20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T19:59:43.873 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:43.873+0000 7fb1357fa700 1 -- 172.21.15.154:0/354855511 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb12801f020 con 0x7fb138102490 2024-05-20T19:59:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi033 2024-05-20T19:59:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi033 to 4273M 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:43.879 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:43.879 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:43 smithi033 ceph-mon[36179]: osdmap e66: 11 total, 11 up, 11 in 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi033 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T19:59:44.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi033 to 4273M 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T19:59:44.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:43 smithi028 ceph-mon[31178]: osdmap e66: 11 total, 11 up, 11 in 2024-05-20T19:59:44.089 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T19:59:44.088+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) -- 0x7fb1380ffc50 con 0x7fb1200b2880 2024-05-20T19:59:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:44 smithi154 ceph-mon[36314]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 309 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:59:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:59:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:44 smithi033 ceph-mon[36179]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 309 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:59:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:59:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:45.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:44 smithi028 ceph-mon[31178]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 309 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:45.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T19:59:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T19:59:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:45 smithi154 ceph-mon[36314]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:59:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:45 smithi033 ceph-mon[36179]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:59:45.966 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:45 smithi028 ceph-mon[31178]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T19:59:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:46 smithi033 ceph-mon[36179]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-05-20T19:59:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:46 smithi028 ceph-mon[31178]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-05-20T19:59:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:46 smithi154 ceph-mon[36314]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-05-20T19:59:48.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:47 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/3956607537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]: dispatch 2024-05-20T19:59:48.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:47 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]: dispatch 2024-05-20T19:59:48.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:47 smithi028 ceph-mon[31178]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]': finished 2024-05-20T19:59:48.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:47 smithi028 ceph-mon[31178]: osdmap e67: 12 total, 11 up, 12 in 2024-05-20T19:59:48.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:47 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T19:59:48.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:47 smithi028 ceph-mon[31178]: from='client.? 172.21.15.154:0/1735789880' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:59:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:47 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/3956607537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]: dispatch 2024-05-20T19:59:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:47 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]: dispatch 2024-05-20T19:59:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:47 smithi154 ceph-mon[36314]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]': finished 2024-05-20T19:59:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:47 smithi154 ceph-mon[36314]: osdmap e67: 12 total, 11 up, 12 in 2024-05-20T19:59:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:47 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T19:59:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:47 smithi154 ceph-mon[36314]: from='client.? 172.21.15.154:0/1735789880' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:59:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:47 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/3956607537' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]: dispatch 2024-05-20T19:59:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:47 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]: dispatch 2024-05-20T19:59:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:47 smithi033 ceph-mon[36179]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb59bcc3-6dcc-42dd-8492-46bf37eb3be6"}]': finished 2024-05-20T19:59:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:47 smithi033 ceph-mon[36179]: osdmap e67: 12 total, 11 up, 12 in 2024-05-20T19:59:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:47 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T19:59:48.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:47 smithi033 ceph-mon[36179]: from='client.? 172.21.15.154:0/1735789880' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T19:59:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:48 smithi028 ceph-mon[31178]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-05-20T19:59:49.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:48 smithi154 ceph-mon[36314]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-05-20T19:59:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:48 smithi033 ceph-mon[36179]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-05-20T19:59:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:50 smithi154 ceph-mon[36314]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-05-20T19:59:51.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:50 smithi028 ceph-mon[31178]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-05-20T19:59:51.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:50 smithi033 ceph-mon[36179]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-05-20T19:59:53.034 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:52 smithi154 ceph-mon[36314]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:59:53.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:52 smithi028 ceph-mon[31178]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:59:53.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:52 smithi033 ceph-mon[36179]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-05-20T19:59:54.929 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:54 smithi033 ceph-mon[36179]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:59:54.930 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-05-20T19:59:54.930 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:54 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:54.936 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:54 smithi154 ceph-mon[36314]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:59:54.936 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-05-20T19:59:54.936 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:54 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:55.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:54 smithi028 ceph-mon[31178]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-20T19:59:55.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-05-20T19:59:55.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:54 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T19:59:55.934 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:55 smithi154 ceph-mon[36314]: Deploying daemon osd.11 on smithi154 2024-05-20T19:59:56.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:55 smithi028 ceph-mon[31178]: Deploying daemon osd.11 on smithi154 2024-05-20T19:59:56.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:55 smithi033 ceph-mon[36179]: Deploying daemon osd.11 on smithi154 2024-05-20T19:59:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:56 smithi028 ceph-mon[31178]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:57.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:56 smithi154 ceph-mon[36314]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:57.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:56 smithi033 ceph-mon[36179]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:59.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 19:59:58 smithi154 ceph-mon[36314]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:59.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 19:59:58 smithi028 ceph-mon[31178]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T19:59:59.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 19:59:58 smithi033 ceph-mon[36179]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:00.886 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:00 smithi154 ceph-mon[36314]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:00.886 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:00.886 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:00.886 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:00.886 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:00 smithi154 ceph-mon[36314]: overall HEALTH_OK 2024-05-20T20:00:01.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:00 smithi028 ceph-mon[31178]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:01.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:01.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:01.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:01.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:00 smithi028 ceph-mon[31178]: overall HEALTH_OK 2024-05-20T20:00:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:00 smithi033 ceph-mon[36179]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:01.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:00 smithi033 ceph-mon[36179]: overall HEALTH_OK 2024-05-20T20:00:02.798 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:02 smithi154 ceph-mon[36314]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:03.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:02 smithi028 ceph-mon[31178]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:02 smithi033 ceph-mon[36179]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:03.927 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.925+0000 7fb1357fa700 1 -- 172.21.15.154:0/354855511 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+38 (secure 0 0 0) 0x7fb1380ffc50 con 0x7fb1200b2880 2024-05-20T20:00:03.927 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 11 on host 'smithi154' 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1200b2880 msgr2=0x7fb1200b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1200b2880 0x7fb1200b4d40 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fb12c0097b0 tx=0x7fb12c006d20 comp rx=0 tx=0).stop 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138102490 msgr2=0x7fb138199800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138102490 0x7fb138199800 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fb12800b530 tx=0x7fb12800b8f0 comp rx=0 tx=0).stop 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 shutdown_connections 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb1200b2880 0x7fb1200b4d40 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb138104030 0x7fb1381a0dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb138102490 0x7fb138199800 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 --2- 172.21.15.154:0/354855511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb138103690 0x7fb138199d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.927+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 >> 172.21.15.154:0/354855511 conn(0x7fb1380fda30 msgr2=0x7fb1380ff540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:03.929 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.928+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 shutdown_connections 2024-05-20T20:00:03.930 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:03.928+0000 7fb13e8fa700 1 -- 172.21.15.154:0/354855511 wait complete. 2024-05-20T20:00:04.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:04.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:04.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:04.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-05-20T20:00:04.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-05-20T20:00:04.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:04.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:04.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:04.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:04.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-05-20T20:00:04.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-05-20T20:00:04.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:04.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:04.643 DEBUG:teuthology.orchestra.run.smithi154:osd.11> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.11.service 2024-05-20T20:00:04.645 INFO:tasks.cephadm:Waiting for 12 OSDs to come up... 2024-05-20T20:00:04.646 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd stat -f json 2024-05-20T20:00:05.131 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:05.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:05 smithi028 ceph-mon[31178]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-05-20T20:00:05.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:05 smithi028 ceph-mon[31178]: from='osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T20:00:05.283 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:05 smithi028 ceph-mon[31178]: osdmap e68: 12 total, 11 up, 12 in 2024-05-20T20:00:05.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:05 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:05.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:05 smithi028 ceph-mon[31178]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T20:00:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:05 smithi154 ceph-mon[36314]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-05-20T20:00:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:05 smithi154 ceph-mon[36314]: from='osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T20:00:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:05 smithi154 ceph-mon[36314]: osdmap e68: 12 total, 11 up, 12 in 2024-05-20T20:00:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:05 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:05.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:05 smithi154 ceph-mon[36314]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T20:00:05.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:00:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:00:04.999+0000 7f9b4c068700 -1 osd.11 0 waiting for initial osdmap 2024-05-20T20:00:05.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:00:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:00:05.015+0000 7f9b47e5b700 -1 osd.11 69 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T20:00:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:05 smithi033 ceph-mon[36179]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-05-20T20:00:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:05 smithi033 ceph-mon[36179]: from='osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T20:00:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:05 smithi033 ceph-mon[36179]: osdmap e68: 12 total, 11 up, 12 in 2024-05-20T20:00:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:05 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:05 smithi033 ceph-mon[36179]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-05-20T20:00:06.085 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:06 smithi154 ceph-mon[36314]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T20:00:06.085 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:06 smithi154 ceph-mon[36314]: osdmap e69: 12 total, 11 up, 12 in 2024-05-20T20:00:06.085 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:06.085 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:06.085 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:06 smithi154 ceph-mon[36314]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:06.277 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:06 smithi028 ceph-mon[31178]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T20:00:06.277 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:06 smithi028 ceph-mon[31178]: osdmap e69: 12 total, 11 up, 12 in 2024-05-20T20:00:06.277 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:06.277 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:06.277 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:06 smithi028 ceph-mon[31178]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:06.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:06 smithi033 ceph-mon[36179]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-05-20T20:00:06.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:06 smithi033 ceph-mon[36179]: osdmap e69: 12 total, 11 up, 12 in 2024-05-20T20:00:06.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:06.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:06.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:06 smithi033 ceph-mon[36179]: pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 310 MiB used, 983 GiB / 983 GiB avail 2024-05-20T20:00:07.258 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:07 smithi154 ceph-mon[36314]: purged_snaps scrub starts 2024-05-20T20:00:07.259 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:07 smithi154 ceph-mon[36314]: purged_snaps scrub ok 2024-05-20T20:00:07.259 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:07.259 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:07 smithi154 ceph-mon[36314]: osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] boot 2024-05-20T20:00:07.259 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:07 smithi154 ceph-mon[36314]: osdmap e70: 12 total, 12 up, 12 in 2024-05-20T20:00:07.259 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:07.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:07 smithi028 ceph-mon[31178]: purged_snaps scrub starts 2024-05-20T20:00:07.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:07 smithi028 ceph-mon[31178]: purged_snaps scrub ok 2024-05-20T20:00:07.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:07.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:07 smithi028 ceph-mon[31178]: osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] boot 2024-05-20T20:00:07.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:07 smithi028 ceph-mon[31178]: osdmap e70: 12 total, 12 up, 12 in 2024-05-20T20:00:07.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:07.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:07 smithi033 ceph-mon[36179]: purged_snaps scrub starts 2024-05-20T20:00:07.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:07 smithi033 ceph-mon[36179]: purged_snaps scrub ok 2024-05-20T20:00:07.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:07.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:07 smithi033 ceph-mon[36179]: osd.11 [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] boot 2024-05-20T20:00:07.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:07 smithi033 ceph-mon[36179]: osdmap e70: 12 total, 12 up, 12 in 2024-05-20T20:00:07.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 -- 172.21.15.28:0/3415541731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee54077570 msgr2=0x7fee540759d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 --2- 172.21.15.28:0/3415541731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee54077570 0x7fee540759d0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fee48009990 tx=0x7fee48009ca0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 -- 172.21.15.28:0/3415541731 shutdown_connections 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 --2- 172.21.15.28:0/3415541731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee5410b480 0x7fee5410d910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 --2- 172.21.15.28:0/3415541731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee54077570 0x7fee540759d0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 --2- 172.21.15.28:0/3415541731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 0x7fee54076420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 -- 172.21.15.28:0/3415541731 >> 172.21.15.28:0/3415541731 conn(0x7fee540feb50 msgr2=0x7fee54100f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 -- 172.21.15.28:0/3415541731 shutdown_connections 2024-05-20T20:00:07.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.658+0000 7fee5c000700 1 -- 172.21.15.28:0/3415541731 wait complete. 2024-05-20T20:00:07.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.659+0000 7fee5c000700 1 Processor -- start 2024-05-20T20:00:07.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.659+0000 7fee5c000700 1 -- start start 2024-05-20T20:00:07.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.659+0000 7fee5c000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 0x7fee5419b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:07.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee5c000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee54077570 0x7fee5419be50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee5c000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee5410b480 0x7fee541a2ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee5c000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fee54111270 con 0x7fee5410b480 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee5c000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fee541110f0 con 0x7fee54075fa0 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee5c000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fee541113f0 con 0x7fee54077570 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee59600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 0x7fee5419b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee58c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee54077570 0x7fee5419be50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee59600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 0x7fee5419b910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48268/0 (socket says 172.21.15.28:48268) 2024-05-20T20:00:07.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee59600700 1 -- 172.21.15.28:0/3353247966 learned_addr learned my addr 172.21.15.28:0/3353247966 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.660+0000 7fee5a000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee5410b480 0x7fee541a2ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee59600700 1 -- 172.21.15.28:0/3353247966 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee54077570 msgr2=0x7fee5419be50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee59600700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee54077570 0x7fee5419be50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee59600700 1 -- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee5410b480 msgr2=0x7fee541a2ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee59600700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee5410b480 0x7fee541a2ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee59600700 1 -- 172.21.15.28:0/3353247966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fee541a3570 con 0x7fee54075fa0 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee59600700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 0x7fee5419b910 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fee480047b0 tx=0x7fee48009510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee52000700 1 -- 172.21.15.28:0/3353247966 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee48026070 con 0x7fee54075fa0 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee52000700 1 -- 172.21.15.28:0/3353247966 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fee48021410 con 0x7fee54075fa0 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee52000700 1 -- 172.21.15.28:0/3353247966 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee48017650 con 0x7fee54075fa0 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.661+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fee541a3800 con 0x7fee54075fa0 2024-05-20T20:00:07.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.662+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fee541a3c00 con 0x7fee54075fa0 2024-05-20T20:00:07.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.663+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fee5406a230 con 0x7fee54075fa0 2024-05-20T20:00:07.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.665+0000 7fee52000700 1 -- 172.21.15.28:0/3353247966 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fee48029070 con 0x7fee54075fa0 2024-05-20T20:00:07.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.666+0000 7fee52000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fee3c0b2900 0x7fee3c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:07.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.667+0000 7fee52000700 1 -- 172.21.15.28:0/3353247966 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7fee480e2a50 con 0x7fee54075fa0 2024-05-20T20:00:07.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.668+0000 7fee58c00700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fee3c0b2900 0x7fee3c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:07.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.669+0000 7fee58c00700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fee3c0b2900 0x7fee3c0b4dc0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fee540772a0 tx=0x7fee40009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:07.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.671+0000 7fee52000700 1 -- 172.21.15.28:0/3353247966 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fee4805edf0 con 0x7fee54075fa0 2024-05-20T20:00:07.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.917+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) -- 0x7fee54064150 con 0x7fee54075fa0 2024-05-20T20:00:07.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.918+0000 7fee52000700 1 -- 172.21.15.28:0/3353247966 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v71) ==== 74+0+133 (secure 0 0 0) 0x7fee48013070 con 0x7fee54075fa0 2024-05-20T20:00:07.919 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:07.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fee3c0b2900 msgr2=0x7fee3c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:07.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fee3c0b2900 0x7fee3c0b4dc0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fee540772a0 tx=0x7fee40009260 comp rx=0 tx=0).stop 2024-05-20T20:00:07.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 msgr2=0x7fee5419b910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:07.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 0x7fee5419b910 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fee480047b0 tx=0x7fee48009510 comp rx=0 tx=0).stop 2024-05-20T20:00:07.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 shutdown_connections 2024-05-20T20:00:07.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fee3c0b2900 0x7fee3c0b4dc0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee5410b480 0x7fee541a2ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee54075fa0 0x7fee5419b910 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 --2- 172.21.15.28:0/3353247966 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee54077570 0x7fee5419be50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:07.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 >> 172.21.15.28:0/3353247966 conn(0x7fee540feb50 msgr2=0x7fee54107380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:07.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 shutdown_connections 2024-05-20T20:00:07.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:07.921+0000 7fee5c000700 1 -- 172.21.15.28:0/3353247966 wait complete. 2024-05-20T20:00:08.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: osdmap e71: 12 total, 12 up, 12 in 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: Detected new or changed devices on smithi154 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi154 to 4272M 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:08.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:08.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:08 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3353247966' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T20:00:08.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: osdmap e71: 12 total, 12 up, 12 in 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: Detected new or changed devices on smithi154 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi154 to 4272M 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:08.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:08.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:08 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3353247966' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T20:00:08.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: osdmap e71: 12 total, 12 up, 12 in 2024-05-20T20:00:08.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:08.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: Detected new or changed devices on smithi154 2024-05-20T20:00:08.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi154 to 4272M 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:08.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:08 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3353247966' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T20:00:08.714 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":71,"num_osds":12,"num_up_osds":12,"osd_up_since":1716235206,"num_in_osds":12,"osd_in_since":1716235186,"num_remapped_pgs":0} 2024-05-20T20:00:08.714 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:00:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:10.732 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:10 smithi028 ceph-mon[31178]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:10 smithi154 ceph-mon[36314]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:10.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:10 smithi033 ceph-mon[36179]: pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 337 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.374+0000 7effdae00700 1 -- 172.21.15.28:0/467109798 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 msgr2=0x7effd4110470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.374+0000 7effdae00700 1 --2- 172.21.15.28:0/467109798 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 0x7effd4110470 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7effcc0098f0 tx=0x7effcc009c00 comp rx=0 tx=0).stop 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.375+0000 7effdae00700 1 -- 172.21.15.28:0/467109798 shutdown_connections 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.375+0000 7effdae00700 1 --2- 172.21.15.28:0/467109798 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 0x7effd4110470 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.375+0000 7effdae00700 1 --2- 172.21.15.28:0/467109798 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 0x7effd4103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.375+0000 7effdae00700 1 --2- 172.21.15.28:0/467109798 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd41041a0 0x7effd41081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.375+0000 7effdae00700 1 -- 172.21.15.28:0/467109798 >> 172.21.15.28:0/467109798 conn(0x7effd407a6a0 msgr2=0x7effd407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.375+0000 7effdae00700 1 -- 172.21.15.28:0/467109798 shutdown_connections 2024-05-20T20:00:11.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.375+0000 7effdae00700 1 -- 172.21.15.28:0/467109798 wait complete. 2024-05-20T20:00:11.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.376+0000 7effdae00700 1 Processor -- start 2024-05-20T20:00:11.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.376+0000 7effdae00700 1 -- start start 2024-05-20T20:00:11.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effdae00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 0x7effd4075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:11.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effdae00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd41041a0 0x7effd4075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:11.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effdae00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 0x7effd40762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:11.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effdae00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7effd4113710 con 0x7effd41041a0 2024-05-20T20:00:11.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effdae00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7effd4113590 con 0x7effd41037f0 2024-05-20T20:00:11.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effdae00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7effd4113890 con 0x7effd41088f0 2024-05-20T20:00:11.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 0x7effd4075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:11.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effd3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd41041a0 0x7effd4075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:11.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 0x7effd4075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48280/0 (socket says 172.21.15.28:48280) 2024-05-20T20:00:11.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.377+0000 7effd3e00700 1 -- 172.21.15.28:0/2376380378 learned_addr learned my addr 172.21.15.28:0/2376380378 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:11.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd8e00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 0x7effd40762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd3e00700 1 -- 172.21.15.28:0/2376380378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 msgr2=0x7effd40762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd3e00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 0x7effd40762c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd3e00700 1 -- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd41041a0 msgr2=0x7effd4075d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd3e00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd41041a0 0x7effd4075d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd3e00700 1 -- 172.21.15.28:0/2376380378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7effd41a80e0 con 0x7effd41037f0 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd8e00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 0x7effd40762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd3400700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd41041a0 0x7effd4075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.378+0000 7effd3e00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 0x7effd4075840 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7effc400c0c0 tx=0x7effc400c3d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.379+0000 7effd0c00700 1 -- 172.21.15.28:0/2376380378 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effc40092e0 con 0x7effd41037f0 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.379+0000 7effd0c00700 1 -- 172.21.15.28:0/2376380378 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7effc40075c0 con 0x7effd41037f0 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.379+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7effd41a8370 con 0x7effd41037f0 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.379+0000 7effd0c00700 1 -- 172.21.15.28:0/2376380378 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effc40146e0 con 0x7effd41037f0 2024-05-20T20:00:11.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.379+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7effd41a88b0 con 0x7effd41037f0 2024-05-20T20:00:11.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.380+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7effd406a230 con 0x7effd41037f0 2024-05-20T20:00:11.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.382+0000 7effd0c00700 1 -- 172.21.15.28:0/2376380378 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7effc401b020 con 0x7effd41037f0 2024-05-20T20:00:11.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.384+0000 7effd0c00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7effb80b2880 0x7effb80b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:11.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.384+0000 7effd0c00700 1 -- 172.21.15.28:0/2376380378 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7effc40e1c00 con 0x7effd41037f0 2024-05-20T20:00:11.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.384+0000 7effd3400700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7effb80b2880 0x7effb80b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:11.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.385+0000 7effd3400700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7effb80b2880 0x7effb80b4d40 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7effc0009680 tx=0x7effc0009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:11.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.386+0000 7effd0c00700 1 -- 172.21.15.28:0/2376380378 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7effc4010040 con 0x7effd41037f0 2024-05-20T20:00:11.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.624+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7effd402e720 con 0x7effd41037f0 2024-05-20T20:00:11.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.625+0000 7effd0c00700 1 -- 172.21.15.28:0/2376380378 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v71) ==== 74+0+18383 (secure 0 0 0) 0x7effc40a9d20 con 0x7effd41037f0 2024-05-20T20:00:11.627 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:11.627 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":71,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:00:07.014619+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":47,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:00:11.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7effb80b2880 msgr2=0x7effb80b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7effb80b2880 0x7effb80b4d40 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7effc0009680 tx=0x7effc0009260 comp rx=0 tx=0).stop 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 msgr2=0x7effd4075840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 0x7effd4075840 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7effc400c0c0 tx=0x7effc400c3d0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 shutdown_connections 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7effb80b2880 0x7effb80b4d40 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd41088f0 0x7effd40762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd41037f0 0x7effd4075840 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 --2- 172.21.15.28:0/2376380378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd41041a0 0x7effd4075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 >> 172.21.15.28:0/2376380378 conn(0x7effd407a6a0 msgr2=0x7effd41018b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 shutdown_connections 2024-05-20T20:00:11.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:11.629+0000 7effdae00700 1 -- 172.21.15.28:0/2376380378 wait complete. 2024-05-20T20:00:12.453 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-20T19:55:47.376785+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-05-20T20:00:12.453 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-20T20:00:12.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:12 smithi028 ceph-mon[31178]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:12.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:12 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2376380378' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:00:12.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:12 smithi154 ceph-mon[36314]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:12.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:12 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2376380378' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:00:12.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:12 smithi033 ceph-mon[36179]: pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:12.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:12 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2376380378' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:00:12.918 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:14.512 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:14 smithi028 ceph-mon[31178]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:14.677 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:14 smithi154 ceph-mon[36314]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:14.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:14 smithi033 ceph-mon[36179]: pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:14.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.994+0000 7fcd20e00700 1 -- 172.21.15.28:0/3500510942 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 msgr2=0x7fcd1c106d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:14.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.994+0000 7fcd20e00700 1 --2- 172.21.15.28:0/3500510942 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c106d50 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fcd1400b0a0 tx=0x7fcd1400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:00:14.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.995+0000 7fcd20e00700 1 -- 172.21.15.28:0/3500510942 shutdown_connections 2024-05-20T20:00:14.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.995+0000 7fcd20e00700 1 --2- 172.21.15.28:0/3500510942 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c106d50 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:14.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.995+0000 7fcd20e00700 1 --2- 172.21.15.28:0/3500510942 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcd1c06b3e0 0x7fcd1c102160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:14.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.995+0000 7fcd20e00700 1 --2- 172.21.15.28:0/3500510942 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd1c06b850 0x7fcd1c1026a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:14.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.995+0000 7fcd20e00700 1 -- 172.21.15.28:0/3500510942 >> 172.21.15.28:0/3500510942 conn(0x7fcd1c077780 msgr2=0x7fcd1c077b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:14.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.995+0000 7fcd20e00700 1 -- 172.21.15.28:0/3500510942 shutdown_connections 2024-05-20T20:00:14.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.995+0000 7fcd20e00700 1 -- 172.21.15.28:0/3500510942 wait complete. 2024-05-20T20:00:14.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.996+0000 7fcd20e00700 1 Processor -- start 2024-05-20T20:00:14.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.996+0000 7fcd20e00700 1 -- start start 2024-05-20T20:00:14.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.996+0000 7fcd20e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcd1c06b3e0 0x7fcd1c19f810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:14.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.996+0000 7fcd20e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd1c06b850 0x7fcd1c19fd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:14.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.996+0000 7fcd20e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c1999d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:14.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.996+0000 7fcd20e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fcd1c112690 con 0x7fcd1c1068d0 2024-05-20T20:00:14.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd20e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fcd1c112510 con 0x7fcd1c06b3e0 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd20e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fcd1c112810 con 0x7fcd1c06b850 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c1999d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c1999d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48224/0 (socket says 172.21.15.28:48224) 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 -- 172.21.15.28:0/1285644269 learned_addr learned my addr 172.21.15.28:0/1285644269 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd19400700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd1c06b850 0x7fcd1c19fd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd19e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcd1c06b3e0 0x7fcd1c19f810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 -- 172.21.15.28:0/1285644269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd1c06b850 msgr2=0x7fcd1c19fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd1c06b850 0x7fcd1c19fd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 -- 172.21.15.28:0/1285644269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcd1c06b3e0 msgr2=0x7fcd1c19f810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcd1c06b3e0 0x7fcd1c19f810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:14.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.997+0000 7fcd1a800700 1 -- 172.21.15.28:0/1285644269 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcd1c19a130 con 0x7fcd1c1068d0 2024-05-20T20:00:15.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.998+0000 7fcd1a800700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c1999d0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fcd1400b070 tx=0x7fcd14009c90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:15.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.998+0000 7fcd12a00700 1 -- 172.21.15.28:0/1285644269 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd14021070 con 0x7fcd1c1068d0 2024-05-20T20:00:15.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.998+0000 7fcd12a00700 1 -- 172.21.15.28:0/1285644269 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcd14004e40 con 0x7fcd1c1068d0 2024-05-20T20:00:15.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.998+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcd1c19a3c0 con 0x7fcd1c1068d0 2024-05-20T20:00:15.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.998+0000 7fcd12a00700 1 -- 172.21.15.28:0/1285644269 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd14004630 con 0x7fcd1c1068d0 2024-05-20T20:00:15.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.998+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcd1c1a66e0 con 0x7fcd1c1068d0 2024-05-20T20:00:15.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:14.999+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcd1c06a230 con 0x7fcd1c1068d0 2024-05-20T20:00:15.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.002+0000 7fcd12a00700 1 -- 172.21.15.28:0/1285644269 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fcd14007980 con 0x7fcd1c1068d0 2024-05-20T20:00:15.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.004+0000 7fcd12a00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcd000b2900 0x7fcd000b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:15.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.004+0000 7fcd12a00700 1 -- 172.21.15.28:0/1285644269 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7fcd1400f020 con 0x7fcd1c1068d0 2024-05-20T20:00:15.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.004+0000 7fcd19e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcd000b2900 0x7fcd000b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:15.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.005+0000 7fcd19e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcd000b2900 0x7fcd000b4dc0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fcd1c19ab10 tx=0x7fcd0c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:15.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.006+0000 7fcd12a00700 1 -- 172.21.15.28:0/1285644269 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fcd1402a020 con 0x7fcd1c1068d0 2024-05-20T20:00:15.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.243+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fcd1c1a68b0 con 0x7fcd1c1068d0 2024-05-20T20:00:15.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.244+0000 7fcd12a00700 1 -- 172.21.15.28:0/1285644269 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v71) ==== 93+0+10 (secure 0 0 0) 0x7fcd140ab6c0 con 0x7fcd1c1068d0 2024-05-20T20:00:15.245 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 1 2024-05-20T20:00:15.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.247+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcd000b2900 msgr2=0x7fcd000b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:15.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.247+0000 7fcd20e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcd000b2900 0x7fcd000b4dc0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fcd1c19ab10 tx=0x7fcd0c009260 comp rx=0 tx=0).stop 2024-05-20T20:00:15.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 msgr2=0x7fcd1c1999d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:15.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c1999d0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fcd1400b070 tx=0x7fcd14009c90 comp rx=0 tx=0).stop 2024-05-20T20:00:15.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 shutdown_connections 2024-05-20T20:00:15.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcd000b2900 0x7fcd000b4dc0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:15.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcd1c1068d0 0x7fcd1c1999d0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:15.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcd1c06b3e0 0x7fcd1c19f810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:15.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 --2- 172.21.15.28:0/1285644269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd1c06b850 0x7fcd1c19fd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:15.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 >> 172.21.15.28:0/1285644269 conn(0x7fcd1c077780 msgr2=0x7fcd1c108b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:15.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 shutdown_connections 2024-05-20T20:00:15.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:15.248+0000 7fcd20e00700 1 -- 172.21.15.28:0/1285644269 wait complete. 2024-05-20T20:00:15.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:15 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1285644269' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:00:15.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:15 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1285644269' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:00:15.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:15 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1285644269' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:00:16.055 INFO:tasks.cephadm:Adding mds.a on smithi028 2024-05-20T20:00:16.055 INFO:tasks.cephadm:Adding mds.d on smithi028 2024-05-20T20:00:16.055 INFO:tasks.cephadm:Adding mds.g on smithi028 2024-05-20T20:00:16.055 INFO:tasks.cephadm:Adding mds.j on smithi028 2024-05-20T20:00:16.055 INFO:tasks.cephadm:Adding mds.b on smithi033 2024-05-20T20:00:16.055 INFO:tasks.cephadm:Adding mds.e on smithi033 2024-05-20T20:00:16.055 INFO:tasks.cephadm:Adding mds.h on smithi033 2024-05-20T20:00:16.056 INFO:tasks.cephadm:Adding mds.k on smithi033 2024-05-20T20:00:16.056 INFO:tasks.cephadm:Adding mds.c on smithi154 2024-05-20T20:00:16.056 INFO:tasks.cephadm:Adding mds.f on smithi154 2024-05-20T20:00:16.056 INFO:tasks.cephadm:Adding mds.i on smithi154 2024-05-20T20:00:16.056 INFO:tasks.cephadm:Adding mds.l on smithi154 2024-05-20T20:00:16.056 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph orch apply mds all '12;smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l' 2024-05-20T20:00:16.291 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T20:00:16.669 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:16 smithi154 ceph-mon[36314]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:16 smithi028 ceph-mon[31178]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:16.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:16 smithi033 ceph-mon[36179]: pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:17.934 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 -- 172.21.15.154:0/638938564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 msgr2=0x7fd4881039c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:17.934 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/638938564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4881039c0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fd484009990 tx=0x7fd484009ca0 comp rx=0 tx=0).stop 2024-05-20T20:00:17.934 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 -- 172.21.15.154:0/638938564 shutdown_connections 2024-05-20T20:00:17.934 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/638938564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd488105140 0x7fd48810b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:17.934 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/638938564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4881039c0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:17.934 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/638938564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4881047a0 0x7fd488104c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 -- 172.21.15.154:0/638938564 >> 172.21.15.154:0/638938564 conn(0x7fd4880feb40 msgr2=0x7fd488100f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 -- 172.21.15.154:0/638938564 shutdown_connections 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 -- 172.21.15.154:0/638938564 wait complete. 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 Processor -- start 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 -- start start 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4880772e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.932+0000 7fd48fb5c700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4881047a0 0x7fd488075930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48fb5c700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd488105140 0x7fd488075e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48fb5c700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd48810f090 con 0x7fd4881047a0 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48fb5c700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd48810ef10 con 0x7fd488105140 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48fb5c700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd48810f210 con 0x7fd4881035a0 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4880772e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4880772e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:38756/0 (socket says 172.21.15.154:38756) 2024-05-20T20:00:17.935 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 -- 172.21.15.154:0/1580602188 learned_addr learned my addr 172.21.15.154:0/1580602188 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48e0f9700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd488105140 0x7fd488075e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 -- 172.21.15.154:0/1580602188 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd488105140 msgr2=0x7fd488075e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd488105140 0x7fd488075e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 -- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4881047a0 msgr2=0x7fd488075930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4881047a0 0x7fd488075930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 -- 172.21.15.154:0/1580602188 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd488076720 con 0x7fd4881035a0 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48d8f8700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4880772e0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fd4840036f0 tx=0x7fd484003c00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd47effd700 1 -- 172.21.15.154:0/1580602188 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd484027070 con 0x7fd4881035a0 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd4881a3840 con 0x7fd4881035a0 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd47effd700 1 -- 172.21.15.154:0/1580602188 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd48400bd90 con 0x7fd4881035a0 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd47effd700 1 -- 172.21.15.154:0/1580602188 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd48400f460 con 0x7fd4881035a0 2024-05-20T20:00:17.936 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.933+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd4881a3c40 con 0x7fd4881035a0 2024-05-20T20:00:17.937 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.934+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd48806a230 con 0x7fd4881035a0 2024-05-20T20:00:17.937 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.936+0000 7fd47effd700 1 -- 172.21.15.154:0/1580602188 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fd48402a070 con 0x7fd4881035a0 2024-05-20T20:00:17.937 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.937+0000 7fd47effd700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd4740b2880 0x7fd4740b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:17.937 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.937+0000 7fd47effd700 1 -- 172.21.15.154:0/1580602188 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7fd4840e3140 con 0x7fd4881035a0 2024-05-20T20:00:17.938 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.938+0000 7fd48d0f7700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd4740b2880 0x7fd4740b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:17.938 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.938+0000 7fd48d0f7700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd4740b2880 0x7fd4740b4d40 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fd4780097b0 tx=0x7fd478006d20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:17.940 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:17.940+0000 7fd47effd700 1 -- 172.21.15.154:0/1580602188 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fd484013070 con 0x7fd4881035a0 2024-05-20T20:00:18.157 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.156+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l", "target": ["mon-mgr", ""]}) -- 0x7fd488063c40 con 0x7fd4740b2880 2024-05-20T20:00:18.173 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.173+0000 7fd47effd700 1 -- 172.21.15.154:0/1580602188 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+28 (secure 0 0 0) 0x7fd488063c40 con 0x7fd4740b2880 2024-05-20T20:00:18.174 INFO:teuthology.orchestra.run.smithi154.stdout:Scheduled mds.all update... 2024-05-20T20:00:18.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd4740b2880 msgr2=0x7fd4740b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:18.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd4740b2880 0x7fd4740b4d40 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fd4780097b0 tx=0x7fd478006d20 comp rx=0 tx=0).stop 2024-05-20T20:00:18.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 msgr2=0x7fd4880772e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:18.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4880772e0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fd4840036f0 tx=0x7fd484003c00 comp rx=0 tx=0).stop 2024-05-20T20:00:18.176 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 shutdown_connections 2024-05-20T20:00:18.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd4740b2880 0x7fd4740b4d40 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:18.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd488105140 0x7fd488075e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:18.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4881035a0 0x7fd4880772e0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:18.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 --2- 172.21.15.154:0/1580602188 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4881047a0 0x7fd488075930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:18.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 >> 172.21.15.154:0/1580602188 conn(0x7fd4880feb40 msgr2=0x7fd488107bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:18.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 shutdown_connections 2024-05-20T20:00:18.177 INFO:teuthology.orchestra.run.smithi154.stderr:2024-05-20T20:00:18.176+0000 7fd48fb5c700 1 -- 172.21.15.154:0/1580602188 wait complete. 2024-05-20T20:00:18.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:18 smithi028 ceph-mon[31178]: pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:18.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:18.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:18.808 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:18 smithi154 ceph-mon[36314]: pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:18.808 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:18.808 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:18.839 DEBUG:teuthology.orchestra.run.smithi028:mds.a> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.a.service 2024-05-20T20:00:18.849 DEBUG:teuthology.orchestra.run.smithi028:mds.d> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.d.service 2024-05-20T20:00:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:18 smithi033 ceph-mon[36179]: pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:00:18.894 DEBUG:teuthology.orchestra.run.smithi028:mds.g> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.g.service 2024-05-20T20:00:18.941 DEBUG:teuthology.orchestra.run.smithi028:mds.j> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.j.service 2024-05-20T20:00:18.987 DEBUG:teuthology.orchestra.run.smithi033:mds.b> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.b.service 2024-05-20T20:00:18.989 DEBUG:teuthology.orchestra.run.smithi033:mds.e> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.e.service 2024-05-20T20:00:19.032 DEBUG:teuthology.orchestra.run.smithi033:mds.h> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.h.service 2024-05-20T20:00:19.075 DEBUG:teuthology.orchestra.run.smithi033:mds.k> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.k.service 2024-05-20T20:00:19.118 DEBUG:teuthology.orchestra.run.smithi154:mds.c> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.c.service 2024-05-20T20:00:19.120 DEBUG:teuthology.orchestra.run.smithi154:mds.f> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.f.service 2024-05-20T20:00:19.163 DEBUG:teuthology.orchestra.run.smithi154:mds.i> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.i.service 2024-05-20T20:00:19.206 DEBUG:teuthology.orchestra.run.smithi154:mds.l> sudo journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.l.service 2024-05-20T20:00:19.249 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-05-20T20:00:19.249 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:00:19.708 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:19.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:00:19.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: Saving service mds.all spec with placement smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l;count:12 2024-05-20T20:00:19.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:19.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:19.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:19.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:19.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:19.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:19.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:00:19.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: Saving service mds.all spec with placement smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l;count:12 2024-05-20T20:00:19.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:19.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:19.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:19.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:19.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:19.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: from='client.24415 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:00:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: Saving service mds.all spec with placement smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l;count:12 2024-05-20T20:00:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:00:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:19.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:20.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:20 smithi028 ceph-mon[31178]: pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:20.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:20 smithi028 ceph-mon[31178]: Deploying daemon mds.l on smithi154 2024-05-20T20:00:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:20 smithi154 ceph-mon[36314]: pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:20 smithi154 ceph-mon[36314]: Deploying daemon mds.l on smithi154 2024-05-20T20:00:20.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:20 smithi033 ceph-mon[36179]: pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:20.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:20 smithi033 ceph-mon[36179]: Deploying daemon mds.l on smithi154 2024-05-20T20:00:21.846 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:21 smithi154 systemd[1]: Starting Ceph mds.l for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:22.101 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:21 smithi154 podman[66538]: 2024-05-20 20:00:21.748462302 +0000 UTC m=+0.016900807 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:22.102 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:21 smithi154 podman[66538]: 2024-05-20 20:00:21.863785266 +0000 UTC m=+0.132223783 container create 07de68c6d6f34fe50bce2dbb31c9ee819d4679ac545aee7c05050cd9ff474fe1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:00:22.102 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:22 smithi154 podman[66538]: 2024-05-20 20:00:22.101841317 +0000 UTC m=+0.370279805 container init 07de68c6d6f34fe50bce2dbb31c9ee819d4679ac545aee7c05050cd9ff474fe1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5) 2024-05-20T20:00:22.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.104+0000 7f027be00700 1 -- 172.21.15.28:0/3293264879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 msgr2=0x7f027c10d4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:22.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.104+0000 7f027be00700 1 --2- 172.21.15.28:0/3293264879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 0x7f027c10d4c0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f02680098f0 tx=0x7f0268009c00 comp rx=0 tx=0).stop 2024-05-20T20:00:22.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.104+0000 7f027be00700 1 -- 172.21.15.28:0/3293264879 shutdown_connections 2024-05-20T20:00:22.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.104+0000 7f027be00700 1 --2- 172.21.15.28:0/3293264879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f027c10da90 0x7f027c10ff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.104+0000 7f027be00700 1 --2- 172.21.15.28:0/3293264879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 0x7f027c104a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.104+0000 7f027be00700 1 --2- 172.21.15.28:0/3293264879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 0x7f027c10d4c0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.104+0000 7f027be00700 1 -- 172.21.15.28:0/3293264879 >> 172.21.15.28:0/3293264879 conn(0x7f027c0fde60 msgr2=0x7f027c100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:22.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.105+0000 7f027be00700 1 -- 172.21.15.28:0/3293264879 shutdown_connections 2024-05-20T20:00:22.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.105+0000 7f027be00700 1 -- 172.21.15.28:0/3293264879 wait complete. 2024-05-20T20:00:22.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 Processor -- start 2024-05-20T20:00:22.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 -- start start 2024-05-20T20:00:22.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 0x7f027c105530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:22.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 0x7f027c105a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:22.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f027c10da90 0x7f027c105fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:22.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f027c113630 con 0x7f027c104fd0 2024-05-20T20:00:22.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f027c1134b0 con 0x7f027c1046b0 2024-05-20T20:00:22.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.106+0000 7f027be00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f027c1137b0 con 0x7f027c10da90 2024-05-20T20:00:22.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 0x7f027c105530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:22.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 0x7f027c105530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:39554/0 (socket says 172.21.15.28:39554) 2024-05-20T20:00:22.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 -- 172.21.15.28:0/3513932401 learned_addr learned my addr 172.21.15.28:0/3513932401 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:22.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027b400700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f027c10da90 0x7f027c105fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:22.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027a000700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 0x7f027c105a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:22.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 -- 172.21.15.28:0/3513932401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f027c10da90 msgr2=0x7f027c105fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:22.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f027c10da90 0x7f027c105fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 -- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 msgr2=0x7f027c105a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:22.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 0x7f027c105a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 -- 172.21.15.28:0/3513932401 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f027c1ac4d0 con 0x7f027c1046b0 2024-05-20T20:00:22.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.107+0000 7f027aa00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 0x7f027c105530 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f026c00e9d0 tx=0x7f026c00ece0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:22.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.108+0000 7f0273400700 1 -- 172.21.15.28:0/3513932401 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f026c00cbd0 con 0x7f027c1046b0 2024-05-20T20:00:22.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.108+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f027c1ac7c0 con 0x7f027c1046b0 2024-05-20T20:00:22.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.108+0000 7f0273400700 1 -- 172.21.15.28:0/3513932401 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f026c00ee70 con 0x7f027c1046b0 2024-05-20T20:00:22.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.108+0000 7f0273400700 1 -- 172.21.15.28:0/3513932401 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f026c018680 con 0x7f027c1046b0 2024-05-20T20:00:22.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.108+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f027c1acd20 con 0x7f027c1046b0 2024-05-20T20:00:22.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.109+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f025c005330 con 0x7f027c1046b0 2024-05-20T20:00:22.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.111+0000 7f0273400700 1 -- 172.21.15.28:0/3513932401 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f026c018820 con 0x7f027c1046b0 2024-05-20T20:00:22.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.113+0000 7f0273400700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02600b2900 0x7f02600b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:22.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.113+0000 7f0273400700 1 -- 172.21.15.28:0/3513932401 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f026c0e1ec0 con 0x7f027c1046b0 2024-05-20T20:00:22.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.113+0000 7f027a000700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02600b2900 0x7f02600b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:22.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.114+0000 7f027a000700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02600b2900 0x7f02600b4dc0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f0268009810 tx=0x7f0268019040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:22.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.118+0000 7f0273400700 1 -- 172.21.15.28:0/3513932401 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f026c0a9fe0 con 0x7f027c1046b0 2024-05-20T20:00:22.375 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:22 smithi154 podman[66538]: 2024-05-20 20:00:22.10489175 +0000 UTC m=+0.373330246 container start 07de68c6d6f34fe50bce2dbb31c9ee819d4679ac545aee7c05050cd9ff474fe1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD) 2024-05-20T20:00:22.375 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mds[66557]: -- 172.21.15.154:0/1924250497 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55994b021a40 con 0x55994aff4800 2024-05-20T20:00:22.376 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:22 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l[66553]: starting mds.l at 2024-05-20T20:00:22.376 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:22 smithi154 bash[66538]: 07de68c6d6f34fe50bce2dbb31c9ee819d4679ac545aee7c05050cd9ff474fe1 2024-05-20T20:00:22.376 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:00:22 smithi154 systemd[1]: Started Ceph mds.l for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:22.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.430+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f025c005620 con 0x7f027c1046b0 2024-05-20T20:00:22.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.431+0000 7f0273400700 1 -- 172.21.15.28:0/3513932401 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) ==== 93+0+1396 (secure 0 0 0) 0x7f026c0a98e0 con 0x7f027c1046b0 2024-05-20T20:00:22.431 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:22.431 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2}],"filesystems":[]} 2024-05-20T20:00:22.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02600b2900 msgr2=0x7f02600b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02600b2900 0x7f02600b4dc0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f0268009810 tx=0x7f0268019040 comp rx=0 tx=0).stop 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 msgr2=0x7f027c105530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 0x7f027c105530 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f026c00e9d0 tx=0x7f026c00ece0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 shutdown_connections 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02600b2900 0x7f02600b4dc0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f027c10da90 0x7f027c105fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f027c1046b0 0x7f027c105530 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 --2- 172.21.15.28:0/3513932401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f027c104fd0 0x7f027c105a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.434+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 >> 172.21.15.28:0/3513932401 conn(0x7f027c0fde60 msgr2=0x7f027c10ed90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.435+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 shutdown_connections 2024-05-20T20:00:22.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:22.435+0000 7f027be00700 1 -- 172.21.15.28:0/3513932401 wait complete. 2024-05-20T20:00:22.436 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 2 2024-05-20T20:00:22.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:22.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6832/2654302434,v1:172.21.15.154:6833/2654302434] up:boot 2024-05-20T20:00:22.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: fsmap 1 up:standby 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:22.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:22 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3513932401' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6832/2654302434,v1:172.21.15.154:6833/2654302434] up:boot 2024-05-20T20:00:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: fsmap 1 up:standby 2024-05-20T20:00:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:00:22.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:22.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:22.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:22.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:22 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3513932401' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6832/2654302434,v1:172.21.15.154:6833/2654302434] up:boot 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: fsmap 1 up:standby 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:22.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:22 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3513932401' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:23.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:23 smithi033 ceph-mon[36179]: Deploying daemon mds.k on smithi033 2024-05-20T20:00:23.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:23 smithi028 ceph-mon[31178]: Deploying daemon mds.k on smithi033 2024-05-20T20:00:23.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:23 smithi154 ceph-mon[36314]: Deploying daemon mds.k on smithi033 2024-05-20T20:00:24.666 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:24 smithi033 ceph-mon[36179]: pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:24.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:24 smithi028 ceph-mon[31178]: pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:24.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:24 smithi154 ceph-mon[36314]: pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:25.244 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:00:25.707 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: Deploying daemon mds.j on smithi028 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:26.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6832/260147664,v1:172.21.15.33:6833/260147664] up:boot 2024-05-20T20:00:26.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: fsmap 2 up:standby 2024-05-20T20:00:26.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:26 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:00:26.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: Deploying daemon mds.j on smithi028 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6832/260147664,v1:172.21.15.33:6833/260147664] up:boot 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: fsmap 2 up:standby 2024-05-20T20:00:26.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:26 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: Deploying daemon mds.j on smithi028 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:26.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6832/260147664,v1:172.21.15.33:6833/260147664] up:boot 2024-05-20T20:00:26.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: fsmap 2 up:standby 2024-05-20T20:00:26.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:26 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:00:28.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:28 smithi028 ceph-mon[31178]: pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:28.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:28 smithi154 ceph-mon[36314]: pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:28.879 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:28 smithi033 ceph-mon[36179]: pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 -- 172.21.15.28:0/3528662070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06bbe0 msgr2=0x7f062c1103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 --2- 172.21.15.28:0/3528662070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06bbe0 0x7f062c1103d0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f062400b0a0 tx=0x7f062400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:00:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 -- 172.21.15.28:0/3528662070 shutdown_connections 2024-05-20T20:00:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 --2- 172.21.15.28:0/3528662070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06bbe0 0x7f062c1103d0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 --2- 172.21.15.28:0/3528662070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f062c1095b0 0x7f062c109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 --2- 172.21.15.28:0/3528662070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c06b220 0x7f062c06b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 -- 172.21.15.28:0/3528662070 >> 172.21.15.28:0/3528662070 conn(0x7f062c0793f0 msgr2=0x7f062c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 -- 172.21.15.28:0/3528662070 shutdown_connections 2024-05-20T20:00:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.937+0000 7f0632600700 1 -- 172.21.15.28:0/3528662070 wait complete. 2024-05-20T20:00:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.938+0000 7f0632600700 1 Processor -- start 2024-05-20T20:00:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.938+0000 7f0632600700 1 -- start start 2024-05-20T20:00:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f0632600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06b220 0x7f062c19bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f0632600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f062c06bbe0 0x7f062c19c320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f0632600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c1095b0 0x7f062c1a06b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f0632600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f062c113740 con 0x7f062c06b220 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f0632600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f062c1135c0 con 0x7f062c1095b0 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f0632600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f062c1138c0 con 0x7f062c06bbe0 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06b220 0x7f062c19bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c1095b0 0x7f062c1a06b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c1095b0 0x7f062c1a06b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52202/0 (socket says 172.21.15.28:52202) 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 -- 172.21.15.28:0/3573278655 learned_addr learned my addr 172.21.15.28:0/3573278655 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062aa00700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f062c06bbe0 0x7f062c19c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 -- 172.21.15.28:0/3573278655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f062c06bbe0 msgr2=0x7f062c19c320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f062c06bbe0 0x7f062c19c320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 -- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06b220 msgr2=0x7f062c19bde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06b220 0x7f062c19bde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.939+0000 7f062be00700 1 -- 172.21.15.28:0/3573278655 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f062c1a0df0 con 0x7f062c1095b0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.940+0000 7f062be00700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c1095b0 0x7f062c1a06b0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f062400b070 tx=0x7f0624009c90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.940+0000 7f0623e00700 1 -- 172.21.15.28:0/3573278655 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0624021070 con 0x7f062c1095b0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.940+0000 7f0623e00700 1 -- 172.21.15.28:0/3573278655 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0624004e40 con 0x7f062c1095b0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.940+0000 7f0623e00700 1 -- 172.21.15.28:0/3573278655 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0624004630 con 0x7f062c1095b0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.940+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f062c1a1080 con 0x7f062c1095b0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.940+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f062c1a8960 con 0x7f062c1095b0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.942+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f062c104c70 con 0x7f062c1095b0 2024-05-20T20:00:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.944+0000 7f0623e00700 1 -- 172.21.15.28:0/3573278655 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f0624007980 con 0x7f062c1095b0 2024-05-20T20:00:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.945+0000 7f0623e00700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f06100b2880 0x7f06100b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.946+0000 7f0623e00700 1 -- 172.21.15.28:0/3573278655 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f06240e2be0 con 0x7f062c1095b0 2024-05-20T20:00:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.946+0000 7f062b400700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f06100b2880 0x7f06100b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.947+0000 7f062b400700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f06100b2880 0x7f06100b4d40 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f061c009be0 tx=0x7f061c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:28.948+0000 7f0623e00700 1 -- 172.21.15.28:0/3573278655 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f06240aad90 con 0x7f062c1095b0 2024-05-20T20:00:29.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.260+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f062c1a8b30 con 0x7f062c1095b0 2024-05-20T20:00:29.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.261+0000 7f0623e00700 1 -- 172.21.15.28:0/3573278655 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) ==== 93+0+2218 (secure 0 0 0) 0x7f06240aa690 con 0x7f062c1095b0 2024-05-20T20:00:29.262 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:29.262 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24455,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[]} 2024-05-20T20:00:29.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.264+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f06100b2880 msgr2=0x7f06100b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:29.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.264+0000 7f0632600700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f06100b2880 0x7f06100b4d40 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f061c009be0 tx=0x7f061c009260 comp rx=0 tx=0).stop 2024-05-20T20:00:29.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c1095b0 msgr2=0x7f062c1a06b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:29.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c1095b0 0x7f062c1a06b0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f062400b070 tx=0x7f0624009c90 comp rx=0 tx=0).stop 2024-05-20T20:00:29.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 shutdown_connections 2024-05-20T20:00:29.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f06100b2880 0x7f06100b4d40 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:29.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f062c1095b0 0x7f062c1a06b0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:29.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f062c06b220 0x7f062c19bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:29.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 --2- 172.21.15.28:0/3573278655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f062c06bbe0 0x7f062c19c320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:29.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 >> 172.21.15.28:0/3573278655 conn(0x7f062c0793f0 msgr2=0x7f062c100f10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:29.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 shutdown_connections 2024-05-20T20:00:29.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:29.265+0000 7f0632600700 1 -- 172.21.15.28:0/3573278655 wait complete. 2024-05-20T20:00:29.267 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 3 2024-05-20T20:00:29.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:29 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3573278655' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:29.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:29 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3573278655' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:29.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:29 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3573278655' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:30.637 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:30 smithi028 ceph-mon[31178]: pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:30.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:30 smithi154 ceph-mon[36314]: pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:30 smithi033 ceph-mon[36179]: pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:31.054 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:00:30 smithi028 systemd[1]: Started Ceph mds.j for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:32.125 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: Deploying daemon mds.d on smithi028 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:32.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.28:6834/34026174,v1:172.21.15.28:6835/34026174] up:boot 2024-05-20T20:00:32.210 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: fsmap 3 up:standby 2024-05-20T20:00:32.210 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:31 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:00:32.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: Deploying daemon mds.d on smithi028 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.28:6834/34026174,v1:172.21.15.28:6835/34026174] up:boot 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: fsmap 3 up:standby 2024-05-20T20:00:32.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:31 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: Deploying daemon mds.d on smithi028 2024-05-20T20:00:32.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:32.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.28:6834/34026174,v1:172.21.15.28:6835/34026174] up:boot 2024-05-20T20:00:32.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: fsmap 3 up:standby 2024-05-20T20:00:32.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:31 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:00:33.418 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:34.305 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:34 smithi028 systemd[1]: Starting Ceph mds.d for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:34.701 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:34 smithi028 ceph-mon[31178]: pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:34.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:34 smithi154 ceph-mon[36314]: pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:34.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:34 smithi033 ceph-mon[36179]: pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:35.056 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:34 smithi028 podman[72461]: 2024-05-20 20:00:34.803046989 +0000 UTC m=+0.052107093 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:35.056 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:34 smithi028 podman[72461]: 2024-05-20 20:00:34.924875349 +0000 UTC m=+0.173935460 container create e1a49fd899fd61b85d94a711b7d515f3cebda5d02a124547901d5ad56479c9dd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-20T20:00:35.727 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:35 smithi028 podman[72461]: 2024-05-20 20:00:35.465887 +0000 UTC m=+0.714947189 container init e1a49fd899fd61b85d94a711b7d515f3cebda5d02a124547901d5ad56479c9dd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T20:00:35.727 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:35 smithi028 podman[72461]: 2024-05-20 20:00:35.477338008 +0000 UTC m=+0.726398113 container start e1a49fd899fd61b85d94a711b7d515f3cebda5d02a124547901d5ad56479c9dd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-05-20T20:00:35.727 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:35 smithi028 ceph-mds[72503]: -- 172.21.15.28:0/755215371 <== mon.0 v2:172.21.15.28:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564b6bc75a40 con 0x564b6bc49000 2024-05-20T20:00:35.727 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d[72494]: starting mds.d at 2024-05-20T20:00:35.727 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:35 smithi028 bash[72461]: e1a49fd899fd61b85d94a711b7d515f3cebda5d02a124547901d5ad56479c9dd 2024-05-20T20:00:35.727 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:00:35 smithi028 systemd[1]: Started Ceph mds.d for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:35.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 -- 172.21.15.28:0/4251153890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 msgr2=0x7f8a9806b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 --2- 172.21.15.28:0/4251153890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9806b630 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f8a88009990 tx=0x7f8a88009ca0 comp rx=0 tx=0).stop 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 -- 172.21.15.28:0/4251153890 shutdown_connections 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 --2- 172.21.15.28:0/4251153890 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a9810fff0 0x7f8a981123e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 --2- 172.21.15.28:0/4251153890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9806b630 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 --2- 172.21.15.28:0/4251153890 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a9806bb70 0x7f8a9810fab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 -- 172.21.15.28:0/4251153890 >> 172.21.15.28:0/4251153890 conn(0x7f8a980793f0 msgr2=0x7f8a98079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 -- 172.21.15.28:0/4251153890 shutdown_connections 2024-05-20T20:00:35.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.902+0000 7f8a9f800700 1 -- 172.21.15.28:0/4251153890 wait complete. 2024-05-20T20:00:35.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.904+0000 7f8a9f800700 1 Processor -- start 2024-05-20T20:00:35.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.904+0000 7f8a9f800700 1 -- start start 2024-05-20T20:00:35.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9f800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9819be00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:35.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9f800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a9806bb70 0x7f8a9819c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:35.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9f800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a9810fff0 0x7f8a981a06d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9f800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8a981158f0 con 0x7f8a9806bb70 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9f800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8a98115770 con 0x7f8a9806b250 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9f800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8a98115a70 con 0x7f8a9810fff0 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9819be00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a8fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a9806bb70 0x7f8a9819c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9819be00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52266/0 (socket says 172.21.15.28:52266) 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9ce00700 1 -- 172.21.15.28:0/2998080172 learned_addr learned my addr 172.21.15.28:0/2998080172 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:35.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.905+0000 7f8a9d800700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a9810fff0 0x7f8a981a06d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:35.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a8fe00700 1 -- 172.21.15.28:0/2998080172 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a9810fff0 msgr2=0x7f8a981a06d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a8fe00700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a9810fff0 0x7f8a981a06d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a8fe00700 1 -- 172.21.15.28:0/2998080172 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 msgr2=0x7f8a9819be00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a8fe00700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9819be00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a8fe00700 1 -- 172.21.15.28:0/2998080172 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a981a0e10 con 0x7f8a9806bb70 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a9ce00700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9819be00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a8fe00700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a9806bb70 0x7f8a9819c340 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f8a8000e9f0 tx=0x7f8a8000ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a96000700 1 -- 172.21.15.28:0/2998080172 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a8000c9c0 con 0x7f8a9806bb70 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.906+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a981a1100 con 0x7f8a9806bb70 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.907+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a981a8980 con 0x7f8a9806bb70 2024-05-20T20:00:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.907+0000 7f8a96000700 1 -- 172.21.15.28:0/2998080172 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8a8000cb60 con 0x7f8a9806bb70 2024-05-20T20:00:35.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.907+0000 7f8a96000700 1 -- 172.21.15.28:0/2998080172 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a80018680 con 0x7f8a9806bb70 2024-05-20T20:00:35.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.908+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8a74005330 con 0x7f8a9806bb70 2024-05-20T20:00:35.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.912+0000 7f8a96000700 1 -- 172.21.15.28:0/2998080172 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f8a80018820 con 0x7f8a9806bb70 2024-05-20T20:00:35.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.913+0000 7f8a96000700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8a7c0b2880 0x7f8a7c0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:35.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.914+0000 7f8a96000700 1 -- 172.21.15.28:0/2998080172 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f8a800e21e0 con 0x7f8a9806bb70 2024-05-20T20:00:35.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.914+0000 7f8a9ce00700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8a7c0b2880 0x7f8a7c0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:35.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.915+0000 7f8a9ce00700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8a7c0b2880 0x7f8a7c0b4d40 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f8a88009750 tx=0x7f8a8801a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:35.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:35.916+0000 7f8a96000700 1 -- 172.21.15.28:0/2998080172 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f8a800aa300 con 0x7f8a9806bb70 2024-05-20T20:00:36.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.207+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f8a74005620 con 0x7f8a9806bb70 2024-05-20T20:00:36.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.208+0000 7f8a96000700 1 -- 172.21.15.28:0/2998080172 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 4 v4) ==== 93+0+3037 (secure 0 0 0) 0x7f8a800a9c00 con 0x7f8a9806bb70 2024-05-20T20:00:36.208 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:36.208 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":4,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24442,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24455,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3}],"filesystems":[]} 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.211+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8a7c0b2880 msgr2=0x7f8a7c0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.211+0000 7f8a9f800700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8a7c0b2880 0x7f8a7c0b4d40 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f8a88009750 tx=0x7f8a8801a040 comp rx=0 tx=0).stop 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.211+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a9806bb70 msgr2=0x7f8a9819c340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.211+0000 7f8a9f800700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a9806bb70 0x7f8a9819c340 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f8a8000e9f0 tx=0x7f8a8000ed00 comp rx=0 tx=0).stop 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 shutdown_connections 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8a7c0b2880 0x7f8a7c0b4d40 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a9810fff0 0x7f8a981a06d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a9806b250 0x7f8a9819be00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:36.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 --2- 172.21.15.28:0/2998080172 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a9806bb70 0x7f8a9819c340 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:36.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 >> 172.21.15.28:0/2998080172 conn(0x7f8a980793f0 msgr2=0x7f8a981114c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:36.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 shutdown_connections 2024-05-20T20:00:36.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:36.212+0000 7f8a9f800700 1 -- 172.21.15.28:0/2998080172 wait complete. 2024-05-20T20:00:36.213 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 4 2024-05-20T20:00:36.962 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:36.962 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:36.962 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:36.962 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:36.962 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:36.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:36.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:36.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: Deploying daemon mds.e on smithi033 2024-05-20T20:00:36.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2998080172' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:36.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.28:6836/1950403416,v1:172.21.15.28:6837/1950403416] up:boot 2024-05-20T20:00:36.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: fsmap 4 up:standby 2024-05-20T20:00:36.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:36 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-05-20T20:00:36.990 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:36.990 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:36.990 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:36.990 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:36.990 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:36.990 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:36.990 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:36.991 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: Deploying daemon mds.e on smithi033 2024-05-20T20:00:36.991 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2998080172' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:36.991 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.28:6836/1950403416,v1:172.21.15.28:6837/1950403416] up:boot 2024-05-20T20:00:36.991 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: fsmap 4 up:standby 2024-05-20T20:00:36.991 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:36 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-05-20T20:00:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: Deploying daemon mds.e on smithi033 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2998080172' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.28:6836/1950403416,v1:172.21.15.28:6837/1950403416] up:boot 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: fsmap 4 up:standby 2024-05-20T20:00:37.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:36 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-05-20T20:00:38.336 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 systemd[1]: Starting Ceph mds.e for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:38.627 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 podman[65973]: 2024-05-20 20:00:38.239089114 +0000 UTC m=+0.017759523 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:38.628 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 podman[65973]: 2024-05-20 20:00:38.33868393 +0000 UTC m=+0.117354314 container create 4896299ff43d00ea5d50e3658f5221c80128dbaba94b48ded6bdc4547d357f8b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-20T20:00:38.628 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 podman[65973]: 2024-05-20 20:00:38.558453314 +0000 UTC m=+0.337123696 container init 4896299ff43d00ea5d50e3658f5221c80128dbaba94b48ded6bdc4547d357f8b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-20T20:00:38.628 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 podman[65973]: 2024-05-20 20:00:38.561259822 +0000 UTC m=+0.339930205 container start 4896299ff43d00ea5d50e3658f5221c80128dbaba94b48ded6bdc4547d357f8b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-20T20:00:38.628 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 ceph-mds[65993]: -- 172.21.15.33:0/1410415790 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5600c9133a40 con 0x5600c9106c00 2024-05-20T20:00:38.628 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e[65989]: starting mds.e at 2024-05-20T20:00:39.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:38 smithi028 ceph-mon[31178]: pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:39.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:39.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:38 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:39.065 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:00:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:38 smithi154 ceph-mon[36314]: pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:38 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:39.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:38 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:39.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:38 smithi033 ceph-mon[36179]: pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:39.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:38 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:39.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:38 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:39.128 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 bash[65973]: 4896299ff43d00ea5d50e3658f5221c80128dbaba94b48ded6bdc4547d357f8b 2024-05-20T20:00:39.128 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:00:38 smithi033 systemd[1]: Started Ceph mds.e for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:39.532 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] up:boot 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: fsmap 5 up:standby 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:39.993 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:39 smithi028 ceph-mon[31178]: Deploying daemon mds.f on smithi154 2024-05-20T20:00:40.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] up:boot 2024-05-20T20:00:40.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: fsmap 5 up:standby 2024-05-20T20:00:40.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:00:40.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:40.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:40.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:40.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:40.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:39 smithi154 ceph-mon[36314]: Deploying daemon mds.f on smithi154 2024-05-20T20:00:40.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] up:boot 2024-05-20T20:00:40.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: fsmap 5 up:standby 2024-05-20T20:00:40.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:00:40.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:40.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:40.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:40.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:40.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:39 smithi033 ceph-mon[36179]: Deploying daemon mds.f on smithi154 2024-05-20T20:00:41.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:40 smithi028 ceph-mon[31178]: pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:41.067 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:40 smithi154 ceph-mon[36314]: pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:40 smithi033 ceph-mon[36179]: pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:41.375 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 podman[66837]: 2024-05-20 20:00:41.295667911 +0000 UTC m=+0.125784255 container create d4e4b2e108eccf63c70eb9311a57344546a741a8d98ea9fe5350cfbcd47db071 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:00:41.668 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 podman[66837]: 2024-05-20 20:00:41.56617581 +0000 UTC m=+0.396292163 container init d4e4b2e108eccf63c70eb9311a57344546a741a8d98ea9fe5350cfbcd47db071 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:00:41.668 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 podman[66837]: 2024-05-20 20:00:41.568854984 +0000 UTC m=+0.398971339 container start d4e4b2e108eccf63c70eb9311a57344546a741a8d98ea9fe5350cfbcd47db071 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240514, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T20:00:41.668 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 ceph-mds[66856]: -- 172.21.15.154:0/991609443 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x562bc62f64e0 con 0x562bc62ccc00 2024-05-20T20:00:41.668 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 ceph-mds[66856]: -- 172.21.15.154:0/991609443 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562bc62f9a40 con 0x562bc62ccc00 2024-05-20T20:00:41.669 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f[66852]: starting mds.f at 2024-05-20T20:00:41.669 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 bash[66837]: d4e4b2e108eccf63c70eb9311a57344546a741a8d98ea9fe5350cfbcd47db071 2024-05-20T20:00:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.760+0000 7f6e3c200700 1 -- 172.21.15.28:0/4101057533 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e341095a0 msgr2=0x7f6e34109980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.760+0000 7f6e3c200700 1 --2- 172.21.15.28:0/4101057533 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e341095a0 0x7f6e34109980 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f6e28009990 tx=0x7f6e28009ca0 comp rx=0 tx=0).stop 2024-05-20T20:00:41.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.760+0000 7f6e3c200700 1 -- 172.21.15.28:0/4101057533 shutdown_connections 2024-05-20T20:00:41.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.760+0000 7f6e3c200700 1 --2- 172.21.15.28:0/4101057533 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6e3406c470 0x7f6e341103c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:41.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.760+0000 7f6e3c200700 1 --2- 172.21.15.28:0/4101057533 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e341095a0 0x7f6e34109980 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:41.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.760+0000 7f6e3c200700 1 --2- 172.21.15.28:0/4101057533 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6e3406bab0 0x7f6e3406bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:41.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.761+0000 7f6e3c200700 1 -- 172.21.15.28:0/4101057533 >> 172.21.15.28:0/4101057533 conn(0x7f6e340793b0 msgr2=0x7f6e340797c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:41.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.761+0000 7f6e3c200700 1 -- 172.21.15.28:0/4101057533 shutdown_connections 2024-05-20T20:00:41.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.761+0000 7f6e3c200700 1 -- 172.21.15.28:0/4101057533 wait complete. 2024-05-20T20:00:41.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.762+0000 7f6e3c200700 1 Processor -- start 2024-05-20T20:00:41.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.763+0000 7f6e3c200700 1 -- start start 2024-05-20T20:00:41.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.763+0000 7f6e3c200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e3406bab0 0x7f6e3419be70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:41.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.763+0000 7f6e3c200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6e3406c470 0x7f6e3419c3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:41.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.763+0000 7f6e3c200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6e341095a0 0x7f6e341a0740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:41.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.763+0000 7f6e3c200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6e34113700 con 0x7f6e3406c470 2024-05-20T20:00:41.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.763+0000 7f6e3c200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6e34113580 con 0x7f6e341095a0 2024-05-20T20:00:41.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.763+0000 7f6e3c200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6e34113880 con 0x7f6e3406bab0 2024-05-20T20:00:41.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e3406bab0 0x7f6e3419be70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e3406bab0 0x7f6e3419be70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:40362/0 (socket says 172.21.15.28:40362) 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 -- 172.21.15.28:0/635691993 learned_addr learned my addr 172.21.15.28:0/635691993 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e3a200700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6e341095a0 0x7f6e341a0740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e38e00700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6e3406c470 0x7f6e3419c3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 -- 172.21.15.28:0/635691993 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6e341095a0 msgr2=0x7f6e341a0740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6e341095a0 0x7f6e341a0740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 -- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6e3406c470 msgr2=0x7f6e3419c3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6e3406c470 0x7f6e3419c3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.764+0000 7f6e39800700 1 -- 172.21.15.28:0/635691993 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6e341a0e80 con 0x7f6e3406bab0 2024-05-20T20:00:41.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.765+0000 7f6e39800700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e3406bab0 0x7f6e3419be70 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f6e28000c00 tx=0x7f6e280057a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:41.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.767+0000 7f6e32000700 1 -- 172.21.15.28:0/635691993 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6e28027070 con 0x7f6e3406bab0 2024-05-20T20:00:41.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.767+0000 7f6e32000700 1 -- 172.21.15.28:0/635691993 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6e28003f20 con 0x7f6e3406bab0 2024-05-20T20:00:41.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.767+0000 7f6e32000700 1 -- 172.21.15.28:0/635691993 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6e28022bd0 con 0x7f6e3406bab0 2024-05-20T20:00:41.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.767+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6e341a1110 con 0x7f6e3406bab0 2024-05-20T20:00:41.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.768+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6e341a89f0 con 0x7f6e3406bab0 2024-05-20T20:00:41.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.769+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6e34104c60 con 0x7f6e3406bab0 2024-05-20T20:00:41.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.771+0000 7f6e32000700 1 -- 172.21.15.28:0/635691993 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f6e2802a070 con 0x7f6e3406bab0 2024-05-20T20:00:41.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.772+0000 7f6e32000700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6e1c0b2900 0x7f6e1c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:41.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.772+0000 7f6e32000700 1 -- 172.21.15.28:0/635691993 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f6e280e2ca0 con 0x7f6e3406bab0 2024-05-20T20:00:41.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.773+0000 7f6e38e00700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6e1c0b2900 0x7f6e1c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:41.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.773+0000 7f6e38e00700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6e1c0b2900 0x7f6e1c0b4dc0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f6e20005950 tx=0x7f6e200058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:41.777+0000 7f6e32000700 1 -- 172.21.15.28:0/635691993 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f6e28013070 con 0x7f6e3406bab0 2024-05-20T20:00:42.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.068+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f6e3419d1d0 con 0x7f6e3406bab0 2024-05-20T20:00:42.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.069+0000 7f6e32000700 1 -- 172.21.15.28:0/635691993 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) ==== 93+0+5515 (secure 0 0 0) 0x7f6e280aae50 con 0x7f6e3406bab0 2024-05-20T20:00:42.071 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:42.071 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":7,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24442,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24455,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24463,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24482,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6}],"filesystems":[]} 2024-05-20T20:00:42.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6e1c0b2900 msgr2=0x7f6e1c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:42.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6e1c0b2900 0x7f6e1c0b4dc0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f6e20005950 tx=0x7f6e200058e0 comp rx=0 tx=0).stop 2024-05-20T20:00:42.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e3406bab0 msgr2=0x7f6e3419be70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:42.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e3406bab0 0x7f6e3419be70 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f6e28000c00 tx=0x7f6e280057a0 comp rx=0 tx=0).stop 2024-05-20T20:00:42.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 shutdown_connections 2024-05-20T20:00:42.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f6e1c0b2900 0x7f6e1c0b4dc0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:42.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6e341095a0 0x7f6e341a0740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:42.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6e3406bab0 0x7f6e3419be70 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:42.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 --2- 172.21.15.28:0/635691993 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6e3406c470 0x7f6e3419c3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:42.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.072+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 >> 172.21.15.28:0/635691993 conn(0x7f6e340793b0 msgr2=0x7f6e3410e710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:42.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.073+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 shutdown_connections 2024-05-20T20:00:42.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:42.073+0000 7f6e3c200700 1 -- 172.21.15.28:0/635691993 wait complete. 2024-05-20T20:00:42.075 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 7 2024-05-20T20:00:42.125 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:00:41 smithi154 systemd[1]: Started Ceph mds.f for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: Deploying daemon mds.c on smithi154 2024-05-20T20:00:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6834/3965936826,v1:172.21.15.154:6835/3965936826] up:boot 2024-05-20T20:00:43.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: fsmap 6 up:standby 2024-05-20T20:00:43.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:00:43.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:42 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/635691993' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:43.108 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:43.108 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.108 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.108 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: Deploying daemon mds.c on smithi154 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6834/3965936826,v1:172.21.15.154:6835/3965936826] up:boot 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: fsmap 6 up:standby 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:00:43.109 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:42 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/635691993' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:43.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: Deploying daemon mds.c on smithi154 2024-05-20T20:00:43.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6834/3965936826,v1:172.21.15.154:6835/3965936826] up:boot 2024-05-20T20:00:43.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: fsmap 6 up:standby 2024-05-20T20:00:43.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:00:43.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:42 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/635691993' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:44.125 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 systemd[1]: Starting Ceph mds.c for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:44.390 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 podman[67143]: 2024-05-20 20:00:44.152279989 +0000 UTC m=+0.017413295 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:44.390 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 podman[67143]: 2024-05-20 20:00:44.262171245 +0000 UTC m=+0.127304551 container create f549f9ff21e38a6b3357d4936649858a7bdaa760402f1fd415a09109f5f402e2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-20T20:00:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mon[36314]: pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 podman[67143]: 2024-05-20 20:00:44.507615605 +0000 UTC m=+0.372748912 container init f549f9ff21e38a6b3357d4936649858a7bdaa760402f1fd415a09109f5f402e2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 podman[67143]: 2024-05-20 20:00:44.510202221 +0000 UTC m=+0.375335528 container start f549f9ff21e38a6b3357d4936649858a7bdaa760402f1fd415a09109f5f402e2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-mds[67162]: -- 172.21.15.154:0/3538980223 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fd6e9bba40 con 0x55fd6e98f000 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c[67158]: starting mds.c at 2024-05-20T20:00:44.876 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 bash[67143]: f549f9ff21e38a6b3357d4936649858a7bdaa760402f1fd415a09109f5f402e2 2024-05-20T20:00:44.877 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:00:44 smithi154 systemd[1]: Started Ceph mds.c for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:44.887 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:00:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:44 smithi028 ceph-mon[31178]: pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:45.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:44 smithi033 ceph-mon[36179]: pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:45.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:45.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:45.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:45.357 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:45 smithi028 ceph-mon[31178]: Deploying daemon mds.h on smithi033 2024-05-20T20:00:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:45 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] up:boot 2024-05-20T20:00:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:45 smithi028 ceph-mon[31178]: fsmap 7 up:standby 2024-05-20T20:00:45.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:45 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:00:46.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:45 smithi154 ceph-mon[36314]: Deploying daemon mds.h on smithi033 2024-05-20T20:00:46.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:45 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] up:boot 2024-05-20T20:00:46.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:45 smithi154 ceph-mon[36314]: fsmap 7 up:standby 2024-05-20T20:00:46.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:45 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:00:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:45 smithi033 ceph-mon[36179]: Deploying daemon mds.h on smithi033 2024-05-20T20:00:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:45 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] up:boot 2024-05-20T20:00:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:45 smithi033 ceph-mon[36179]: fsmap 7 up:standby 2024-05-20T20:00:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:45 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:00:46.922 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:46 smithi154 ceph-mon[36314]: pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:46 smithi028 ceph-mon[31178]: pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:47.124 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:46 smithi033 ceph-mon[36179]: pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:47.124 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:46 smithi033 systemd[1]: Starting Ceph mds.h for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:47.377 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 podman[66280]: 2024-05-20 20:00:47.026610224 +0000 UTC m=+0.017741414 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:47.377 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 podman[66280]: 2024-05-20 20:00:47.161604631 +0000 UTC m=+0.152735878 container create ea89059c62bc0516dd41e833137271943068a4f40f657ed09a5a67e26d2ba7df (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-05-20T20:00:47.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.839+0000 7f95e2400700 1 -- 172.21.15.28:0/678415387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 msgr2=0x7f95dc112100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.839+0000 7f95e2400700 1 --2- 172.21.15.28:0/678415387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 0x7f95dc112100 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f95d4009f90 tx=0x7f95d400b330 comp rx=0 tx=0).stop 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.840+0000 7f95e2400700 1 -- 172.21.15.28:0/678415387 shutdown_connections 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.840+0000 7f95e2400700 1 --2- 172.21.15.28:0/678415387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 0x7f95dc112100 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.840+0000 7f95e2400700 1 --2- 172.21.15.28:0/678415387 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc075860 0x7f95dc075c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.840+0000 7f95e2400700 1 --2- 172.21.15.28:0/678415387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f95dc076180 0x7f95dc10f7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.840+0000 7f95e2400700 1 -- 172.21.15.28:0/678415387 >> 172.21.15.28:0/678415387 conn(0x7f95dc0fef60 msgr2=0x7f95dc101380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.840+0000 7f95e2400700 1 -- 172.21.15.28:0/678415387 shutdown_connections 2024-05-20T20:00:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.840+0000 7f95e2400700 1 -- 172.21.15.28:0/678415387 wait complete. 2024-05-20T20:00:47.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.841+0000 7f95e2400700 1 Processor -- start 2024-05-20T20:00:47.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e2400700 1 -- start start 2024-05-20T20:00:47.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e2400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f95dc075860 0x7f95dc1a1a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:47.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e2400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc076180 0x7f95dc1a1f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e2400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 0x7f95dc19bb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e2400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f95dc115840 con 0x7f95dc075860 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e2400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f95dc1156c0 con 0x7f95dc076180 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e2400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f95dc1159c0 con 0x7f95dc10fd10 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95e1000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f95dc075860 0x7f95dc1a1a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95dbe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc076180 0x7f95dc1a1f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.842+0000 7f95dbe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc076180 0x7f95dc1a1f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53704/0 (socket says 172.21.15.28:53704) 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95dbe00700 1 -- 172.21.15.28:0/1199131465 learned_addr learned my addr 172.21.15.28:0/1199131465 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:47.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95e1a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 0x7f95dc19bb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:47.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95dbe00700 1 -- 172.21.15.28:0/1199131465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 msgr2=0x7f95dc19bb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:47.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95dbe00700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 0x7f95dc19bb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:47.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95dbe00700 1 -- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f95dc075860 msgr2=0x7f95dc1a1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:47.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95dbe00700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f95dc075860 0x7f95dc1a1a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:47.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95dbe00700 1 -- 172.21.15.28:0/1199131465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f95dc19c440 con 0x7f95dc076180 2024-05-20T20:00:47.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.843+0000 7f95dbe00700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc076180 0x7f95dc1a1f80 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f95c800e9c0 tx=0x7f95c800ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:47.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.844+0000 7f95d9600700 1 -- 172.21.15.28:0/1199131465 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f95c8004a30 con 0x7f95dc076180 2024-05-20T20:00:47.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.844+0000 7f95d9600700 1 -- 172.21.15.28:0/1199131465 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f95c800cc70 con 0x7f95dc076180 2024-05-20T20:00:47.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.844+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f95dc19c730 con 0x7f95dc076180 2024-05-20T20:00:47.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.844+0000 7f95d9600700 1 -- 172.21.15.28:0/1199131465 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f95c8005490 con 0x7f95dc076180 2024-05-20T20:00:47.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.844+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f95dc1a89c0 con 0x7f95dc076180 2024-05-20T20:00:47.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.845+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f95dc076fa0 con 0x7f95dc076180 2024-05-20T20:00:47.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.847+0000 7f95d9600700 1 -- 172.21.15.28:0/1199131465 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f95c8004bd0 con 0x7f95dc076180 2024-05-20T20:00:47.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.848+0000 7f95d9600700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f95c40b2900 0x7f95c40b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:47.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.849+0000 7f95d9600700 1 -- 172.21.15.28:0/1199131465 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f95c80e1df0 con 0x7f95dc076180 2024-05-20T20:00:47.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.849+0000 7f95e1000700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f95c40b2900 0x7f95c40b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:47.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.849+0000 7f95e1000700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f95c40b2900 0x7f95c40b4dc0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f95d0009c60 tx=0x7f95d0009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:47.853+0000 7f95d9600700 1 -- 172.21.15.28:0/1199131465 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f95c80a9f10 con 0x7f95dc076180 2024-05-20T20:00:47.877 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 podman[66280]: 2024-05-20 20:00:47.380969828 +0000 UTC m=+0.372101012 container init ea89059c62bc0516dd41e833137271943068a4f40f657ed09a5a67e26d2ba7df (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:00:47.877 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 podman[66280]: 2024-05-20 20:00:47.383600915 +0000 UTC m=+0.374732097 container start ea89059c62bc0516dd41e833137271943068a4f40f657ed09a5a67e26d2ba7df (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2) 2024-05-20T20:00:47.878 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 ceph-mds[66299]: -- 172.21.15.33:0/4154937170 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557877ccfa40 con 0x557877ca2800 2024-05-20T20:00:47.878 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h[66295]: starting mds.h at 2024-05-20T20:00:47.878 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 bash[66280]: ea89059c62bc0516dd41e833137271943068a4f40f657ed09a5a67e26d2ba7df 2024-05-20T20:00:47.878 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:00:47 smithi033 systemd[1]: Started Ceph mds.h for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:48.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.144+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f95dc02e560 con 0x7f95dc076180 2024-05-20T20:00:48.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.145+0000 7f95d9600700 1 -- 172.21.15.28:0/1199131465 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 9 v9) ==== 93+0+7168 (secure 0 0 0) 0x7f95c80a9810 con 0x7f95dc076180 2024-05-20T20:00:48.146 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:48.146 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":9,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24442,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24455,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24463,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24475,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24482,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24500,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9}],"filesystems":[]} 2024-05-20T20:00:48.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f95c40b2900 msgr2=0x7f95c40b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f95c40b2900 0x7f95c40b4dc0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f95d0009c60 tx=0x7f95d0009260 comp rx=0 tx=0).stop 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc076180 msgr2=0x7f95dc1a1f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc076180 0x7f95dc1a1f80 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f95c800e9c0 tx=0x7f95c800ecd0 comp rx=0 tx=0).stop 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 shutdown_connections 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f95c40b2900 0x7f95c40b4dc0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f95dc10fd10 0x7f95dc19bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f95dc075860 0x7f95dc1a1a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 --2- 172.21.15.28:0/1199131465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f95dc076180 0x7f95dc1a1f80 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.148+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 >> 172.21.15.28:0/1199131465 conn(0x7f95dc0fef60 msgr2=0x7f95dc0ffc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.149+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 shutdown_connections 2024-05-20T20:00:48.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:48.149+0000 7f95e2400700 1 -- 172.21.15.28:0/1199131465 wait complete. 2024-05-20T20:00:48.151 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 9 2024-05-20T20:00:48.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:48.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: Deploying daemon mds.i on smithi154 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] up:boot 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: fsmap 8 up:standby 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:00:48.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:48 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1199131465' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: Deploying daemon mds.i on smithi154 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] up:boot 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: fsmap 8 up:standby 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:00:48.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:48 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1199131465' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: Deploying daemon mds.i on smithi154 2024-05-20T20:00:48.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] up:boot 2024-05-20T20:00:48.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: fsmap 8 up:standby 2024-05-20T20:00:48.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:00:48.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:48 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1199131465' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:50.052 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:49 smithi154 systemd[1]: Starting Ceph mds.i for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:50.307 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 podman[67460]: 2024-05-20 20:00:49.954499941 +0000 UTC m=+0.017415758 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:50.307 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 podman[67460]: 2024-05-20 20:00:50.053680006 +0000 UTC m=+0.116595834 container create a1ee9ca81892bccd270aa3ae79969fe8846cc4d3628298b1fba4e93d9289df46 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-05-20T20:00:50.308 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 podman[67460]: 2024-05-20 20:00:50.281842308 +0000 UTC m=+0.344758115 container init a1ee9ca81892bccd270aa3ae79969fe8846cc4d3628298b1fba4e93d9289df46 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-05-20T20:00:50.308 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 podman[67460]: 2024-05-20 20:00:50.284309856 +0000 UTC m=+0.347225671 container start a1ee9ca81892bccd270aa3ae79969fe8846cc4d3628298b1fba4e93d9289df46 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240514) 2024-05-20T20:00:50.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mon[36314]: pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:50.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 ceph-mds[67480]: -- 172.21.15.154:0/3394115761 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56007627ba40 con 0x56007624e800 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i[67476]: starting mds.i at 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 bash[67460]: a1ee9ca81892bccd270aa3ae79969fe8846cc4d3628298b1fba4e93d9289df46 2024-05-20T20:00:50.626 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:00:50 smithi154 systemd[1]: Started Ceph mds.i for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:50.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:50 smithi028 ceph-mon[31178]: pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:50.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:50.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:50.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:50 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:50 smithi033 ceph-mon[36179]: pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:50 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:50 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:50 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:50.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:50 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:50.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:50 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:50.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:50 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:51.202 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:00:51.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:51 smithi154 ceph-mon[36314]: Deploying daemon mds.g on smithi028 2024-05-20T20:00:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:51 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] up:boot 2024-05-20T20:00:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:51 smithi154 ceph-mon[36314]: fsmap 9 up:standby 2024-05-20T20:00:51.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:51 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:00:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:51 smithi033 ceph-mon[36179]: Deploying daemon mds.g on smithi028 2024-05-20T20:00:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:51 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] up:boot 2024-05-20T20:00:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:51 smithi033 ceph-mon[36179]: fsmap 9 up:standby 2024-05-20T20:00:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:51 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:00:51.902 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:51 smithi028 ceph-mon[31178]: Deploying daemon mds.g on smithi028 2024-05-20T20:00:51.902 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:51 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] up:boot 2024-05-20T20:00:51.902 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:51 smithi028 ceph-mon[31178]: fsmap 9 up:standby 2024-05-20T20:00:51.902 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:51 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:00:52.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:52 smithi028 ceph-mon[31178]: pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:52 smithi154 ceph-mon[36314]: pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:52 smithi033 ceph-mon[36179]: pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:53.087 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:00:53.804 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:53 smithi028 systemd[1]: Starting Ceph mds.g for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:54.724 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:54 smithi028 ceph-mon[31178]: pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:54.724 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:54 smithi028 podman[73243]: 2024-05-20 20:00:54.576766268 +0000 UTC m=+0.035476915 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:54.724 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:54 smithi028 podman[73243]: 2024-05-20 20:00:54.722093243 +0000 UTC m=+0.180803916 container create 42c50cb5224609c9b5bce74f9c94e63a837ea4a51398e965e80e8633b101b39d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:00:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:54 smithi154 ceph-mon[36314]: pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:54 smithi033 ceph-mon[36179]: pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:55.304 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:54 smithi028 podman[73243]: 2024-05-20 20:00:54.976103097 +0000 UTC m=+0.434813788 container init 42c50cb5224609c9b5bce74f9c94e63a837ea4a51398e965e80e8633b101b39d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T20:00:55.304 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:54 smithi028 podman[73243]: 2024-05-20 20:00:54.98629164 +0000 UTC m=+0.445002305 container start 42c50cb5224609c9b5bce74f9c94e63a837ea4a51398e965e80e8633b101b39d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-05-20T20:00:55.304 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:55 smithi028 ceph-mds[73281]: -- 172.21.15.28:0/774178021 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558042ce1a40 con 0x558042cb4800 2024-05-20T20:00:55.305 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:55 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g[73275]: starting mds.g at 2024-05-20T20:00:55.305 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:55 smithi028 bash[73243]: 42c50cb5224609c9b5bce74f9c94e63a837ea4a51398e965e80e8633b101b39d 2024-05-20T20:00:55.305 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:00:55 smithi028 systemd[1]: Started Ceph mds.g for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:00:55.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 -- 172.21.15.28:0/2234743699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 msgr2=0x7f21bc103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:55.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 --2- 172.21.15.28:0/2234743699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc103bd0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f21b00099c0 tx=0x7f21b0009cd0 comp rx=0 tx=0).stop 2024-05-20T20:00:55.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 -- 172.21.15.28:0/2234743699 shutdown_connections 2024-05-20T20:00:55.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 --2- 172.21.15.28:0/2234743699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f21bc1088f0 0x7f21bc110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:55.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 --2- 172.21.15.28:0/2234743699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc103bd0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:55.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 --2- 172.21.15.28:0/2234743699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f21bc1041a0 0x7f21bc1081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:55.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 -- 172.21.15.28:0/2234743699 >> 172.21.15.28:0/2234743699 conn(0x7f21bc07a6a0 msgr2=0x7f21bc07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:55.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 -- 172.21.15.28:0/2234743699 shutdown_connections 2024-05-20T20:00:55.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.703+0000 7f21c3600700 1 -- 172.21.15.28:0/2234743699 wait complete. 2024-05-20T20:00:55.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.704+0000 7f21c3600700 1 Processor -- start 2024-05-20T20:00:55.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.704+0000 7f21c3600700 1 -- start start 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c3600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c3600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f21bc1041a0 0x7f21bc075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c3600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f21bc1088f0 0x7f21bc0762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c3600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f21bc1136e0 con 0x7f21bc1041a0 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c3600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f21bc113560 con 0x7f21bc1037f0 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c3600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f21bc113860 con 0x7f21bc1088f0 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c0c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c0c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53748/0 (socket says 172.21.15.28:53748) 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c0c00700 1 -- 172.21.15.28:0/2994787945 learned_addr learned my addr 172.21.15.28:0/2994787945 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21bbe00700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f21bc1041a0 0x7f21bc075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.705+0000 7f21c1600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f21bc1088f0 0x7f21bc0762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:55.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c0c00700 1 -- 172.21.15.28:0/2994787945 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f21bc1088f0 msgr2=0x7f21bc0762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c0c00700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f21bc1088f0 0x7f21bc0762c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c0c00700 1 -- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f21bc1041a0 msgr2=0x7f21bc075d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c0c00700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f21bc1041a0 0x7f21bc075d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c0c00700 1 -- 172.21.15.28:0/2994787945 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f21bc1a8070 con 0x7f21bc1037f0 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21bbe00700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f21bc1041a0 0x7f21bc075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c1600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f21bc1088f0 0x7f21bc0762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c0c00700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc075840 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f21b00098e0 tx=0x7f21b0005e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21b9600700 1 -- 172.21.15.28:0/2994787945 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21b0027070 con 0x7f21bc1037f0 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21b9600700 1 -- 172.21.15.28:0/2994787945 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f21b0003fa0 con 0x7f21bc1037f0 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21b9600700 1 -- 172.21.15.28:0/2994787945 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21b0022d40 con 0x7f21bc1037f0 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.706+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f21bc1a8300 con 0x7f21bc1037f0 2024-05-20T20:00:55.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.707+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f21bc1a8700 con 0x7f21bc1037f0 2024-05-20T20:00:55.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.708+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f21bc10bb30 con 0x7f21bc1037f0 2024-05-20T20:00:55.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.710+0000 7f21b9600700 1 -- 172.21.15.28:0/2994787945 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f21b0022470 con 0x7f21bc1037f0 2024-05-20T20:00:55.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.712+0000 7f21b9600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21a40b2900 0x7f21a40b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:00:55.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.712+0000 7f21b9600700 1 -- 172.21.15.28:0/2994787945 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f21b00e2ea0 con 0x7f21bc1037f0 2024-05-20T20:00:55.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.712+0000 7f21bbe00700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21a40b2900 0x7f21a40b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:00:55.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.713+0000 7f21bbe00700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21a40b2900 0x7f21a40b4dc0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f21bc103110 tx=0x7f21a800b5f0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:00:55.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:55.716+0000 7f21b9600700 1 -- 172.21.15.28:0/2994787945 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f21b005edf0 con 0x7f21bc1037f0 2024-05-20T20:00:56.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.008+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f21bc10b510 con 0x7f21bc1037f0 2024-05-20T20:00:56.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.009+0000 7f21b9600700 1 -- 172.21.15.28:0/2994787945 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 11 v11) ==== 94+0+8821 (secure 0 0 0) 0x7f21b0013070 con 0x7f21bc1037f0 2024-05-20T20:00:56.010 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:00:56.010 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":11,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24442,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24455,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24463,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24475,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24482,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24493,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10},{"gid":24500,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9}],"filesystems":[]} 2024-05-20T20:00:56.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21a40b2900 msgr2=0x7f21a40b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:56.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21a40b2900 0x7f21a40b4dc0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f21bc103110 tx=0x7f21a800b5f0 comp rx=0 tx=0).stop 2024-05-20T20:00:56.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 msgr2=0x7f21bc075840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:00:56.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc075840 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f21b00098e0 tx=0x7f21b0005e80 comp rx=0 tx=0).stop 2024-05-20T20:00:56.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 shutdown_connections 2024-05-20T20:00:56.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21a40b2900 0x7f21a40b4dc0 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:56.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f21bc1088f0 0x7f21bc0762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:56.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f21bc1037f0 0x7f21bc075840 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:56.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 --2- 172.21.15.28:0/2994787945 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f21bc1041a0 0x7f21bc075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:00:56.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 >> 172.21.15.28:0/2994787945 conn(0x7f21bc07a6a0 msgr2=0x7f21bc101860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:00:56.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.013+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 shutdown_connections 2024-05-20T20:00:56.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:00:56.014+0000 7f21c3600700 1 -- 172.21.15.28:0/2994787945 wait complete. 2024-05-20T20:00:56.015 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 11 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: Deploying daemon mds.a on smithi028 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:56.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.28:6838/892373280,v1:172.21.15.28:6839/892373280] up:boot 2024-05-20T20:00:56.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: fsmap 10 up:standby 2024-05-20T20:00:56.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-05-20T20:00:56.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:56 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2994787945' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: Deploying daemon mds.a on smithi028 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.28:6838/892373280,v1:172.21.15.28:6839/892373280] up:boot 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: fsmap 10 up:standby 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-05-20T20:00:56.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:56 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2994787945' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: Deploying daemon mds.a on smithi028 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.28:6838/892373280,v1:172.21.15.28:6839/892373280] up:boot 2024-05-20T20:00:56.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: fsmap 10 up:standby 2024-05-20T20:00:56.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-05-20T20:00:56.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:56 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2994787945' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:00:58.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:00:58 smithi028 ceph-mon[31178]: pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:00:58 smithi154 ceph-mon[36314]: pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:00:58 smithi033 ceph-mon[36179]: pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:00:58.953 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:00:59.554 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:00:59 smithi028 systemd[1]: Starting Ceph mds.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:00:59.933 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:00:59 smithi028 podman[73639]: 2024-05-20 20:00:59.523940701 +0000 UTC m=+0.035100251 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:00:59.933 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:00:59 smithi028 podman[73639]: 2024-05-20 20:00:59.644061387 +0000 UTC m=+0.155220974 container create ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:00:59.933 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:00:59 smithi028 podman[73639]: 2024-05-20 20:00:59.931793897 +0000 UTC m=+0.442953490 container init ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-20T20:01:00.053 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:00.211 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:00:59 smithi028 podman[73639]: 2024-05-20 20:00:59.94262173 +0000 UTC m=+0.453781314 container start ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T20:01:00.211 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:00:59 smithi028 ceph-mds[73678]: -- 172.21.15.28:0/1929327426 <== mon.0 v2:172.21.15.28:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557632fd3a40 con 0x557632fa6800 2024-05-20T20:01:00.211 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:00:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a[73673]: starting mds.a at 2024-05-20T20:01:00.211 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:01:00 smithi028 bash[73639]: ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 2024-05-20T20:01:00.211 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:01:00 smithi028 systemd[1]: Started Ceph mds.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:01:00.464 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:00 smithi028 ceph-mon[31178]: pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:00.464 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.464 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.464 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.464 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:01:00.464 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:01:00.464 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:00 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:00.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:00 smithi154 ceph-mon[36314]: pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:00.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:01:00.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:01:00.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:00 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:00.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:00 smithi033 ceph-mon[36179]: pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:00.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:00.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T20:01:00.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T20:01:00.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:00 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:01.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:01 smithi028 ceph-mon[31178]: Deploying daemon mds.b on smithi033 2024-05-20T20:01:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:01 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.28:6840/1736910968,v1:172.21.15.28:6841/1736910968] up:boot 2024-05-20T20:01:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:01 smithi028 ceph-mon[31178]: fsmap 11 up:standby 2024-05-20T20:01:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:01 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:01:01.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:01 smithi154 ceph-mon[36314]: Deploying daemon mds.b on smithi033 2024-05-20T20:01:01.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:01 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.28:6840/1736910968,v1:172.21.15.28:6841/1736910968] up:boot 2024-05-20T20:01:01.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:01 smithi154 ceph-mon[36314]: fsmap 11 up:standby 2024-05-20T20:01:01.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:01 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:01:01.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:01 smithi033 ceph-mon[36179]: Deploying daemon mds.b on smithi033 2024-05-20T20:01:01.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:01 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.28:6840/1736910968,v1:172.21.15.28:6841/1736910968] up:boot 2024-05-20T20:01:01.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:01 smithi033 ceph-mon[36179]: fsmap 11 up:standby 2024-05-20T20:01:01.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:01 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:01:02.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:02 smithi033 ceph-mon[36179]: pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:02.628 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:02 smithi033 systemd[1]: Starting Ceph mds.b for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 -- 172.21.15.28:0/47531832 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe1010ff70 msgr2=0x7efe10112360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 --2- 172.21.15.28:0/47531832 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe1010ff70 0x7efe10112360 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7efe0800b0a0 tx=0x7efe0800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 -- 172.21.15.28:0/47531832 shutdown_connections 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 --2- 172.21.15.28:0/47531832 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe1010ff70 0x7efe10112360 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 --2- 172.21.15.28:0/47531832 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efe10075770 0x7efe10075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 --2- 172.21.15.28:0/47531832 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efe10076090 0x7efe1010fa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 -- 172.21.15.28:0/47531832 >> 172.21.15.28:0/47531832 conn(0x7efe1007a6f0 msgr2=0x7efe1007ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:02.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 -- 172.21.15.28:0/47531832 shutdown_connections 2024-05-20T20:01:02.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.687+0000 7efe17200700 1 -- 172.21.15.28:0/47531832 wait complete. 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.688+0000 7efe17200700 1 Processor -- start 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe17200700 1 -- start start 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe17200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efe10075770 0x7efe1019beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe17200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe10076090 0x7efe1019c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe17200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efe1010ff70 0x7efe101a0730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe17200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7efe10115960 con 0x7efe10076090 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe17200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7efe101157e0 con 0x7efe1010ff70 2024-05-20T20:01:02.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe17200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7efe10115ae0 con 0x7efe10075770 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe0f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe10076090 0x7efe1019c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.689+0000 7efe0fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efe10075770 0x7efe1019beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe10076090 0x7efe1019c3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53976/0 (socket says 172.21.15.28:53976) 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0f400700 1 -- 172.21.15.28:0/2931396829 learned_addr learned my addr 172.21.15.28:0/2931396829 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe15200700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efe1010ff70 0x7efe101a0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0fe00700 1 -- 172.21.15.28:0/2931396829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efe1010ff70 msgr2=0x7efe101a0730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0fe00700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efe1010ff70 0x7efe101a0730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0fe00700 1 -- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe10076090 msgr2=0x7efe1019c3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0fe00700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe10076090 0x7efe1019c3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0fe00700 1 -- 172.21.15.28:0/2931396829 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efe101a0e70 con 0x7efe10075770 2024-05-20T20:01:02.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.690+0000 7efe0fe00700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efe10075770 0x7efe1019beb0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7efe0000e9f0 tx=0x7efe0000ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:02.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.691+0000 7efe0cc00700 1 -- 172.21.15.28:0/2931396829 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe0000c9c0 con 0x7efe10075770 2024-05-20T20:01:02.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.691+0000 7efe0cc00700 1 -- 172.21.15.28:0/2931396829 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efe0000cb60 con 0x7efe10075770 2024-05-20T20:01:02.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.691+0000 7efe0cc00700 1 -- 172.21.15.28:0/2931396829 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efe00018680 con 0x7efe10075770 2024-05-20T20:01:02.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.691+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efe101a1160 con 0x7efe10075770 2024-05-20T20:01:02.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.691+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efe101a89e0 con 0x7efe10075770 2024-05-20T20:01:02.693 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:02 smithi028 ceph-mon[31178]: pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:02.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.693+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efe1006a230 con 0x7efe10075770 2024-05-20T20:01:02.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.696+0000 7efe0cc00700 1 -- 172.21.15.28:0/2931396829 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7efe0004aab0 con 0x7efe10075770 2024-05-20T20:01:02.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.698+0000 7efe0cc00700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efdf40b2900 0x7efdf40b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:02.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.698+0000 7efe0cc00700 1 -- 172.21.15.28:0/2931396829 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7efe000e2060 con 0x7efe10075770 2024-05-20T20:01:02.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.698+0000 7efe0f400700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efdf40b2900 0x7efdf40b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:02.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.699+0000 7efe0cc00700 1 -- 172.21.15.28:0/2931396829 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7efe000aa180 con 0x7efe10075770 2024-05-20T20:01:02.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.699+0000 7efe0f400700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efdf40b2900 0x7efdf40b4dc0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7efe10111c30 tx=0x7efdfc009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:02 smithi154 ceph-mon[36314]: pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:02.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.983+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7efe1019d1c0 con 0x7efe10075770 2024-05-20T20:01:02.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.984+0000 7efe0cc00700 1 -- 172.21.15.28:0/2931396829 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) ==== 94+0+9647 (secure 0 0 0) 0x7efe000b2020 con 0x7efe10075770 2024-05-20T20:01:02.984 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:01:02.984 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14400,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24442,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24455,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24463,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24475,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24482,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24493,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10},{"gid":24500,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9}],"filesystems":[]} 2024-05-20T20:01:02.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efdf40b2900 msgr2=0x7efdf40b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:02.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efdf40b2900 0x7efdf40b4dc0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7efe10111c30 tx=0x7efdfc009260 comp rx=0 tx=0).stop 2024-05-20T20:01:02.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efe10075770 msgr2=0x7efe1019beb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efe10075770 0x7efe1019beb0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7efe0000e9f0 tx=0x7efe0000ed00 comp rx=0 tx=0).stop 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 shutdown_connections 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efdf40b2900 0x7efdf40b4dc0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efe1010ff70 0x7efe101a0730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efe10075770 0x7efe1019beb0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 --2- 172.21.15.28:0/2931396829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efe10076090 0x7efe1019c3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 >> 172.21.15.28:0/2931396829 conn(0x7efe1007a6f0 msgr2=0x7efe10100820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 shutdown_connections 2024-05-20T20:01:02.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:02.987+0000 7efe17200700 1 -- 172.21.15.28:0/2931396829 wait complete. 2024-05-20T20:01:02.989 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 12 2024-05-20T20:01:02.994 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:02 smithi033 podman[66608]: 2024-05-20 20:01:02.631767464 +0000 UTC m=+0.017963564 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:01:02.995 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:02 smithi033 podman[66608]: 2024-05-20 20:01:02.767349249 +0000 UTC m=+0.153545348 container create ee0c30e87bae3688c5f234237209a238b584b7a3b34c54906fae669efdc97cdb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-05-20T20:01:02.995 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:02 smithi033 podman[66608]: 2024-05-20 20:01:02.995052197 +0000 UTC m=+0.381248299 container init ee0c30e87bae3688c5f234237209a238b584b7a3b34c54906fae669efdc97cdb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:01:03.377 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:02 smithi033 podman[66608]: 2024-05-20 20:01:02.997902549 +0000 UTC m=+0.384098658 container start ee0c30e87bae3688c5f234237209a238b584b7a3b34c54906fae669efdc97cdb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-05-20T20:01:03.377 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mds[66627]: -- 172.21.15.33:0/648166664 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560e2206fa40 con 0x560e22042c00 2024-05-20T20:01:03.377 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: starting mds.b at 2024-05-20T20:01:03.377 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:03 smithi033 bash[66608]: ee0c30e87bae3688c5f234237209a238b584b7a3b34c54906fae669efdc97cdb 2024-05-20T20:01:03.377 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:03 smithi033 systemd[1]: Started Ceph mds.b for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:01:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:03 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2931396829' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:01:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:03 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2931396829' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:01:03.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2931396829' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:01:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:04.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:04 smithi154 ceph-mon[36314]: pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:04.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:04 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] up:boot 2024-05-20T20:01:04.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:04 smithi154 ceph-mon[36314]: fsmap 12 up:standby 2024-05-20T20:01:04.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:01:04.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:01:04.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:01:04.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:04 smithi033 ceph-mon[36179]: pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:04.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:04 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] up:boot 2024-05-20T20:01:04.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:04 smithi033 ceph-mon[36179]: fsmap 12 up:standby 2024-05-20T20:01:04.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:01:04.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:01:04.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:01:04.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:04 smithi028 ceph-mon[31178]: pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:04.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:04 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] up:boot 2024-05-20T20:01:04.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:04 smithi028 ceph-mon[31178]: fsmap 12 up:standby 2024-05-20T20:01:04.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:01:04.802 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:01:04.803 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:01:05.914 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:01:06.377 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:06.524 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.524 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.524 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.524 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:06 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.524 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:06 smithi028 ceph-mon[31178]: pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:06 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:06 smithi154 ceph-mon[36314]: pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:06.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:06 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:06.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:06 smithi033 ceph-mon[36179]: pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:07.597 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.597 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.597 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:07.597 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:01:07.597 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.598 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:07 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:01:07.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:07.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:01:07.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:07 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:01:07.673 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.673 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.674 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:07.674 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:01:07.674 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:07.674 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:07 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:01:08.384 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:08 smithi028 ceph-mon[31178]: pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:08.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:08 smithi154 ceph-mon[36314]: pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:08.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:08 smithi033 ceph-mon[36179]: pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:08.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.698+0000 7f627be00700 1 -- 172.21.15.28:0/4277984681 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 msgr2=0x7f627c113f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:08.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.698+0000 7f627be00700 1 --2- 172.21.15.28:0/4277984681 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c113f40 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f62640098f0 tx=0x7f6264009c00 comp rx=0 tx=0).stop 2024-05-20T20:01:08.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.698+0000 7f627be00700 1 -- 172.21.15.28:0/4277984681 shutdown_connections 2024-05-20T20:01:08.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.698+0000 7f627be00700 1 --2- 172.21.15.28:0/4277984681 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f627c114480 0x7f627c116870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:08.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.698+0000 7f627be00700 1 --2- 172.21.15.28:0/4277984681 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f627c0758e0 0x7f627c075cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:08.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.698+0000 7f627be00700 1 --2- 172.21.15.28:0/4277984681 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c113f40 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:08.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.699+0000 7f627be00700 1 -- 172.21.15.28:0/4277984681 >> 172.21.15.28:0/4277984681 conn(0x7f627c0fefc0 msgr2=0x7f627c1013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:08.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.699+0000 7f627be00700 1 -- 172.21.15.28:0/4277984681 shutdown_connections 2024-05-20T20:01:08.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.699+0000 7f627be00700 1 -- 172.21.15.28:0/4277984681 wait complete. 2024-05-20T20:01:08.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 Processor -- start 2024-05-20T20:01:08.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 -- start start 2024-05-20T20:01:08.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f627c0758e0 0x7f627c1a0210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:08.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c1a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:08.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f627c114480 0x7f627c1a4a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:08.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f627c119cc0 con 0x7f627c0758e0 2024-05-20T20:01:08.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f627c119b40 con 0x7f627c076200 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627be00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f627c119e40 con 0x7f627c114480 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.700+0000 7f627aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f627c0758e0 0x7f627c1a0210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c1a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f627b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f627c114480 0x7f627c1a4a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c1a0750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51880/0 (socket says 172.21.15.28:51880) 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 -- 172.21.15.28:0/2502156838 learned_addr learned my addr 172.21.15.28:0/2502156838 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 -- 172.21.15.28:0/2502156838 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f627c114480 msgr2=0x7f627c1a4a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f627c114480 0x7f627c1a4a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 -- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f627c0758e0 msgr2=0x7f627c1a0210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f627c0758e0 0x7f627c1a0210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f6272000700 1 -- 172.21.15.28:0/2502156838 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f627c1a51d0 con 0x7f627c076200 2024-05-20T20:01:08.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f627b400700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f627c114480 0x7f627c1a4a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.701+0000 7f627aa00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f627c0758e0 0x7f627c1a0210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.702+0000 7f6272000700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c1a0750 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6264009230 tx=0x7f626400fa70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.702+0000 7f6273e00700 1 -- 172.21.15.28:0/2502156838 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6264005140 con 0x7f627c076200 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.702+0000 7f6273e00700 1 -- 172.21.15.28:0/2502156838 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f626400b560 con 0x7f627c076200 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.702+0000 7f6273e00700 1 -- 172.21.15.28:0/2502156838 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6264003970 con 0x7f627c076200 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.702+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f627c1a5460 con 0x7f627c076200 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.702+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f627c1acd40 con 0x7f627c076200 2024-05-20T20:01:08.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.703+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f627c077020 con 0x7f627c076200 2024-05-20T20:01:08.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.705+0000 7f6273e00700 1 -- 172.21.15.28:0/2502156838 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f626400ba50 con 0x7f627c076200 2024-05-20T20:01:08.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.706+0000 7f6273e00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f62600b2900 0x7f62600b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:08.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.706+0000 7f6273e00700 1 -- 172.21.15.28:0/2502156838 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f62640e2ce0 con 0x7f627c076200 2024-05-20T20:01:08.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.707+0000 7f627aa00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f62600b2900 0x7f62600b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:08.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.707+0000 7f627aa00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f62600b2900 0x7f62600b4dc0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f626c009480 tx=0x7f626c00b040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:08.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:08.711+0000 7f6273e00700 1 -- 172.21.15.28:0/2502156838 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f6264013030 con 0x7f627c076200 2024-05-20T20:01:09.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.006+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f627c1a1520 con 0x7f627c076200 2024-05-20T20:01:09.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.007+0000 7f6273e00700 1 -- 172.21.15.28:0/2502156838 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 13 v13) ==== 94+0+10473 (secure 0 0 0) 0x7f62640aae90 con 0x7f627c076200 2024-05-20T20:01:09.007 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:01:09.008 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":13,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11},{"gid":14400,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":12},{"gid":24427,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":2},{"gid":24442,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":4},{"gid":24455,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":3},{"gid":24463,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":7},{"gid":24475,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":8},{"gid":24482,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":6},{"gid":24493,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":10},{"gid":24500,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":9},{"gid":24542,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":13}],"filesystems":[]} 2024-05-20T20:01:09.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f62600b2900 msgr2=0x7f62600b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:09.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f62600b2900 0x7f62600b4dc0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f626c009480 tx=0x7f626c00b040 comp rx=0 tx=0).stop 2024-05-20T20:01:09.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 msgr2=0x7f627c1a0750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:09.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c1a0750 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f6264009230 tx=0x7f626400fa70 comp rx=0 tx=0).stop 2024-05-20T20:01:09.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 shutdown_connections 2024-05-20T20:01:09.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f62600b2900 0x7f62600b4dc0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:09.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f627c114480 0x7f627c1a4a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:09.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f627c0758e0 0x7f627c1a0210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:09.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 --2- 172.21.15.28:0/2502156838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f627c076200 0x7f627c1a0750 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:09.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 >> 172.21.15.28:0/2502156838 conn(0x7f627c0fefc0 msgr2=0x7f627c1008c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:09.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 shutdown_connections 2024-05-20T20:01:09.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:09.010+0000 7f627be00700 1 -- 172.21.15.28:0/2502156838 wait complete. 2024-05-20T20:01:09.011 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 13 2024-05-20T20:01:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:09 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:09 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:09 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:09 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:09 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2502156838' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:01:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:09 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:09 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:09 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:09 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:09 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2502156838' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:01:09.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:09 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:09 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:09 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:09 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:09.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:09 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2502156838' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:01:10.334 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph --format=json-pretty osd lspools 2024-05-20T20:01:10.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:10 smithi028 ceph-mon[31178]: pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:10.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:10 smithi154 ceph-mon[36314]: pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:10.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:10 smithi033 ceph-mon[36179]: pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:10.801 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:12.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:12 smithi154 ceph-mon[36314]: pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:12.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:12 smithi033 ceph-mon[36179]: pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:12.644 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:12 smithi028 ceph-mon[31178]: pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:13.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 -- 172.21.15.28:0/497177785 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28105760 msgr2=0x7f9d28105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:13.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 --2- 172.21.15.28:0/497177785 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28105760 0x7f9d28105be0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f9d14009990 tx=0x7f9d14009ca0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 -- 172.21.15.28:0/497177785 shutdown_connections 2024-05-20T20:01:13.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 --2- 172.21.15.28:0/497177785 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d28106120 0x7f9d28112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 --2- 172.21.15.28:0/497177785 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9d2810b760 0x7f9d2810bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 --2- 172.21.15.28:0/497177785 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28105760 0x7f9d28105be0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 -- 172.21.15.28:0/497177785 >> 172.21.15.28:0/497177785 conn(0x7f9d28101250 msgr2=0x7f9d28103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:13.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 -- 172.21.15.28:0/497177785 shutdown_connections 2024-05-20T20:01:13.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.179+0000 7f9d2e800700 1 -- 172.21.15.28:0/497177785 wait complete. 2024-05-20T20:01:13.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 Processor -- start 2024-05-20T20:01:13.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 -- start start 2024-05-20T20:01:13.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9d28105760 0x7f9d2819be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:13.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28106120 0x7f9d2819c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:13.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d2810b760 0x7f9d281a0700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9d281159a0 con 0x7f9d2810b760 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9d28115820 con 0x7f9d28106120 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.181+0000 7f9d2e800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9d28115b20 con 0x7f9d28105760 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d2810b760 0x7f9d281a0700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d2810b760 0x7f9d281a0700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50568/0 (socket says 172.21.15.28:50568) 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 -- 172.21.15.28:0/477270275 learned_addr learned my addr 172.21.15.28:0/477270275 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d26a00700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28106120 0x7f9d2819c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27400700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9d28105760 0x7f9d2819be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 -- 172.21.15.28:0/477270275 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9d28105760 msgr2=0x7f9d2819be30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9d28105760 0x7f9d2819be30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 -- 172.21.15.28:0/477270275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28106120 msgr2=0x7f9d2819c370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28106120 0x7f9d2819c370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.182+0000 7f9d27e00700 1 -- 172.21.15.28:0/477270275 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9d281a0e40 con 0x7f9d2810b760 2024-05-20T20:01:13.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.183+0000 7f9d27e00700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d2810b760 0x7f9d281a0700 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f9d2000c370 tx=0x7f9d2000c680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:13.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.183+0000 7f9d1fe00700 1 -- 172.21.15.28:0/477270275 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d200092e0 con 0x7f9d2810b760 2024-05-20T20:01:13.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.183+0000 7f9d1fe00700 1 -- 172.21.15.28:0/477270275 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9d20003a00 con 0x7f9d2810b760 2024-05-20T20:01:13.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.183+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9d281a10d0 con 0x7f9d2810b760 2024-05-20T20:01:13.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.183+0000 7f9d1fe00700 1 -- 172.21.15.28:0/477270275 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9d200046f0 con 0x7f9d2810b760 2024-05-20T20:01:13.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.183+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9d281a89b0 con 0x7f9d2810b760 2024-05-20T20:01:13.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.184+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9d2810dcf0 con 0x7f9d2810b760 2024-05-20T20:01:13.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.187+0000 7f9d1fe00700 1 -- 172.21.15.28:0/477270275 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f9d2001a020 con 0x7f9d2810b760 2024-05-20T20:01:13.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.189+0000 7f9d1fe00700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9d0c0b2880 0x7f9d0c0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:13.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.189+0000 7f9d1fe00700 1 -- 172.21.15.28:0/477270275 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7f9d200e1cd0 con 0x7f9d2810b760 2024-05-20T20:01:13.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.189+0000 7f9d27400700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9d0c0b2880 0x7f9d0c0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:13.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.190+0000 7f9d27400700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9d0c0b2880 0x7f9d0c0b4d40 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f9d18009be0 tx=0x7f9d18009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:13.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.191+0000 7f9d1fe00700 1 -- 172.21.15.28:0/477270275 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f9d20010040 con 0x7f9d2810b760 2024-05-20T20:01:13.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.437+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) -- 0x7f9d2819d190 con 0x7f9d2810b760 2024-05-20T20:01:13.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.438+0000 7f9d1fe00700 1 -- 172.21.15.28:0/477270275 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v71) ==== 84+0+65 (secure 0 0 0) 0x7f9d200b0e20 con 0x7f9d2810b760 2024-05-20T20:01:13.440 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:01:13.440 INFO:teuthology.orchestra.run.smithi028.stdout:[ 2024-05-20T20:01:13.440 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:01:13.440 INFO:teuthology.orchestra.run.smithi028.stdout: "poolnum": 1, 2024-05-20T20:01:13.440 INFO:teuthology.orchestra.run.smithi028.stdout: "poolname": ".mgr" 2024-05-20T20:01:13.440 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:01:13.440 INFO:teuthology.orchestra.run.smithi028.stdout:] 2024-05-20T20:01:13.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.441+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9d0c0b2880 msgr2=0x7f9d0c0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:13.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.441+0000 7f9d2e800700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9d0c0b2880 0x7f9d0c0b4d40 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f9d18009be0 tx=0x7f9d18009260 comp rx=0 tx=0).stop 2024-05-20T20:01:13.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.441+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d2810b760 msgr2=0x7f9d281a0700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:13.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.441+0000 7f9d2e800700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d2810b760 0x7f9d281a0700 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f9d2000c370 tx=0x7f9d2000c680 comp rx=0 tx=0).stop 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 shutdown_connections 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9d0c0b2880 0x7f9d0c0b4d40 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9d2810b760 0x7f9d281a0700 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9d28105760 0x7f9d2819be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 --2- 172.21.15.28:0/477270275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9d28106120 0x7f9d2819c370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 >> 172.21.15.28:0/477270275 conn(0x7f9d28101250 msgr2=0x7f9d28101a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 shutdown_connections 2024-05-20T20:01:13.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:13.442+0000 7f9d2e800700 1 -- 172.21.15.28:0/477270275 wait complete. 2024-05-20T20:01:13.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:13.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:01:13.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:13 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:13.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:01:13.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:13 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:13.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:01:13.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:01:13.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:13 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:01:14.263 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-05-20T20:01:14.264 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd pool create cephfs_metadata --pg_num_min 64 2024-05-20T20:01:14.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:14 smithi028 ceph-mon[31178]: pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:14.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:14 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/477270275' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-05-20T20:01:14.735 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:14.776 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:14 smithi154 ceph-mon[36314]: pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:14.777 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:14 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/477270275' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-05-20T20:01:14.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:14 smithi033 ceph-mon[36179]: pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:14.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:14 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/477270275' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-05-20T20:01:16.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:16 smithi154 ceph-mon[36314]: pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:16.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:16 smithi028 ceph-mon[31178]: pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:16 smithi033 ceph-mon[36179]: pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:17.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 -- 172.21.15.28:0/4289183214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd03406b220 msgr2=0x7fd03406b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:17.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 --2- 172.21.15.28:0/4289183214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd03406b220 0x7fd03406b6a0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fd020009990 tx=0x7fd020009ca0 comp rx=0 tx=0).stop 2024-05-20T20:01:17.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 -- 172.21.15.28:0/4289183214 shutdown_connections 2024-05-20T20:01:17.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 --2- 172.21.15.28:0/4289183214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd03406bbe0 0x7fd0341103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:17.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 --2- 172.21.15.28:0/4289183214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0341095b0 0x7fd034109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:17.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 --2- 172.21.15.28:0/4289183214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd03406b220 0x7fd03406b6a0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:17.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 -- 172.21.15.28:0/4289183214 >> 172.21.15.28:0/4289183214 conn(0x7fd0340793f0 msgr2=0x7fd034079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:17.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 -- 172.21.15.28:0/4289183214 shutdown_connections 2024-05-20T20:01:17.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.177+0000 7fd039a00700 1 -- 172.21.15.28:0/4289183214 wait complete. 2024-05-20T20:01:17.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.178+0000 7fd039a00700 1 Processor -- start 2024-05-20T20:01:17.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.178+0000 7fd039a00700 1 -- start start 2024-05-20T20:01:17.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd039a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd03406b220 0x7fd03419bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:17.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd039a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd03406bbe0 0x7fd03419c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:17.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd039a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0341095b0 0x7fd0341a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd039a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd0341136e0 con 0x7fd03406bbe0 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd039a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd034113560 con 0x7fd0341095b0 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd039a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd034113860 con 0x7fd03406b220 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd032a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd03406b220 0x7fd03419bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd032a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd03406b220 0x7fd03419bdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55218/0 (socket says 172.21.15.28:55218) 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd032a00700 1 -- 172.21.15.28:0/2841454961 learned_addr learned my addr 172.21.15.28:0/2841454961 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd032000700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd03406bbe0 0x7fd03419c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:17.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.179+0000 7fd033400700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0341095b0 0x7fd0341a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:17.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd032a00700 1 -- 172.21.15.28:0/2841454961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0341095b0 msgr2=0x7fd0341a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:17.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd032a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0341095b0 0x7fd0341a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:17.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd032a00700 1 -- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd03406bbe0 msgr2=0x7fd03419c310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:17.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd032a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd03406bbe0 0x7fd03419c310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd032a00700 1 -- 172.21.15.28:0/2841454961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd0341a0de0 con 0x7fd03406b220 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd032a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd03406b220 0x7fd03419bdd0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fd02400e9c0 tx=0x7fd02400ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd02b400700 1 -- 172.21.15.28:0/2841454961 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd02400cbd0 con 0x7fd03406b220 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd02b400700 1 -- 172.21.15.28:0/2841454961 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd02400ee60 con 0x7fd03406b220 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd0341a10d0 con 0x7fd03406b220 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd02b400700 1 -- 172.21.15.28:0/2841454961 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0240055c0 con 0x7fd03406b220 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.180+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd0341a8950 con 0x7fd03406b220 2024-05-20T20:01:17.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.182+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd03406a230 con 0x7fd03406b220 2024-05-20T20:01:17.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.184+0000 7fd02b400700 1 -- 172.21.15.28:0/2841454961 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fd024047960 con 0x7fd03406b220 2024-05-20T20:01:17.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.185+0000 7fd02b400700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd0180b2900 0x7fd0180b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:17.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.186+0000 7fd02b400700 1 -- 172.21.15.28:0/2841454961 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(71..71 src has 1..71) ==== 7127+0+0 (secure 0 0 0) 0x7fd0240e1f80 con 0x7fd03406b220 2024-05-20T20:01:17.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.186+0000 7fd032000700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd0180b2900 0x7fd0180b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:17.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.186+0000 7fd032000700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd0180b2900 0x7fd0180b4dc0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fd0200098e0 tx=0x7fd020005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:17.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.188+0000 7fd02b400700 1 -- 172.21.15.28:0/2841454961 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fd0240aa0a0 con 0x7fd03406b220 2024-05-20T20:01:17.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:17.434+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) -- 0x7fd0341a8b80 con 0x7fd03406b220 2024-05-20T20:01:18.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.432+0000 7fd02b400700 1 -- 172.21.15.28:0/2841454961 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v72) ==== 138+0+0 (secure 0 0 0) 0x7fd0240a99a0 con 0x7fd03406b220 2024-05-20T20:01:18.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.440+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd0180b2900 msgr2=0x7fd0180b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:18.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.440+0000 7fd039a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd0180b2900 0x7fd0180b4dc0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fd0200098e0 tx=0x7fd020005960 comp rx=0 tx=0).stop 2024-05-20T20:01:18.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.440+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd03406b220 msgr2=0x7fd03419bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:18.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.440+0000 7fd039a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd03406b220 0x7fd03419bdd0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fd02400e9c0 tx=0x7fd02400ecd0 comp rx=0 tx=0).stop 2024-05-20T20:01:18.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.441+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 shutdown_connections 2024-05-20T20:01:18.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.441+0000 7fd039a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd0180b2900 0x7fd0180b4dc0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:18.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.441+0000 7fd039a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0341095b0 0x7fd0341a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:18.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.441+0000 7fd039a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd03406b220 0x7fd03419bdd0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:18.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.441+0000 7fd039a00700 1 --2- 172.21.15.28:0/2841454961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd03406bbe0 0x7fd03419c310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:18.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.441+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 >> 172.21.15.28:0/2841454961 conn(0x7fd0340793f0 msgr2=0x7fd034100f30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:18.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.442+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 shutdown_connections 2024-05-20T20:01:18.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:18.442+0000 7fd039a00700 1 -- 172.21.15.28:0/2841454961 wait complete. 2024-05-20T20:01:18.445 INFO:teuthology.orchestra.run.smithi028.stderr:pool 'cephfs_metadata' created 2024-05-20T20:01:18.789 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:18 smithi028 ceph-mon[31178]: pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:18.789 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:18 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2841454961' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-05-20T20:01:18.789 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:18 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-05-20T20:01:18.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:18 smithi154 ceph-mon[36314]: pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:18.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:18 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2841454961' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-05-20T20:01:18.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:18 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-05-20T20:01:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:18 smithi033 ceph-mon[36179]: pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:18 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2841454961' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-05-20T20:01:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:18 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-05-20T20:01:19.298 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-05-20T20:01:19.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:19 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-05-20T20:01:19.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:19 smithi028 ceph-mon[31178]: osdmap e72: 12 total, 12 up, 12 in 2024-05-20T20:01:19.782 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:19.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:19 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-05-20T20:01:19.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:19 smithi154 ceph-mon[36314]: osdmap e72: 12 total, 12 up, 12 in 2024-05-20T20:01:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:19 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-05-20T20:01:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:19 smithi033 ceph-mon[36179]: osdmap e72: 12 total, 12 up, 12 in 2024-05-20T20:01:20.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:20 smithi028 ceph-mon[31178]: pgmap v310: 33 pgs: 10 creating+peering, 22 unknown, 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:20.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:20 smithi028 ceph-mon[31178]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-05-20T20:01:20.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:20 smithi028 ceph-mon[31178]: osdmap e73: 12 total, 12 up, 12 in 2024-05-20T20:01:20.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:20 smithi154 ceph-mon[36314]: pgmap v310: 33 pgs: 10 creating+peering, 22 unknown, 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:20.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:20 smithi154 ceph-mon[36314]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-05-20T20:01:20.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:20 smithi154 ceph-mon[36314]: osdmap e73: 12 total, 12 up, 12 in 2024-05-20T20:01:20.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:20 smithi033 ceph-mon[36179]: pgmap v310: 33 pgs: 10 creating+peering, 22 unknown, 1 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:20.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:20 smithi033 ceph-mon[36179]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-05-20T20:01:20.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:20 smithi033 ceph-mon[36179]: osdmap e73: 12 total, 12 up, 12 in 2024-05-20T20:01:21.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:21 smithi028 ceph-mon[31178]: osdmap e74: 12 total, 12 up, 12 in 2024-05-20T20:01:21.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:21 smithi154 ceph-mon[36314]: osdmap e74: 12 total, 12 up, 12 in 2024-05-20T20:01:21.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:21 smithi033 ceph-mon[36179]: osdmap e74: 12 total, 12 up, 12 in 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 -- 172.21.15.28:0/3195331596 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 msgr2=0x7f02a81103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 --2- 172.21.15.28:0/3195331596 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 0x7f02a81103d0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f02a000b0a0 tx=0x7f02a000b3b0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 -- 172.21.15.28:0/3195331596 shutdown_connections 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 --2- 172.21.15.28:0/3195331596 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 0x7f02a81103d0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 --2- 172.21.15.28:0/3195331596 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02a81095b0 0x7f02a8109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 --2- 172.21.15.28:0/3195331596 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 0x7f02a806c0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 -- 172.21.15.28:0/3195331596 >> 172.21.15.28:0/3195331596 conn(0x7f02a80793f0 msgr2=0x7f02a8079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.149+0000 7f02ae200700 1 -- 172.21.15.28:0/3195331596 shutdown_connections 2024-05-20T20:01:22.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.150+0000 7f02ae200700 1 -- 172.21.15.28:0/3195331596 wait complete. 2024-05-20T20:01:22.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.151+0000 7f02ae200700 1 Processor -- start 2024-05-20T20:01:22.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.151+0000 7f02ae200700 1 -- start start 2024-05-20T20:01:22.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02ae200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 0x7f02a819f870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:22.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02ae200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 0x7f02a819fdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02ae200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02a81095b0 0x7f02a8199a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02ae200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f02a8113720 con 0x7f02a806c630 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02ae200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f02a81135a0 con 0x7f02a806bc70 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02ae200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f02a81138a0 con 0x7f02a81095b0 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02a7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 0x7f02a819f870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f029ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 0x7f02a819fdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02a7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 0x7f02a819f870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53750/0 (socket says 172.21.15.28:53750) 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02a7400700 1 -- 172.21.15.28:0/3873826870 learned_addr learned my addr 172.21.15.28:0/3873826870 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.152+0000 7f02a7e00700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02a81095b0 0x7f02a8199a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a7400700 1 -- 172.21.15.28:0/3873826870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02a81095b0 msgr2=0x7f02a8199a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:22.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a7400700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02a81095b0 0x7f02a8199a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a7400700 1 -- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 msgr2=0x7f02a819fdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:22.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a7400700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 0x7f02a819fdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a7400700 1 -- 172.21.15.28:0/3873826870 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f02a819a1e0 con 0x7f02a806bc70 2024-05-20T20:01:22.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f029ea00700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 0x7f02a819fdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:01:22.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a7400700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 0x7f02a819f870 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f029800e9a0 tx=0x7f029800ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:22.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a4c00700 1 -- 172.21.15.28:0/3873826870 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f029800ca10 con 0x7f02a806bc70 2024-05-20T20:01:22.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a4c00700 1 -- 172.21.15.28:0/3873826870 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f029800cbb0 con 0x7f02a806bc70 2024-05-20T20:01:22.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02a4c00700 1 -- 172.21.15.28:0/3873826870 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0298018680 con 0x7f02a806bc70 2024-05-20T20:01:22.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.153+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f02a819a4d0 con 0x7f02a806bc70 2024-05-20T20:01:22.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.154+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f02a81a6740 con 0x7f02a806bc70 2024-05-20T20:01:22.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.155+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f02a806a230 con 0x7f02a806bc70 2024-05-20T20:01:22.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.157+0000 7f02a4c00700 1 -- 172.21.15.28:0/3873826870 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f0298010430 con 0x7f02a806bc70 2024-05-20T20:01:22.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.158+0000 7f02a4c00700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02880b2900 0x7f02880b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:22.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.159+0000 7f02a4c00700 1 -- 172.21.15.28:0/3873826870 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(74..74 src has 1..74) ==== 7497+0+0 (secure 0 0 0) 0x7f02980e2370 con 0x7f02a806bc70 2024-05-20T20:01:22.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.159+0000 7f029ea00700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02880b2900 0x7f02880b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:22.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.160+0000 7f029ea00700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02880b2900 0x7f02880b4dc0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f02a8079a60 tx=0x7f0290009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:22.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.161+0000 7f02a4c00700 1 -- 172.21.15.28:0/3873826870 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f02980aa320 con 0x7f02a806bc70 2024-05-20T20:01:22.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.401+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) -- 0x7f02a802e560 con 0x7f02a806bc70 2024-05-20T20:01:22.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.478+0000 7f02a4c00700 1 -- 172.21.15.28:0/3873826870 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v75) ==== 170+0+0 (secure 0 0 0) 0x7f02980b23a0 con 0x7f02a806bc70 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.486+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02880b2900 msgr2=0x7f02880b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.486+0000 7f02ae200700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02880b2900 0x7f02880b4dc0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f02a8079a60 tx=0x7f0290009260 comp rx=0 tx=0).stop 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.486+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 msgr2=0x7f02a819f870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.486+0000 7f02ae200700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 0x7f02a819f870 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f029800e9a0 tx=0x7f029800ecb0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.487+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 shutdown_connections 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.487+0000 7f02ae200700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f02880b2900 0x7f02880b4dc0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.487+0000 7f02ae200700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02a81095b0 0x7f02a8199a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.487+0000 7f02ae200700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02a806bc70 0x7f02a819f870 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.487+0000 7f02ae200700 1 --2- 172.21.15.28:0/3873826870 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02a806c630 0x7f02a819fdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:22.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.487+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 >> 172.21.15.28:0/3873826870 conn(0x7f02a80793f0 msgr2=0x7f02a81011d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:22.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.489+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 shutdown_connections 2024-05-20T20:01:22.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:22.489+0000 7f02ae200700 1 -- 172.21.15.28:0/3873826870 wait complete. 2024-05-20T20:01:22.496 INFO:teuthology.orchestra.run.smithi028.stderr:pool 'cephfs_data' created 2024-05-20T20:01:22.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:22 smithi028 ceph-mon[31178]: pgmap v313: 33 pgs: 10 creating+peering, 17 unknown, 6 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:22.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:22 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3873826870' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-05-20T20:01:22.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:22 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-05-20T20:01:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:22 smithi154 ceph-mon[36314]: pgmap v313: 33 pgs: 10 creating+peering, 17 unknown, 6 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:22 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3873826870' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-05-20T20:01:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:22 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-05-20T20:01:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:22 smithi033 ceph-mon[36179]: pgmap v313: 33 pgs: 10 creating+peering, 17 unknown, 6 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:22 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3873826870' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-05-20T20:01:22.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:22 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-05-20T20:01:23.258 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs new cephfs cephfs_metadata cephfs_data 2024-05-20T20:01:23.727 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:23.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:23 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-05-20T20:01:23.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:23 smithi028 ceph-mon[31178]: osdmap e75: 12 total, 12 up, 12 in 2024-05-20T20:01:23.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:23 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-05-20T20:01:23.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:23 smithi154 ceph-mon[36314]: osdmap e75: 12 total, 12 up, 12 in 2024-05-20T20:01:23.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:23 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-05-20T20:01:23.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:23 smithi033 ceph-mon[36179]: osdmap e75: 12 total, 12 up, 12 in 2024-05-20T20:01:24.778 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:24 smithi033 ceph-mon[36179]: pgmap v315: 97 pgs: 22 creating+peering, 52 unknown, 23 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:24.779 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:24 smithi033 ceph-mon[36179]: osdmap e76: 12 total, 12 up, 12 in 2024-05-20T20:01:24.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:24 smithi028 ceph-mon[31178]: pgmap v315: 97 pgs: 22 creating+peering, 52 unknown, 23 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:24.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:24 smithi028 ceph-mon[31178]: osdmap e76: 12 total, 12 up, 12 in 2024-05-20T20:01:24.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:24 smithi154 ceph-mon[36314]: pgmap v315: 97 pgs: 22 creating+peering, 52 unknown, 23 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:24.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:24 smithi154 ceph-mon[36314]: osdmap e76: 12 total, 12 up, 12 in 2024-05-20T20:01:25.748 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:25 smithi028 ceph-mon[31178]: osdmap e77: 12 total, 12 up, 12 in 2024-05-20T20:01:25.748 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:25 smithi028 ceph-mon[31178]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-05-20T20:01:25.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:25 smithi154 ceph-mon[36314]: osdmap e77: 12 total, 12 up, 12 in 2024-05-20T20:01:25.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:25 smithi154 ceph-mon[36314]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-05-20T20:01:25.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:25 smithi033 ceph-mon[36179]: osdmap e77: 12 total, 12 up, 12 in 2024-05-20T20:01:25.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:25 smithi033 ceph-mon[36179]: Health check update: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-05-20T20:01:25.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 -- 172.21.15.28:0/3620085816 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62006b250 msgr2=0x7ff62006b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:25.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 --2- 172.21.15.28:0/3620085816 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62006b250 0x7ff62006b630 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7ff61000b0a0 tx=0x7ff61000b3b0 comp rx=0 tx=0).stop 2024-05-20T20:01:25.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 -- 172.21.15.28:0/3620085816 shutdown_connections 2024-05-20T20:01:25.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 --2- 172.21.15.28:0/3620085816 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62010fff0 0x7ff6201123e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:25.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 --2- 172.21.15.28:0/3620085816 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62006b250 0x7ff62006b630 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:25.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 --2- 172.21.15.28:0/3620085816 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff62006bb70 0x7ff62010fab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:25.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 -- 172.21.15.28:0/3620085816 >> 172.21.15.28:0/3620085816 conn(0x7ff6200793f0 msgr2=0x7ff620079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:25.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 -- 172.21.15.28:0/3620085816 shutdown_connections 2024-05-20T20:01:25.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.924+0000 7ff627000700 1 -- 172.21.15.28:0/3620085816 wait complete. 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.926+0000 7ff627000700 1 Processor -- start 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.926+0000 7ff627000700 1 -- start start 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.926+0000 7ff627000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62006b250 0x7ff62019be00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.926+0000 7ff627000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff62006bb70 0x7ff62019c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.926+0000 7ff627000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62010fff0 0x7ff6201a06d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff627000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff6201158f0 con 0x7ff62006bb70 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff627000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff620115770 con 0x7ff62006b250 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff627000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff620115a70 con 0x7ff62010fff0 2024-05-20T20:01:25.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff61fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62006b250 0x7ff62019be00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:25.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff61fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62006b250 0x7ff62019be00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53766/0 (socket says 172.21.15.28:53766) 2024-05-20T20:01:25.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff61fe00700 1 -- 172.21.15.28:0/3948616500 learned_addr learned my addr 172.21.15.28:0/3948616500 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff617400700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff62006bb70 0x7ff62019c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff625000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62010fff0 0x7ff6201a06d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff625000700 1 -- 172.21.15.28:0/3948616500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62006b250 msgr2=0x7ff62019be00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff625000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62006b250 0x7ff62019be00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff625000700 1 -- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff62006bb70 msgr2=0x7ff62019c340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff625000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff62006bb70 0x7ff62019c340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff625000700 1 -- 172.21.15.28:0/3948616500 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff6201a0e10 con 0x7ff62010fff0 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.928+0000 7ff61fe00700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62006b250 0x7ff62019be00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.927+0000 7ff617400700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff62006bb70 0x7ff62019c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.928+0000 7ff625000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62010fff0 0x7ff6201a06d0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7ff61800e8d0 tx=0x7ff61800ebe0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.928+0000 7ff61d600700 1 -- 172.21.15.28:0/3948616500 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff618004220 con 0x7ff62010fff0 2024-05-20T20:01:25.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.928+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff6201a1100 con 0x7ff62010fff0 2024-05-20T20:01:25.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.928+0000 7ff61d600700 1 -- 172.21.15.28:0/3948616500 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6180043c0 con 0x7ff62010fff0 2024-05-20T20:01:25.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.928+0000 7ff61d600700 1 -- 172.21.15.28:0/3948616500 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff618003d60 con 0x7ff62010fff0 2024-05-20T20:01:25.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.928+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff6201a8980 con 0x7ff62010fff0 2024-05-20T20:01:25.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.929+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff62006a230 con 0x7ff62010fff0 2024-05-20T20:01:25.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.932+0000 7ff61d600700 1 -- 172.21.15.28:0/3948616500 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7ff61800c760 con 0x7ff62010fff0 2024-05-20T20:01:25.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.933+0000 7ff61d600700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff6040b2880 0x7ff6040b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:25.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.933+0000 7ff61d600700 1 -- 172.21.15.28:0/3948616500 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(77..77 src has 1..77) ==== 7879+0+0 (secure 0 0 0) 0x7ff6180e21e0 con 0x7ff62010fff0 2024-05-20T20:01:25.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.934+0000 7ff61fe00700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff6040b2880 0x7ff6040b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:25.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.934+0000 7ff61fe00700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff6040b2880 0x7ff6040b4d40 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7ff610009e60 tx=0x7ff61000b740 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:25.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:25.936+0000 7ff61d600700 1 -- 172.21.15.28:0/3948616500 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7ff6180aa0a0 con 0x7ff62010fff0 2024-05-20T20:01:26.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.228+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) -- 0x7ff62002e560 con 0x7ff62010fff0 2024-05-20T20:01:26.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.520+0000 7ff61d600700 1 -- 172.21.15.28:0/3948616500 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v14) ==== 172+0+0 (secure 0 0 0) 0x7ff6180a99a0 con 0x7ff62010fff0 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.527+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff6040b2880 msgr2=0x7ff6040b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.527+0000 7ff627000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff6040b2880 0x7ff6040b4d40 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7ff610009e60 tx=0x7ff61000b740 comp rx=0 tx=0).stop 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.527+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62010fff0 msgr2=0x7ff6201a06d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.527+0000 7ff627000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62010fff0 0x7ff6201a06d0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7ff61800e8d0 tx=0x7ff61800ebe0 comp rx=0 tx=0).stop 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.529+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 shutdown_connections 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.529+0000 7ff627000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff6040b2880 0x7ff6040b4d40 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.529+0000 7ff627000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff62010fff0 0x7ff6201a06d0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.529+0000 7ff627000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff62006b250 0x7ff62019be00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.529+0000 7ff627000700 1 --2- 172.21.15.28:0/3948616500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff62006bb70 0x7ff62019c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.529+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 >> 172.21.15.28:0/3948616500 conn(0x7ff6200793f0 msgr2=0x7ff6201114c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:26.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.531+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 shutdown_connections 2024-05-20T20:01:26.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:26.531+0000 7ff627000700 1 -- 172.21.15.28:0/3948616500 wait complete. 2024-05-20T20:01:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:new fs with metadata pool 2 and data pool 3 2024-05-20T20:01:26.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:26 smithi028 ceph-mon[31178]: pgmap v318: 97 pgs: 20 creating+peering, 16 unknown, 61 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:26.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:26 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3948616500' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-05-20T20:01:26.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:26 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-05-20T20:01:26.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:26 smithi028 ceph-mon[31178]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-05-20T20:01:26.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:26 smithi154 ceph-mon[36314]: pgmap v318: 97 pgs: 20 creating+peering, 16 unknown, 61 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:26.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:26 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3948616500' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-05-20T20:01:26.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:26 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-05-20T20:01:26.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:26 smithi154 ceph-mon[36314]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-05-20T20:01:26.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:26 smithi033 ceph-mon[36179]: pgmap v318: 97 pgs: 20 creating+peering, 16 unknown, 61 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:01:26.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:26 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3948616500' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-05-20T20:01:26.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:26 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-05-20T20:01:26.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:26 smithi033 ceph-mon[36179]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-05-20T20:01:27.405 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:01:27.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:27 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-05-20T20:01:27.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:27 smithi028 ceph-mon[31178]: osdmap e78: 12 total, 12 up, 12 in 2024-05-20T20:01:27.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:27 smithi028 ceph-mon[31178]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-05-20T20:01:27.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:27 smithi028 ceph-mon[31178]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-05-20T20:01:27.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:27 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-05-20T20:01:27.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:27 smithi154 ceph-mon[36314]: osdmap e78: 12 total, 12 up, 12 in 2024-05-20T20:01:27.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:27 smithi154 ceph-mon[36314]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-05-20T20:01:27.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:27 smithi154 ceph-mon[36314]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-05-20T20:01:27.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:27 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-05-20T20:01:27.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:27 smithi033 ceph-mon[36179]: osdmap e78: 12 total, 12 up, 12 in 2024-05-20T20:01:27.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:27 smithi033 ceph-mon[36179]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-05-20T20:01:27.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:27 smithi033 ceph-mon[36179]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-05-20T20:01:27.889 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:28.790 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:28 smithi028 ceph-mon[31178]: pgmap v320: 97 pgs: 20 creating+peering, 4 unknown, 73 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail; 418 B/s wr, 1 op/s 2024-05-20T20:01:28.790 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:28 smithi028 ceph-mon[31178]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-05-20T20:01:28.790 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:28 smithi028 ceph-mon[31178]: Cluster is now healthy 2024-05-20T20:01:28.791 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:28 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] up:active 2024-05-20T20:01:28.791 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:28 smithi028 ceph-mon[31178]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:28.791 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:28 smithi028 ceph-mon[31178]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:28 smithi154 ceph-mon[36314]: pgmap v320: 97 pgs: 20 creating+peering, 4 unknown, 73 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail; 418 B/s wr, 1 op/s 2024-05-20T20:01:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:28 smithi154 ceph-mon[36314]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-05-20T20:01:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:28 smithi154 ceph-mon[36314]: Cluster is now healthy 2024-05-20T20:01:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:28 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] up:active 2024-05-20T20:01:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:28 smithi154 ceph-mon[36314]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:28.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:28 smithi154 ceph-mon[36314]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:28 smithi033 ceph-mon[36179]: pgmap v320: 97 pgs: 20 creating+peering, 4 unknown, 73 active+clean; 577 KiB data, 338 MiB used, 1.0 TiB / 1.0 TiB avail; 418 B/s wr, 1 op/s 2024-05-20T20:01:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:28 smithi033 ceph-mon[36179]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-05-20T20:01:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:28 smithi033 ceph-mon[36179]: Cluster is now healthy 2024-05-20T20:01:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:28 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] up:active 2024-05-20T20:01:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:28 smithi033 ceph-mon[36179]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:28 smithi033 ceph-mon[36179]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:29.877 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: 2024-05-20T20:01:29.462+0000 7feffa946700 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-05-20T20:01:29.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 -- 172.21.15.28:0/2161196630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488105760 msgr2=0x7fc488105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:29.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 --2- 172.21.15.28:0/2161196630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488105760 0x7fc488105be0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fc474009990 tx=0x7fc474009ca0 comp rx=0 tx=0).stop 2024-05-20T20:01:29.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 -- 172.21.15.28:0/2161196630 shutdown_connections 2024-05-20T20:01:29.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 --2- 172.21.15.28:0/2161196630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488106120 0x7fc488112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:29.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 --2- 172.21.15.28:0/2161196630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc48810b760 0x7fc48810bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:29.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 --2- 172.21.15.28:0/2161196630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488105760 0x7fc488105be0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:29.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 -- 172.21.15.28:0/2161196630 >> 172.21.15.28:0/2161196630 conn(0x7fc488101250 msgr2=0x7fc488103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:29.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 -- 172.21.15.28:0/2161196630 shutdown_connections 2024-05-20T20:01:29.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.962+0000 7fc48f800700 1 -- 172.21.15.28:0/2161196630 wait complete. 2024-05-20T20:01:29.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.964+0000 7fc48f800700 1 Processor -- start 2024-05-20T20:01:29.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48f800700 1 -- start start 2024-05-20T20:01:29.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48f800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488105760 0x7fc48819be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:29.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48f800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488106120 0x7fc48819c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:29.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48f800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc48810b760 0x7fc4881a06b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:29.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48f800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc488115960 con 0x7fc488105760 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48f800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc4881157e0 con 0x7fc488106120 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48f800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc488115ae0 con 0x7fc48810b760 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.965+0000 7fc48ce00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488105760 0x7fc48819be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488105760 0x7fc48819be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:46020/0 (socket says 172.21.15.28:46020) 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 -- 172.21.15.28:0/3540626650 learned_addr learned my addr 172.21.15.28:0/3540626650 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc487e00700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488106120 0x7fc48819c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48d800700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc48810b760 0x7fc4881a06b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 -- 172.21.15.28:0/3540626650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc48810b760 msgr2=0x7fc4881a06b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc48810b760 0x7fc4881a06b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:29.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 -- 172.21.15.28:0/3540626650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488106120 msgr2=0x7fc48819c370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488106120 0x7fc48819c370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 -- 172.21.15.28:0/3540626650 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc4881a0df0 con 0x7fc488105760 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.966+0000 7fc48ce00700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488105760 0x7fc48819be30 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fc47c00e9c0 tx=0x7fc47c00ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.967+0000 7fc485600700 1 -- 172.21.15.28:0/3540626650 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc47c00cbd0 con 0x7fc488105760 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.967+0000 7fc485600700 1 -- 172.21.15.28:0/3540626650 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc47c00ee60 con 0x7fc488105760 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.967+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc4881a10e0 con 0x7fc488105760 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.967+0000 7fc485600700 1 -- 172.21.15.28:0/3540626650 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc47c0055c0 con 0x7fc488105760 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.967+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc4881a8960 con 0x7fc488105760 2024-05-20T20:01:29.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.968+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc48806a230 con 0x7fc488105760 2024-05-20T20:01:29.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.971+0000 7fc485600700 1 -- 172.21.15.28:0/3540626650 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fc47c004750 con 0x7fc488105760 2024-05-20T20:01:29.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.972+0000 7fc485600700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc4700b2880 0x7fc4700b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:29.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.972+0000 7fc485600700 1 -- 172.21.15.28:0/3540626650 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7fc47c0e25e0 con 0x7fc488105760 2024-05-20T20:01:29.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.973+0000 7fc487e00700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc4700b2880 0x7fc4700b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:29.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.973+0000 7fc487e00700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc4700b2880 0x7fc4700b4d40 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fc4740098e0 tx=0x7fc474005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:29.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:29.975+0000 7fc485600700 1 -- 172.21.15.28:0/3540626650 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fc47c054df0 con 0x7fc488105760 2024-05-20T20:01:30.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.217+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fc48819d140 con 0x7fc488105760 2024-05-20T20:01:30.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.221+0000 7fc485600700 1 -- 172.21.15.28:0/3540626650 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v78) ==== 74+0+22146 (secure 0 0 0) 0x7fc47c0aa3b0 con 0x7fc488105760 2024-05-20T20:01:30.225 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:01:30.228 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":78,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:01:26.232038+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:01:30.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc4700b2880 msgr2=0x7fc4700b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc4700b2880 0x7fc4700b4d40 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fc4740098e0 tx=0x7fc474005960 comp rx=0 tx=0).stop 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488105760 msgr2=0x7fc48819be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488105760 0x7fc48819be30 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fc47c00e9c0 tx=0x7fc47c00ecd0 comp rx=0 tx=0).stop 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 shutdown_connections 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc4700b2880 0x7fc4700b4d40 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc48810b760 0x7fc4881a06b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc488105760 0x7fc48819be30 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 --2- 172.21.15.28:0/3540626650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc488106120 0x7fc48819c370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.224+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 >> 172.21.15.28:0/3540626650 conn(0x7fc488101250 msgr2=0x7fc488101850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.225+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 shutdown_connections 2024-05-20T20:01:30.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:30.225+0000 7fc48f800700 1 -- 172.21.15.28:0/3540626650 wait complete. 2024-05-20T20:01:30.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:30 smithi028 ceph-mon[31178]: pgmap v321: 97 pgs: 97 active+clean; 578 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 6 op/s 2024-05-20T20:01:30.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:30 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3540626650' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:01:30.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:30 smithi154 ceph-mon[36314]: pgmap v321: 97 pgs: 97 active+clean; 578 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 6 op/s 2024-05-20T20:01:30.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:30 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3540626650' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:01:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:30 smithi033 ceph-mon[36179]: pgmap v321: 97 pgs: 97 active+clean; 578 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 6 op/s 2024-05-20T20:01:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:30 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3540626650' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:01:31.144 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:01:31.619 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:32.815 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:32 smithi028 ceph-mon[31178]: pgmap v322: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 KiB/s wr, 5 op/s 2024-05-20T20:01:32.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:32 smithi154 ceph-mon[36314]: pgmap v322: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 KiB/s wr, 5 op/s 2024-05-20T20:01:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:32 smithi033 ceph-mon[36179]: pgmap v322: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.8 KiB/s wr, 5 op/s 2024-05-20T20:01:33.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1242242325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 msgr2=0x7fa7ac10fa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:33.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1242242325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 0x7fa7ac10fa00 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fa794009990 tx=0x7fa794009ca0 comp rx=0 tx=0).stop 2024-05-20T20:01:33.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1242242325 shutdown_connections 2024-05-20T20:01:33.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1242242325 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac10ff40 0x7fa7ac112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:33.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1242242325 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7ac075770 0x7fa7ac075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:33.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1242242325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 0x7fa7ac10fa00 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:33.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1242242325 >> 172.21.15.28:0/1242242325 conn(0x7fa7ac07a6a0 msgr2=0x7fa7ac07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:33.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1242242325 shutdown_connections 2024-05-20T20:01:33.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.822+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1242242325 wait complete. 2024-05-20T20:01:33.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.823+0000 7fa7b3c00700 1 Processor -- start 2024-05-20T20:01:33.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.823+0000 7fa7b3c00700 1 -- start start 2024-05-20T20:01:33.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b3c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac075770 0x7fa7ac19bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:33.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b3c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 0x7fa7ac19c330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:33.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b3c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7ac10ff40 0x7fa7ac1a0670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:33.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b3c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa7ac115900 con 0x7fa7ac076090 2024-05-20T20:01:33.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b3c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa7ac115780 con 0x7fa7ac10ff40 2024-05-20T20:01:33.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b3c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa7ac115a80 con 0x7fa7ac075770 2024-05-20T20:01:33.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac075770 0x7fa7ac19bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:33.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7a3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 0x7fa7ac19c330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:33.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac075770 0x7fa7ac19bdf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47926/0 (socket says 172.21.15.28:47926) 2024-05-20T20:01:33.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 -- 172.21.15.28:0/1438554117 learned_addr learned my addr 172.21.15.28:0/1438554117 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:33.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1c00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7ac10ff40 0x7fa7ac1a0670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 -- 172.21.15.28:0/1438554117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7ac10ff40 msgr2=0x7fa7ac1a0670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7ac10ff40 0x7fa7ac1a0670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 -- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 msgr2=0x7fa7ac19c330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 0x7fa7ac19c330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.824+0000 7fa7b1200700 1 -- 172.21.15.28:0/1438554117 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa7ac1a0db0 con 0x7fa7ac075770 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.825+0000 7fa7b1200700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac075770 0x7fa7ac19bdf0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa79c00e9a0 tx=0x7fa79c00ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.825+0000 7fa7aa000700 1 -- 172.21.15.28:0/1438554117 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa79c00ca10 con 0x7fa7ac075770 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.825+0000 7fa7aa000700 1 -- 172.21.15.28:0/1438554117 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa79c00cbb0 con 0x7fa7ac075770 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.825+0000 7fa7aa000700 1 -- 172.21.15.28:0/1438554117 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa79c018680 con 0x7fa7ac075770 2024-05-20T20:01:33.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.825+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa7ac1a10a0 con 0x7fa7ac075770 2024-05-20T20:01:33.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.825+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa7ac1a8920 con 0x7fa7ac075770 2024-05-20T20:01:33.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.826+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa7ac06a230 con 0x7fa7ac075770 2024-05-20T20:01:33.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.829+0000 7fa7aa000700 1 -- 172.21.15.28:0/1438554117 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fa79c010430 con 0x7fa7ac075770 2024-05-20T20:01:33.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.830+0000 7fa7aa000700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa78c0b2880 0x7fa78c0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:33.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.830+0000 7fa7aa000700 1 -- 172.21.15.28:0/1438554117 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7fa79c0e22b0 con 0x7fa7ac075770 2024-05-20T20:01:33.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.830+0000 7fa7a3e00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa78c0b2880 0x7fa78c0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:33.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.831+0000 7fa7a3e00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa78c0b2880 0x7fa78c0b4d40 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fa79400b2e0 tx=0x7fa794005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:33.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:33.834+0000 7fa7aa000700 1 -- 172.21.15.28:0/1438554117 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa79c0aa080 con 0x7fa7ac075770 2024-05-20T20:01:34.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.072+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fa7ac02e560 con 0x7fa7ac075770 2024-05-20T20:01:34.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.074+0000 7fa7aa000700 1 -- 172.21.15.28:0/1438554117 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v78) ==== 74+0+22146 (secure 0 0 0) 0x7fa79c0b2100 con 0x7fa7ac075770 2024-05-20T20:01:34.075 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:01:34.075 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":78,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:01:26.232038+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.077+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa78c0b2880 msgr2=0x7fa78c0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.077+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa78c0b2880 0x7fa78c0b4d40 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fa79400b2e0 tx=0x7fa794005960 comp rx=0 tx=0).stop 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.077+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac075770 msgr2=0x7fa7ac19bdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.077+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac075770 0x7fa7ac19bdf0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fa79c00e9a0 tx=0x7fa79c00ecb0 comp rx=0 tx=0).stop 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 shutdown_connections 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa78c0b2880 0x7fa78c0b4d40 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7ac10ff40 0x7fa7ac1a0670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7ac075770 0x7fa7ac19bdf0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 --2- 172.21.15.28:0/1438554117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7ac076090 0x7fa7ac19c330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:34.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 >> 172.21.15.28:0/1438554117 conn(0x7fa7ac07a6a0 msgr2=0x7fa7ac100750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:34.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 shutdown_connections 2024-05-20T20:01:34.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:34.078+0000 7fa7b3c00700 1 -- 172.21.15.28:0/1438554117 wait complete. 2024-05-20T20:01:34.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:34 smithi028 ceph-mon[31178]: pgmap v323: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.6 KiB/s wr, 5 op/s 2024-05-20T20:01:34.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:34 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1438554117' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:01:34.875 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs set cephfs standby_count_wanted 0 2024-05-20T20:01:34.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:34 smithi154 ceph-mon[36314]: pgmap v323: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.6 KiB/s wr, 5 op/s 2024-05-20T20:01:34.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:34 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1438554117' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:01:34.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:34 smithi033 ceph-mon[36179]: pgmap v323: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.6 KiB/s wr, 5 op/s 2024-05-20T20:01:34.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:34 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1438554117' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:01:35.350 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:36.751 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:36 smithi028 ceph-mon[31178]: pgmap v324: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.4 KiB/s wr, 4 op/s 2024-05-20T20:01:36.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:36 smithi154 ceph-mon[36314]: pgmap v324: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.4 KiB/s wr, 4 op/s 2024-05-20T20:01:36.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:36 smithi033 ceph-mon[36179]: pgmap v324: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.4 KiB/s wr, 4 op/s 2024-05-20T20:01:37.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.635+0000 7fa30fe00700 1 -- 172.21.15.28:0/3420323388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 msgr2=0x7fa310105ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3420323388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa310105ad0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fa2fc009990 tx=0x7fa2fc009ca0 comp rx=0 tx=0).stop 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 -- 172.21.15.28:0/3420323388 shutdown_connections 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3420323388 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa310106010 0x7fa310112520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3420323388 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa31010b650 0x7fa31010ba30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3420323388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa310105ad0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 -- 172.21.15.28:0/3420323388 >> 172.21.15.28:0/3420323388 conn(0x7fa310101180 msgr2=0x7fa3101035a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 -- 172.21.15.28:0/3420323388 shutdown_connections 2024-05-20T20:01:37.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.636+0000 7fa30fe00700 1 -- 172.21.15.28:0/3420323388 wait complete. 2024-05-20T20:01:37.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.637+0000 7fa30fe00700 1 Processor -- start 2024-05-20T20:01:37.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.637+0000 7fa30fe00700 1 -- start start 2024-05-20T20:01:37.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.637+0000 7fa30fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa31019bd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:37.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa310106010 0x7fa31019c2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa31010b650 0x7fa3101a0660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30fe00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa3101158e0 con 0x7fa310105650 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30fe00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa310115760 con 0x7fa310106010 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30fe00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa310115a60 con 0x7fa31010b650 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa31019bd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa31019bd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60698/0 (socket says 172.21.15.28:60698) 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 -- 172.21.15.28:0/3389859549 learned_addr learned my addr 172.21.15.28:0/3389859549 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30e000700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa310106010 0x7fa31019c2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30f400700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa31010b650 0x7fa3101a0660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 -- 172.21.15.28:0/3389859549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa31010b650 msgr2=0x7fa3101a0660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:37.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa31010b650 0x7fa3101a0660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 -- 172.21.15.28:0/3389859549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa310106010 msgr2=0x7fa31019c2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa310106010 0x7fa31019c2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.638+0000 7fa30ea00700 1 -- 172.21.15.28:0/3389859549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa3101a0da0 con 0x7fa310105650 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.639+0000 7fa30ea00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa31019bd90 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fa30000e9a0 tx=0x7fa30000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.639+0000 7fa307400700 1 -- 172.21.15.28:0/3389859549 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa30000ca10 con 0x7fa310105650 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.639+0000 7fa307400700 1 -- 172.21.15.28:0/3389859549 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa30000cbb0 con 0x7fa310105650 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.639+0000 7fa307400700 1 -- 172.21.15.28:0/3389859549 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa300018680 con 0x7fa310105650 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.639+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa3101a1090 con 0x7fa310105650 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.639+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa3101a8910 con 0x7fa310105650 2024-05-20T20:01:37.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.640+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa31006a230 con 0x7fa310105650 2024-05-20T20:01:37.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.644+0000 7fa307400700 1 -- 172.21.15.28:0/3389859549 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fa300010430 con 0x7fa310105650 2024-05-20T20:01:37.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.646+0000 7fa307400700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa2f40b2900 0x7fa2f40b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:37.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.646+0000 7fa307400700 1 -- 172.21.15.28:0/3389859549 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7fa3000e2550 con 0x7fa310105650 2024-05-20T20:01:37.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.647+0000 7fa30e000700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa2f40b2900 0x7fa2f40b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:37.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.647+0000 7fa30e000700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa2f40b2900 0x7fa2f40b4dc0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fa310103880 tx=0x7fa2fc00b260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:37.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.648+0000 7fa307400700 1 -- 172.21.15.28:0/3389859549 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa3000aa320 con 0x7fa310105650 2024-05-20T20:01:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:37.931+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) -- 0x7fa31019d0f0 con 0x7fa310105650 2024-05-20T20:01:38.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.570+0000 7fa307400700 1 -- 172.21.15.28:0/3389859549 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v17) ==== 118+0+0 (secure 0 0 0) 0x7fa3000b23a0 con 0x7fa310105650 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa2f40b2900 msgr2=0x7fa2f40b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa2f40b2900 0x7fa2f40b4dc0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fa310103880 tx=0x7fa2fc00b260 comp rx=0 tx=0).stop 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 msgr2=0x7fa31019bd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa31019bd90 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fa30000e9a0 tx=0x7fa30000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 shutdown_connections 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa2f40b2900 0x7fa2f40b4dc0 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa31010b650 0x7fa3101a0660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa310105650 0x7fa31019bd90 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 --2- 172.21.15.28:0/3389859549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa310106010 0x7fa31019c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.574+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 >> 172.21.15.28:0/3389859549 conn(0x7fa310101180 msgr2=0x7fa310101a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:38.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.575+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 shutdown_connections 2024-05-20T20:01:38.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:38.575+0000 7fa30fe00700 1 -- 172.21.15.28:0/3389859549 wait complete. 2024-05-20T20:01:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:38 smithi028 ceph-mon[31178]: pgmap v325: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 KiB/s wr, 4 op/s 2024-05-20T20:01:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:38 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3389859549' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-05-20T20:01:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:38 smithi154 ceph-mon[36314]: pgmap v325: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 KiB/s wr, 4 op/s 2024-05-20T20:01:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:38 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3389859549' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-05-20T20:01:38.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:38 smithi033 ceph-mon[36179]: pgmap v325: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 KiB/s wr, 4 op/s 2024-05-20T20:01:38.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:38 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3389859549' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-05-20T20:01:39.343 DEBUG:tasks.cephfs.filesystem:fs_config: {'max_mds': 5, 'session_timeout': 300, 'standby_replay': True, 'subvols': {'create': 2, 'subvol_options': '--namespace-isolated --size 25000000000'}} 2024-05-20T20:01:39.344 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs set cephfs max_mds 5 2024-05-20T20:01:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:39 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3389859549' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-05-20T20:01:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:39 smithi154 ceph-mon[36314]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:39 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3389859549' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-05-20T20:01:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:39 smithi033 ceph-mon[36179]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:39.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:39 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3389859549' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-05-20T20:01:39.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:39 smithi028 ceph-mon[31178]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:40 smithi028 ceph-mon[31178]: pgmap v326: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 3 op/s 2024-05-20T20:01:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:40 smithi154 ceph-mon[36314]: pgmap v326: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 3 op/s 2024-05-20T20:01:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:40 smithi033 ceph-mon[36179]: pgmap v326: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 3 op/s 2024-05-20T20:01:42.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.183+0000 7f79e7200700 1 -- 172.21.15.28:0/3160542943 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e010ff40 msgr2=0x7f79e0112330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:42.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.183+0000 7f79e7200700 1 --2- 172.21.15.28:0/3160542943 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e010ff40 0x7f79e0112330 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f79d800b0a0 tx=0x7f79d800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.183+0000 7f79e7200700 1 -- 172.21.15.28:0/3160542943 shutdown_connections 2024-05-20T20:01:42.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.183+0000 7f79e7200700 1 --2- 172.21.15.28:0/3160542943 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e010ff40 0x7f79e0112330 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.183+0000 7f79e7200700 1 --2- 172.21.15.28:0/3160542943 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e0075770 0x7f79e0075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.183+0000 7f79e7200700 1 --2- 172.21.15.28:0/3160542943 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f79e0076090 0x7f79e010fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.183+0000 7f79e7200700 1 -- 172.21.15.28:0/3160542943 >> 172.21.15.28:0/3160542943 conn(0x7f79e007a6a0 msgr2=0x7f79e007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:42.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.184+0000 7f79e7200700 1 -- 172.21.15.28:0/3160542943 shutdown_connections 2024-05-20T20:01:42.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.184+0000 7f79e7200700 1 -- 172.21.15.28:0/3160542943 wait complete. 2024-05-20T20:01:42.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.186+0000 7f79e7200700 1 Processor -- start 2024-05-20T20:01:42.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.186+0000 7f79e7200700 1 -- start start 2024-05-20T20:01:42.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e7200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f79e0075770 0x7f79e019bd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:42.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e7200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e0076090 0x7f79e019c270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:42.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e7200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e010ff40 0x7f79e01a0600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:42.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e7200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f79e01158c0 con 0x7f79e010ff40 2024-05-20T20:01:42.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e7200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f79e0115740 con 0x7f79e0075770 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e7200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f79e0115a40 con 0x7f79e0076090 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e5200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e010ff40 0x7f79e01a0600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79dfe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f79e0075770 0x7f79e019bd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79d7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e0076090 0x7f79e019c270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79dfe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f79e0075770 0x7f79e019bd30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42782/0 (socket says 172.21.15.28:42782) 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79e5200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e010ff40 0x7f79e01a0600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60714/0 (socket says 172.21.15.28:60714) 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79dfe00700 1 -- 172.21.15.28:0/2441612978 learned_addr learned my addr 172.21.15.28:0/2441612978 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:42.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.187+0000 7f79d7e00700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e0076090 0x7f79e019c270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54622/0 (socket says 172.21.15.28:54622) 2024-05-20T20:01:42.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79e5200700 1 -- 172.21.15.28:0/2441612978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e0076090 msgr2=0x7f79e019c270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:42.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79e5200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e0076090 0x7f79e019c270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79e5200700 1 -- 172.21.15.28:0/2441612978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f79e0075770 msgr2=0x7f79e019bd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:42.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79e5200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f79e0075770 0x7f79e019bd30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79e5200700 1 -- 172.21.15.28:0/2441612978 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f79e01a0d40 con 0x7f79e010ff40 2024-05-20T20:01:42.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79d7e00700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e0076090 0x7f79e019c270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:01:42.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79e5200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e010ff40 0x7f79e01a0600 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f79d800b070 tx=0x7f79d8004d00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:42.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79dd600700 1 -- 172.21.15.28:0/2441612978 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f79d8021070 con 0x7f79e010ff40 2024-05-20T20:01:42.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f79e01a0fd0 con 0x7f79e010ff40 2024-05-20T20:01:42.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.188+0000 7f79dd600700 1 -- 172.21.15.28:0/2441612978 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f79d8007e30 con 0x7f79e010ff40 2024-05-20T20:01:42.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.189+0000 7f79dd600700 1 -- 172.21.15.28:0/2441612978 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f79d80045d0 con 0x7f79e010ff40 2024-05-20T20:01:42.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.189+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f79e01a88b0 con 0x7f79e010ff40 2024-05-20T20:01:42.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.190+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f79e006a230 con 0x7f79e010ff40 2024-05-20T20:01:42.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.193+0000 7f79dd600700 1 -- 172.21.15.28:0/2441612978 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f79d8003c00 con 0x7f79e010ff40 2024-05-20T20:01:42.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.194+0000 7f79dd600700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f79c40b2880 0x7f79c40b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:42.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.194+0000 7f79dd600700 1 -- 172.21.15.28:0/2441612978 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7f79d800f020 con 0x7f79e010ff40 2024-05-20T20:01:42.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.194+0000 7f79dfe00700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f79c40b2880 0x7f79c40b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:42.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.195+0000 7f79dfe00700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f79c40b2880 0x7f79c40b4d40 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f79d0009d20 tx=0x7f79d0009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:42.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.197+0000 7f79dd600700 1 -- 172.21.15.28:0/2441612978 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f79d802a020 con 0x7f79e010ff40 2024-05-20T20:01:42.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.483+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"} v 0) -- 0x7f79e002e560 con 0x7f79e010ff40 2024-05-20T20:01:42.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.595+0000 7f79dd600700 1 -- 172.21.15.28:0/2441612978 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]=0 v18) ==== 105+0+0 (secure 0 0 0) 0x7f79d802e3e0 con 0x7f79e010ff40 2024-05-20T20:01:42.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f79c40b2880 msgr2=0x7f79c40b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:42.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f79c40b2880 0x7f79c40b4d40 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f79d0009d20 tx=0x7f79d0009260 comp rx=0 tx=0).stop 2024-05-20T20:01:42.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e010ff40 msgr2=0x7f79e01a0600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:42.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e010ff40 0x7f79e01a0600 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f79d800b070 tx=0x7f79d8004d00 comp rx=0 tx=0).stop 2024-05-20T20:01:42.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 shutdown_connections 2024-05-20T20:01:42.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f79c40b2880 0x7f79c40b4d40 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f79e010ff40 0x7f79e01a0600 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f79e0075770 0x7f79e019bd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 --2- 172.21.15.28:0/2441612978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f79e0076090 0x7f79e019c270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:42.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.599+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 >> 172.21.15.28:0/2441612978 conn(0x7f79e007a6a0 msgr2=0x7f79e01007b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:42.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.600+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 shutdown_connections 2024-05-20T20:01:42.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:42.600+0000 7f79e7200700 1 -- 172.21.15.28:0/2441612978 wait complete. 2024-05-20T20:01:42.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:42 smithi028 ceph-mon[31178]: pgmap v327: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 85 B/s wr, 0 op/s 2024-05-20T20:01:42.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:42 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2441612978' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]: dispatch 2024-05-20T20:01:42.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:42 smithi154 ceph-mon[36314]: pgmap v327: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 85 B/s wr, 0 op/s 2024-05-20T20:01:42.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:42 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2441612978' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]: dispatch 2024-05-20T20:01:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:42 smithi033 ceph-mon[36179]: pgmap v327: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 85 B/s wr, 0 op/s 2024-05-20T20:01:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:42 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2441612978' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]: dispatch 2024-05-20T20:01:43.493 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs set cephfs allow_standby_replay true 2024-05-20T20:01:43.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:43 smithi028 ceph-mon[31178]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-20T20:01:43.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:43 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2441612978' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]': finished 2024-05-20T20:01:43.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:43 smithi028 ceph-mon[31178]: daemon mds.h assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-05-20T20:01:43.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:43 smithi028 ceph-mon[31178]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:43.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:43 smithi028 ceph-mon[31178]: fsmap cephfs:2 {0=b=up:active,1=h=up:creating} 10 up:standby 2024-05-20T20:01:43.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:43 smithi028 ceph-mon[31178]: daemon mds.h is now active in filesystem cephfs as rank 1 2024-05-20T20:01:43.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:43 smithi154 ceph-mon[36314]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-20T20:01:43.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:43 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2441612978' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]': finished 2024-05-20T20:01:43.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:43 smithi154 ceph-mon[36314]: daemon mds.h assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-05-20T20:01:43.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:43 smithi154 ceph-mon[36314]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:43.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:43 smithi154 ceph-mon[36314]: fsmap cephfs:2 {0=b=up:active,1=h=up:creating} 10 up:standby 2024-05-20T20:01:43.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:43 smithi154 ceph-mon[36314]: daemon mds.h is now active in filesystem cephfs as rank 1 2024-05-20T20:01:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:43 smithi033 ceph-mon[36179]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-20T20:01:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:43 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2441612978' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]': finished 2024-05-20T20:01:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:43 smithi033 ceph-mon[36179]: daemon mds.h assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-05-20T20:01:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:43 smithi033 ceph-mon[36179]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-05-20T20:01:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:43 smithi033 ceph-mon[36179]: fsmap cephfs:2 {0=b=up:active,1=h=up:creating} 10 up:standby 2024-05-20T20:01:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:43 smithi033 ceph-mon[36179]: daemon mds.h is now active in filesystem cephfs as rank 1 2024-05-20T20:01:43.987 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:44 smithi028 ceph-mon[31178]: pgmap v328: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2024-05-20T20:01:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:44 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] up:active 2024-05-20T20:01:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:44 smithi028 ceph-mon[31178]: daemon mds.i assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-05-20T20:01:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:44 smithi028 ceph-mon[31178]: fsmap cephfs:2 {0=b=up:active,1=h=up:active} 10 up:standby 2024-05-20T20:01:44.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:44 smithi028 ceph-mon[31178]: fsmap cephfs:3 {0=b=up:active,1=h=up:active,2=i=up:creating} 9 up:standby 2024-05-20T20:01:44.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:44 smithi028 ceph-mon[31178]: daemon mds.i is now active in filesystem cephfs as rank 2 2024-05-20T20:01:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:44 smithi033 ceph-mon[36179]: pgmap v328: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2024-05-20T20:01:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:44 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] up:active 2024-05-20T20:01:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:44 smithi033 ceph-mon[36179]: daemon mds.i assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-05-20T20:01:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:44 smithi033 ceph-mon[36179]: fsmap cephfs:2 {0=b=up:active,1=h=up:active} 10 up:standby 2024-05-20T20:01:44.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:44 smithi033 ceph-mon[36179]: fsmap cephfs:3 {0=b=up:active,1=h=up:active,2=i=up:creating} 9 up:standby 2024-05-20T20:01:44.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:44 smithi033 ceph-mon[36179]: daemon mds.i is now active in filesystem cephfs as rank 2 2024-05-20T20:01:44.878 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:01:44 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: 2024-05-20T20:01:44.463+0000 7feffa946700 -1 mds.pinger is_rank_lagging: rank=1 was never sent ping request. 2024-05-20T20:01:45.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:44 smithi154 ceph-mon[36314]: pgmap v328: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2024-05-20T20:01:45.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:44 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] up:active 2024-05-20T20:01:45.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:44 smithi154 ceph-mon[36314]: daemon mds.i assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-05-20T20:01:45.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:44 smithi154 ceph-mon[36314]: fsmap cephfs:2 {0=b=up:active,1=h=up:active} 10 up:standby 2024-05-20T20:01:45.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:44 smithi154 ceph-mon[36314]: fsmap cephfs:3 {0=b=up:active,1=h=up:active,2=i=up:creating} 9 up:standby 2024-05-20T20:01:45.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:44 smithi154 ceph-mon[36314]: daemon mds.i is now active in filesystem cephfs as rank 2 2024-05-20T20:01:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:45 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] up:active 2024-05-20T20:01:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:45 smithi028 ceph-mon[31178]: daemon mds.e assigned to filesystem cephfs as rank 3 (now has 4 ranks) 2024-05-20T20:01:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:45 smithi028 ceph-mon[31178]: fsmap cephfs:3 {0=b=up:active,1=h=up:active,2=i=up:active} 9 up:standby 2024-05-20T20:01:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:45 smithi028 ceph-mon[31178]: fsmap cephfs:4 {0=b=up:active,1=h=up:active,2=i=up:active,3=e=up:creating} 8 up:standby 2024-05-20T20:01:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:45 smithi028 ceph-mon[31178]: daemon mds.e is now active in filesystem cephfs as rank 3 2024-05-20T20:01:46.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 -- 172.21.15.28:0/155548948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 msgr2=0x7f43bc075b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 --2- 172.21.15.28:0/155548948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc075b90 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f43ac009990 tx=0x7f43ac009ca0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 -- 172.21.15.28:0/155548948 shutdown_connections 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 --2- 172.21.15.28:0/155548948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43bc10ff40 0x7f43bc112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 --2- 172.21.15.28:0/155548948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc075b90 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 --2- 172.21.15.28:0/155548948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43bc0760d0 0x7f43bc10fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 -- 172.21.15.28:0/155548948 >> 172.21.15.28:0/155548948 conn(0x7f43bc07a730 msgr2=0x7f43bc07ab40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 -- 172.21.15.28:0/155548948 shutdown_connections 2024-05-20T20:01:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.075+0000 7f43c2200700 1 -- 172.21.15.28:0/155548948 wait complete. 2024-05-20T20:01:46.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.076+0000 7f43c2200700 1 Processor -- start 2024-05-20T20:01:46.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.076+0000 7f43c2200700 1 -- start start 2024-05-20T20:01:46.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43c2200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc19beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43c2200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43bc0760d0 0x7f43bc19c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43c2200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43bc10ff40 0x7f43bc1a0780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43c2200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f43bc1159a0 con 0x7f43bc0760d0 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43c2200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f43bc115820 con 0x7f43bc0757b0 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43c2200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f43bc115b20 con 0x7f43bc10ff40 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc19beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc19beb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42802/0 (socket says 172.21.15.28:42802) 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43baa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43bc0760d0 0x7f43bc19c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43bb400700 1 -- 172.21.15.28:0/3085948049 learned_addr learned my addr 172.21.15.28:0/3085948049 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:46.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.077+0000 7f43bbe00700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43bc10ff40 0x7f43bc1a0780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:46.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43bb400700 1 -- 172.21.15.28:0/3085948049 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43bc10ff40 msgr2=0x7f43bc1a0780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:46.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43bb400700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43bc10ff40 0x7f43bc1a0780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43bb400700 1 -- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43bc0760d0 msgr2=0x7f43bc19c3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:46.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43bb400700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43bc0760d0 0x7f43bc19c3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43bb400700 1 -- 172.21.15.28:0/3085948049 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f43bc1a0ec0 con 0x7f43bc0757b0 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43baa00700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43bc0760d0 0x7f43bc19c3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43bb400700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc19beb0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f43ac000c00 tx=0x7f43ac0057c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43b3e00700 1 -- 172.21.15.28:0/3085948049 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43ac027070 con 0x7f43bc0757b0 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43b3e00700 1 -- 172.21.15.28:0/3085948049 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43ac022470 con 0x7f43bc0757b0 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f43bc1a1150 con 0x7f43bc0757b0 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43b3e00700 1 -- 172.21.15.28:0/3085948049 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43ac017650 con 0x7f43bc0757b0 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.078+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f43bc1a8a30 con 0x7f43bc0757b0 2024-05-20T20:01:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.080+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f43bc06a230 con 0x7f43bc0757b0 2024-05-20T20:01:46.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.082+0000 7f43b3e00700 1 -- 172.21.15.28:0/3085948049 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7f43ac02a070 con 0x7f43bc0757b0 2024-05-20T20:01:46.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.083+0000 7f43b3e00700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f43a00b2900 0x7f43a00b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:46.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.084+0000 7f43b3e00700 1 -- 172.21.15.28:0/3085948049 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7f43ac0e2e40 con 0x7f43bc0757b0 2024-05-20T20:01:46.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.084+0000 7f43baa00700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f43a00b2900 0x7f43a00b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:46.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.084+0000 7f43baa00700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f43a00b2900 0x7f43a00b4dc0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f43bc1030b0 tx=0x7f43a8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:46.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.086+0000 7f43b3e00700 1 -- 172.21.15.28:0/3085948049 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f43ac013070 con 0x7f43bc0757b0 2024-05-20T20:01:46.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:45 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] up:active 2024-05-20T20:01:46.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:45 smithi154 ceph-mon[36314]: daemon mds.e assigned to filesystem cephfs as rank 3 (now has 4 ranks) 2024-05-20T20:01:46.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:45 smithi154 ceph-mon[36314]: fsmap cephfs:3 {0=b=up:active,1=h=up:active,2=i=up:active} 9 up:standby 2024-05-20T20:01:46.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:45 smithi154 ceph-mon[36314]: fsmap cephfs:4 {0=b=up:active,1=h=up:active,2=i=up:active,3=e=up:creating} 8 up:standby 2024-05-20T20:01:46.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:45 smithi154 ceph-mon[36314]: daemon mds.e is now active in filesystem cephfs as rank 3 2024-05-20T20:01:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:45 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] up:active 2024-05-20T20:01:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:45 smithi033 ceph-mon[36179]: daemon mds.e assigned to filesystem cephfs as rank 3 (now has 4 ranks) 2024-05-20T20:01:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:45 smithi033 ceph-mon[36179]: fsmap cephfs:3 {0=b=up:active,1=h=up:active,2=i=up:active} 9 up:standby 2024-05-20T20:01:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:45 smithi033 ceph-mon[36179]: fsmap cephfs:4 {0=b=up:active,1=h=up:active,2=i=up:active,3=e=up:creating} 8 up:standby 2024-05-20T20:01:46.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:45 smithi033 ceph-mon[36179]: daemon mds.e is now active in filesystem cephfs as rank 3 2024-05-20T20:01:46.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.384+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"} v 0) -- 0x7f43bc19d210 con 0x7f43bc0757b0 2024-05-20T20:01:46.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.652+0000 7f43b3e00700 1 -- 172.21.15.28:0/3085948049 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]=0 v26) ==== 121+0+0 (secure 0 0 0) 0x7f43ac0aaca0 con 0x7f43bc0757b0 2024-05-20T20:01:46.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f43a00b2900 msgr2=0x7f43a00b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:46.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f43a00b2900 0x7f43a00b4dc0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f43bc1030b0 tx=0x7f43a8009260 comp rx=0 tx=0).stop 2024-05-20T20:01:46.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 msgr2=0x7f43bc19beb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:46.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc19beb0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f43ac000c00 tx=0x7f43ac0057c0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 shutdown_connections 2024-05-20T20:01:46.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f43a00b2900 0x7f43a00b4dc0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43bc10ff40 0x7f43bc1a0780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43bc0757b0 0x7f43bc19beb0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 --2- 172.21.15.28:0/3085948049 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43bc0760d0 0x7f43bc19c3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:46.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.656+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 >> 172.21.15.28:0/3085948049 conn(0x7f43bc07a730 msgr2=0x7f43bc1007b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:46.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.657+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 shutdown_connections 2024-05-20T20:01:46.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:46.657+0000 7f43c2200700 1 -- 172.21.15.28:0/3085948049 wait complete. 2024-05-20T20:01:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: pgmap v329: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 852 B/s wr, 4 op/s 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] up:active 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: daemon mds.c assigned to filesystem cephfs as rank 4 (now has 5 ranks) 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: Cluster is now healthy 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: fsmap cephfs:4 {0=b=up:active,1=h=up:active,2=i=up:active,3=e=up:active} 8 up:standby 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: fsmap cephfs:5 1 up:creating 7 up:standby 4 up:active 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: daemon mds.c is now active in filesystem cephfs as rank 4 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3085948049' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-05-20T20:01:47.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:46 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-05-20T20:01:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: pgmap v329: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 852 B/s wr, 4 op/s 2024-05-20T20:01:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] up:active 2024-05-20T20:01:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: daemon mds.c assigned to filesystem cephfs as rank 4 (now has 5 ranks) 2024-05-20T20:01:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-20T20:01:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: Cluster is now healthy 2024-05-20T20:01:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: fsmap cephfs:4 {0=b=up:active,1=h=up:active,2=i=up:active,3=e=up:active} 8 up:standby 2024-05-20T20:01:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: fsmap cephfs:5 1 up:creating 7 up:standby 4 up:active 2024-05-20T20:01:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: daemon mds.c is now active in filesystem cephfs as rank 4 2024-05-20T20:01:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3085948049' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-05-20T20:01:47.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:46 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: pgmap v329: 97 pgs: 97 active+clean; 579 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 852 B/s wr, 4 op/s 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] up:active 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: daemon mds.c assigned to filesystem cephfs as rank 4 (now has 5 ranks) 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: Cluster is now healthy 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: fsmap cephfs:4 {0=b=up:active,1=h=up:active,2=i=up:active,3=e=up:active} 8 up:standby 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: fsmap cephfs:5 1 up:creating 7 up:standby 4 up:active 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: daemon mds.c is now active in filesystem cephfs as rank 4 2024-05-20T20:01:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3085948049' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-05-20T20:01:47.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:46 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-05-20T20:01:47.483 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs set cephfs session_timeout 300 2024-05-20T20:01:47.946 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:48.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:47 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-05-20T20:01:48.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:47 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] up:active 2024-05-20T20:01:48.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:47 smithi028 ceph-mon[31178]: fsmap cephfs:5 7 up:standby 5 up:active 2024-05-20T20:01:48.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:47 smithi028 ceph-mon[31178]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:01:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:47 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-05-20T20:01:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:47 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] up:active 2024-05-20T20:01:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:47 smithi154 ceph-mon[36314]: fsmap cephfs:5 7 up:standby 5 up:active 2024-05-20T20:01:48.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:47 smithi154 ceph-mon[36314]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:01:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:47 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-05-20T20:01:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:47 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] up:active 2024-05-20T20:01:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:47 smithi033 ceph-mon[36179]: fsmap cephfs:5 7 up:standby 5 up:active 2024-05-20T20:01:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:47 smithi033 ceph-mon[36179]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:01:48.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:48 smithi028 ceph-mon[31178]: pgmap v330: 97 pgs: 97 active+clean; 581 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 597 B/s rd, 1.7 KiB/s wr, 5 op/s 2024-05-20T20:01:49.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:48 smithi154 ceph-mon[36314]: pgmap v330: 97 pgs: 97 active+clean; 581 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 597 B/s rd, 1.7 KiB/s wr, 5 op/s 2024-05-20T20:01:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:48 smithi033 ceph-mon[36179]: pgmap v330: 97 pgs: 97 active+clean; 581 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 597 B/s rd, 1.7 KiB/s wr, 5 op/s 2024-05-20T20:01:50.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 -- 172.21.15.28:0/2849840194 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 msgr2=0x7fc6201103c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:50.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 --2- 172.21.15.28:0/2849840194 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 0x7fc6201103c0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fc61800b0a0 tx=0x7fc61800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 -- 172.21.15.28:0/2849840194 shutdown_connections 2024-05-20T20:01:50.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 --2- 172.21.15.28:0/2849840194 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 0x7fc6201103c0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 --2- 172.21.15.28:0/2849840194 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6201095a0 0x7fc620109980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 --2- 172.21.15.28:0/2849840194 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 0x7fc62006bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 -- 172.21.15.28:0/2849840194 >> 172.21.15.28:0/2849840194 conn(0x7fc6200793b0 msgr2=0x7fc6200797c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:50.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 -- 172.21.15.28:0/2849840194 shutdown_connections 2024-05-20T20:01:50.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.123+0000 7fc627800700 1 -- 172.21.15.28:0/2849840194 wait complete. 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.124+0000 7fc627800700 1 Processor -- start 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.124+0000 7fc627800700 1 -- start start 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.124+0000 7fc627800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 0x7fc62019bdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc627800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 0x7fc62019c300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc627800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6201095a0 0x7fc6201a0690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc627800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc620113720 con 0x7fc62006bab0 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc627800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc6201135a0 con 0x7fc62006c470 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc627800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc6201138a0 con 0x7fc6201095a0 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 0x7fc62019bdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 0x7fc62019bdc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:44628/0 (socket says 172.21.15.28:44628) 2024-05-20T20:01:50.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 -- 172.21.15.28:0/3704216100 learned_addr learned my addr 172.21.15.28:0/3704216100 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc617e00700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 0x7fc62019c300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc625800700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6201095a0 0x7fc6201a0690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 -- 172.21.15.28:0/3704216100 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6201095a0 msgr2=0x7fc6201a0690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6201095a0 0x7fc6201a0690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 -- 172.21.15.28:0/3704216100 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 msgr2=0x7fc62019c300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 0x7fc62019c300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.125+0000 7fc624e00700 1 -- 172.21.15.28:0/3704216100 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc6201a0dd0 con 0x7fc62006bab0 2024-05-20T20:01:50.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.126+0000 7fc624e00700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 0x7fc62019bdc0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fc61000e9c0 tx=0x7fc61000ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:50.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.126+0000 7fc61e000700 1 -- 172.21.15.28:0/3704216100 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc61000cbd0 con 0x7fc62006bab0 2024-05-20T20:01:50.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.126+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc6201a10c0 con 0x7fc62006bab0 2024-05-20T20:01:50.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.126+0000 7fc61e000700 1 -- 172.21.15.28:0/3704216100 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc61000ee60 con 0x7fc62006bab0 2024-05-20T20:01:50.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.126+0000 7fc61e000700 1 -- 172.21.15.28:0/3704216100 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc610005580 con 0x7fc62006bab0 2024-05-20T20:01:50.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.126+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc6201a8940 con 0x7fc62006bab0 2024-05-20T20:01:50.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.128+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc62006a230 con 0x7fc62006bab0 2024-05-20T20:01:50.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.130+0000 7fc61e000700 1 -- 172.21.15.28:0/3704216100 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fc610004750 con 0x7fc62006bab0 2024-05-20T20:01:50.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.131+0000 7fc61e000700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc6040b2880 0x7fc6040b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:50.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.131+0000 7fc61e000700 1 -- 172.21.15.28:0/3704216100 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7fc6100e24a0 con 0x7fc62006bab0 2024-05-20T20:01:50.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.132+0000 7fc617e00700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc6040b2880 0x7fc6040b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:50.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.132+0000 7fc617e00700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc6040b2880 0x7fc6040b4d40 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fc6201044f0 tx=0x7fc608009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:50.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.136+0000 7fc61e000700 1 -- 172.21.15.28:0/3704216100 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fc6100aa270 con 0x7fc62006bab0 2024-05-20T20:01:50.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.439+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"} v 0) -- 0x7fc6201a8b70 con 0x7fc62006bab0 2024-05-20T20:01:50.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.689+0000 7fc61e000700 1 -- 172.21.15.28:0/3704216100 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]=0 v28) ==== 115+0+0 (secure 0 0 0) 0x7fc6100cae20 con 0x7fc62006bab0 2024-05-20T20:01:50.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.696+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc6040b2880 msgr2=0x7fc6040b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:50.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.696+0000 7fc627800700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc6040b2880 0x7fc6040b4d40 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fc6201044f0 tx=0x7fc608009260 comp rx=0 tx=0).stop 2024-05-20T20:01:50.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.696+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 msgr2=0x7fc62019bdc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 0x7fc62019bdc0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fc61000e9c0 tx=0x7fc61000ecd0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 shutdown_connections 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc6040b2880 0x7fc6040b4d40 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6201095a0 0x7fc6201a0690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc62006bab0 0x7fc62019bdc0 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 --2- 172.21.15.28:0/3704216100 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc62006c470 0x7fc62019c300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 >> 172.21.15.28:0/3704216100 conn(0x7fc6200793b0 msgr2=0x7fc62010e710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 shutdown_connections 2024-05-20T20:01:50.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:50.697+0000 7fc627800700 1 -- 172.21.15.28:0/3704216100 wait complete. 2024-05-20T20:01:51.037 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:50 smithi028 ceph-mon[31178]: pgmap v331: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 3.9 KiB/s wr, 16 op/s 2024-05-20T20:01:51.037 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:50 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3704216100' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-05-20T20:01:51.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:50 smithi154 ceph-mon[36314]: pgmap v331: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 3.9 KiB/s wr, 16 op/s 2024-05-20T20:01:51.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:50 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3704216100' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-05-20T20:01:51.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:50 smithi033 ceph-mon[36179]: pgmap v331: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 3.9 KiB/s wr, 16 op/s 2024-05-20T20:01:51.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:50 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3704216100' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-05-20T20:01:51.500 DEBUG:tasks.cephfs.filesystem:Creating {'create': 2, 'subvol_options': '--namespace-isolated --size 25000000000'} subvols for filesystem 'cephfs' 2024-05-20T20:01:51.500 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs subvolumegroup create cephfs qa 2024-05-20T20:01:51.968 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:52.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:51 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3704216100' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-05-20T20:01:52.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:51 smithi028 ceph-mon[31178]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:01:52.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:51 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3704216100' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-05-20T20:01:52.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:51 smithi154 ceph-mon[36314]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:01:52.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:51 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3704216100' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-05-20T20:01:52.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:51 smithi033 ceph-mon[36179]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:01:52.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:52 smithi028 ceph-mon[31178]: pgmap v332: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 4.0 KiB/s wr, 17 op/s 2024-05-20T20:01:53.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:52 smithi154 ceph-mon[36314]: pgmap v332: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 4.0 KiB/s wr, 17 op/s 2024-05-20T20:01:53.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:52 smithi033 ceph-mon[36179]: pgmap v332: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 4.0 KiB/s wr, 17 op/s 2024-05-20T20:01:54.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.139+0000 7fba86a00700 1 -- 172.21.15.28:0/1031255344 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba80106120 msgr2=0x7fba80112630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:54.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.139+0000 7fba86a00700 1 --2- 172.21.15.28:0/1031255344 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba80106120 0x7fba80112630 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7fba7800b0a0 tx=0x7fba7800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.140+0000 7fba86a00700 1 -- 172.21.15.28:0/1031255344 shutdown_connections 2024-05-20T20:01:54.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.140+0000 7fba86a00700 1 --2- 172.21.15.28:0/1031255344 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba80106120 0x7fba80112630 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.140+0000 7fba86a00700 1 --2- 172.21.15.28:0/1031255344 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fba8010b760 0x7fba8010bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.140+0000 7fba86a00700 1 --2- 172.21.15.28:0/1031255344 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fba80105760 0x7fba80105be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.140+0000 7fba86a00700 1 -- 172.21.15.28:0/1031255344 >> 172.21.15.28:0/1031255344 conn(0x7fba80101250 msgr2=0x7fba80103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:54.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.140+0000 7fba86a00700 1 -- 172.21.15.28:0/1031255344 shutdown_connections 2024-05-20T20:01:54.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.140+0000 7fba86a00700 1 -- 172.21.15.28:0/1031255344 wait complete. 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.141+0000 7fba86a00700 1 Processor -- start 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.141+0000 7fba86a00700 1 -- start start 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.141+0000 7fba86a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fba80105760 0x7fba8019bd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.141+0000 7fba86a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fba80106120 0x7fba8019c2d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba86a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba8010b760 0x7fba801a0660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba86a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fba801159a0 con 0x7fba80106120 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba86a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fba80115820 con 0x7fba8010b760 2024-05-20T20:01:54.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba86a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fba80115b20 con 0x7fba80105760 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba7fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fba80105760 0x7fba8019bd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba77400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fba80106120 0x7fba8019c2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba84a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba8010b760 0x7fba801a0660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba7fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fba80105760 0x7fba8019bd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52494/0 (socket says 172.21.15.28:52494) 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba7fe00700 1 -- 172.21.15.28:0/2483884824 learned_addr learned my addr 172.21.15.28:0/2483884824 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba84a00700 1 -- 172.21.15.28:0/2483884824 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fba80105760 msgr2=0x7fba8019bd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba84a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fba80105760 0x7fba8019bd90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba84a00700 1 -- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fba80106120 msgr2=0x7fba8019c2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba84a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fba80106120 0x7fba8019c2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.142+0000 7fba84a00700 1 -- 172.21.15.28:0/2483884824 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fba801a0da0 con 0x7fba8010b760 2024-05-20T20:01:54.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.143+0000 7fba84a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba8010b760 0x7fba801a0660 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7fba7800b070 tx=0x7fba78004d00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:54.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.143+0000 7fba7d600700 1 -- 172.21.15.28:0/2483884824 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fba78021070 con 0x7fba8010b760 2024-05-20T20:01:54.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.143+0000 7fba7d600700 1 -- 172.21.15.28:0/2483884824 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fba78007e30 con 0x7fba8010b760 2024-05-20T20:01:54.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.143+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fba801a1030 con 0x7fba8010b760 2024-05-20T20:01:54.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.143+0000 7fba7d600700 1 -- 172.21.15.28:0/2483884824 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fba780045d0 con 0x7fba8010b760 2024-05-20T20:01:54.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.143+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fba801a8910 con 0x7fba8010b760 2024-05-20T20:01:54.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.145+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fba8010dcf0 con 0x7fba8010b760 2024-05-20T20:01:54.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.147+0000 7fba7d600700 1 -- 172.21.15.28:0/2483884824 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 16) ==== 149652+0+0 (secure 0 0 0) 0x7fba78003c00 con 0x7fba8010b760 2024-05-20T20:01:54.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.148+0000 7fba7d600700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fba600b2880 0x7fba600b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:54.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.148+0000 7fba7d600700 1 -- 172.21.15.28:0/2483884824 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7fba780e2f50 con 0x7fba8010b760 2024-05-20T20:01:54.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.148+0000 7fba7fe00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fba600b2880 0x7fba600b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:54.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.149+0000 7fba7fe00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fba600b2880 0x7fba600b4d40 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fba70009ce0 tx=0x7fba70009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:54.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.153+0000 7fba7d600700 1 -- 172.21.15.28:0/2483884824 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fba780aadb0 con 0x7fba8010b760 2024-05-20T20:01:54.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.430+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7fba80063980 con 0x7fba600b2880 2024-05-20T20:01:54.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.468+0000 7fba7d600700 1 -- 172.21.15.28:0/2483884824 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7fba80063980 con 0x7fba600b2880 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.471+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fba600b2880 msgr2=0x7fba600b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.471+0000 7fba86a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fba600b2880 0x7fba600b4d40 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fba70009ce0 tx=0x7fba70009260 comp rx=0 tx=0).stop 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba8010b760 msgr2=0x7fba801a0660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba8010b760 0x7fba801a0660 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7fba7800b070 tx=0x7fba78004d00 comp rx=0 tx=0).stop 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 shutdown_connections 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fba600b2880 0x7fba600b4d40 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fba8010b760 0x7fba801a0660 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fba80105760 0x7fba8019bd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 --2- 172.21.15.28:0/2483884824 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fba80106120 0x7fba8019c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:54.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 >> 172.21.15.28:0/2483884824 conn(0x7fba80101250 msgr2=0x7fba801019e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:54.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.472+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 shutdown_connections 2024-05-20T20:01:54.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:54.473+0000 7fba86a00700 1 -- 172.21.15.28:0/2483884824 wait complete. 2024-05-20T20:01:54.709 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:01:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:01:54.443+0000 7f7ba2c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:01:54.709 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:01:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:01:54.443+0000 7f7ba2c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:01:54.709 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:01:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:01:54.443+0000 7f7ba2c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:01:54.709 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:01:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:01:54.443+0000 7f7ba2c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:01:54.709 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:01:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:01:54.443+0000 7f7ba2c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:01:55.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:54 smithi028 ceph-mon[31178]: pgmap v333: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-05-20T20:01:55.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:54 smithi154 ceph-mon[36314]: pgmap v333: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-05-20T20:01:55.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:54 smithi033 ceph-mon[36179]: pgmap v333: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 4.0 KiB/s wr, 20 op/s 2024-05-20T20:01:55.236 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs subvolume create cephfs sv_0 --group_name qa 2024-05-20T20:01:55.698 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:01:56.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:55 smithi028 ceph-mon[31178]: from='client.24614 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:01:56.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:55 smithi154 ceph-mon[36314]: from='client.24614 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:01:56.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:55 smithi033 ceph-mon[36179]: from='client.24614 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:01:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:56 smithi028 ceph-mon[31178]: pgmap v334: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 4.1 KiB/s wr, 21 op/s 2024-05-20T20:01:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:56 smithi028 ceph-mon[31178]: mgrmap e17: x(active, since 8m), standbys: z, y 2024-05-20T20:01:57.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:56 smithi154 ceph-mon[36314]: pgmap v334: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 4.1 KiB/s wr, 21 op/s 2024-05-20T20:01:57.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:56 smithi154 ceph-mon[36314]: mgrmap e17: x(active, since 8m), standbys: z, y 2024-05-20T20:01:57.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:56 smithi033 ceph-mon[36179]: pgmap v334: 97 pgs: 97 active+clean; 584 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 4.1 KiB/s wr, 21 op/s 2024-05-20T20:01:57.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:56 smithi033 ceph-mon[36179]: mgrmap e17: x(active, since 8m), standbys: z, y 2024-05-20T20:01:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.017+0000 7f512d800700 1 -- 172.21.15.28:0/3788223740 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f512810b760 msgr2=0x7f512810bb40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.017+0000 7f512d800700 1 --2- 172.21.15.28:0/3788223740 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f512810b760 0x7f512810bb40 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f5118009990 tx=0x7f5118009ca0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.017+0000 7f512d800700 1 -- 172.21.15.28:0/3788223740 shutdown_connections 2024-05-20T20:01:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.018+0000 7f512d800700 1 --2- 172.21.15.28:0/3788223740 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5128106120 0x7f5128112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.018+0000 7f512d800700 1 --2- 172.21.15.28:0/3788223740 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f512810b760 0x7f512810bb40 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.018+0000 7f512d800700 1 --2- 172.21.15.28:0/3788223740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5128105760 0x7f5128105be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.018+0000 7f512d800700 1 -- 172.21.15.28:0/3788223740 >> 172.21.15.28:0/3788223740 conn(0x7f5128101250 msgr2=0x7f5128103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.018+0000 7f512d800700 1 -- 172.21.15.28:0/3788223740 shutdown_connections 2024-05-20T20:01:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.018+0000 7f512d800700 1 -- 172.21.15.28:0/3788223740 wait complete. 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.019+0000 7f512d800700 1 Processor -- start 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.019+0000 7f512d800700 1 -- start start 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.019+0000 7f512d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5128105760 0x7f512819c070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.019+0000 7f512d800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5128106120 0x7f512819c5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f512d800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51281a0940 0x7f51281a0dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f512d800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5128115990 con 0x7f5128105760 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f512d800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5128115810 con 0x7f5128106120 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f512d800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5128115b10 con 0x7f51281a0940 2024-05-20T20:01:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5128105760 0x7f512819c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5128105760 0x7f512819c070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59130/0 (socket says 172.21.15.28:59130) 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 -- 172.21.15.28:0/591618754 learned_addr learned my addr 172.21.15.28:0/591618754 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5127400700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51281a0940 0x7f51281a0dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126000700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5128106120 0x7f512819c5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 -- 172.21.15.28:0/591618754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51281a0940 msgr2=0x7f51281a0dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51281a0940 0x7f51281a0dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 -- 172.21.15.28:0/591618754 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5128106120 msgr2=0x7f512819c5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5128106120 0x7f512819c5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5126a00700 1 -- 172.21.15.28:0/591618754 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51281a1460 con 0x7f5128105760 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.020+0000 7f5127400700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51281a0940 0x7f51281a0dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:01:58.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.021+0000 7f5126a00700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5128105760 0x7f512819c070 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f51180098e0 tx=0x7f51180057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:58.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.021+0000 7f511f400700 1 -- 172.21.15.28:0/591618754 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5118026070 con 0x7f5128105760 2024-05-20T20:01:58.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.021+0000 7f511f400700 1 -- 172.21.15.28:0/591618754 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5118021410 con 0x7f5128105760 2024-05-20T20:01:58.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.021+0000 7f511f400700 1 -- 172.21.15.28:0/591618754 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5118017650 con 0x7f5128105760 2024-05-20T20:01:58.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.021+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f51281a16f0 con 0x7f5128105760 2024-05-20T20:01:58.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.021+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f51281a8dc0 con 0x7f5128105760 2024-05-20T20:01:58.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.023+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f512806a230 con 0x7f5128105760 2024-05-20T20:01:58.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.025+0000 7f511f400700 1 -- 172.21.15.28:0/591618754 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f5118029070 con 0x7f5128105760 2024-05-20T20:01:58.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.027+0000 7f511f400700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f510c0b2990 0x7f510c0b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:01:58.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.027+0000 7f511f400700 1 -- 172.21.15.28:0/591618754 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7f51180e2bc0 con 0x7f5128105760 2024-05-20T20:01:58.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.027+0000 7f5126000700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f510c0b2990 0x7f510c0b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:01:58.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.028+0000 7f5126000700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f510c0b2990 0x7f510c0b4e50 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f51281039c0 tx=0x7f5114009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:01:58.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.031+0000 7f511f400700 1 -- 172.21.15.28:0/591618754 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f5118013070 con 0x7f5128105760 2024-05-20T20:01:58.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.304+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7f5128063a10 con 0x7f510c0b2990 2024-05-20T20:01:58.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.357+0000 7f511f400700 1 -- 172.21.15.28:0/591618754 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7f5128063a10 con 0x7f510c0b2990 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.360+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f510c0b2990 msgr2=0x7f510c0b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.360+0000 7f512d800700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f510c0b2990 0x7f510c0b4e50 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f51281039c0 tx=0x7f5114009260 comp rx=0 tx=0).stop 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.360+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5128105760 msgr2=0x7f512819c070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.360+0000 7f512d800700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5128105760 0x7f512819c070 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f51180098e0 tx=0x7f51180057c0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 shutdown_connections 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f510c0b2990 0x7f510c0b4e50 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51281a0940 0x7f51281a0dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5128105760 0x7f512819c070 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 --2- 172.21.15.28:0/591618754 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5128106120 0x7f512819c5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:01:58.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 >> 172.21.15.28:0/591618754 conn(0x7f5128101250 msgr2=0x7f5128101b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:01:58.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 shutdown_connections 2024-05-20T20:01:58.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:01:58.361+0000 7f512d800700 1 -- 172.21.15.28:0/591618754 wait complete. 2024-05-20T20:01:59.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:58 smithi028 ceph-mon[31178]: pgmap v335: 97 pgs: 97 active+clean; 585 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 3.3 KiB/s wr, 17 op/s 2024-05-20T20:01:59.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:58 smithi154 ceph-mon[36314]: pgmap v335: 97 pgs: 97 active+clean; 585 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 3.3 KiB/s wr, 17 op/s 2024-05-20T20:01:59.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:58 smithi033 ceph-mon[36179]: pgmap v335: 97 pgs: 97 active+clean; 585 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 3.3 KiB/s wr, 17 op/s 2024-05-20T20:01:59.196 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs subvolume getpath cephfs --group_name qa sv_0 2024-05-20T20:01:59.662 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:00.006 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:01:59 smithi028 ceph-mon[31178]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:00.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:01:59 smithi154 ceph-mon[36314]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:00.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:01:59 smithi033 ceph-mon[36179]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:00.958 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:00 smithi028 ceph-mon[31178]: pgmap v336: 97 pgs: 97 active+clean; 591 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-05-20T20:02:01.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:00 smithi154 ceph-mon[36314]: pgmap v336: 97 pgs: 97 active+clean; 591 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-05-20T20:02:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:00 smithi033 ceph-mon[36179]: pgmap v336: 97 pgs: 97 active+clean; 591 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 3.2 KiB/s wr, 20 op/s 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 -- 172.21.15.28:0/597654208 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301037f0 msgr2=0x7fb330103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 --2- 172.21.15.28:0/597654208 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301037f0 0x7fb330103bd0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fb3240099c0 tx=0x7fb324009cd0 comp rx=0 tx=0).stop 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 -- 172.21.15.28:0/597654208 shutdown_connections 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 --2- 172.21.15.28:0/597654208 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3301088f0 0x7fb330110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 --2- 172.21.15.28:0/597654208 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301037f0 0x7fb330103bd0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 --2- 172.21.15.28:0/597654208 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3301041a0 0x7fb3301081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 -- 172.21.15.28:0/597654208 >> 172.21.15.28:0/597654208 conn(0x7fb33007a6a0 msgr2=0x7fb33007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:01.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 -- 172.21.15.28:0/597654208 shutdown_connections 2024-05-20T20:02:01.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.832+0000 7fb338800700 1 -- 172.21.15.28:0/597654208 wait complete. 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.834+0000 7fb338800700 1 Processor -- start 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.834+0000 7fb338800700 1 -- start start 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.834+0000 7fb338800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3301037f0 0x7fb33019bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.834+0000 7fb338800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301041a0 0x7fb33019c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.834+0000 7fb338800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3301088f0 0x7fb3301a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.834+0000 7fb338800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb330113710 con 0x7fb3301088f0 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb338800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb330113590 con 0x7fb3301041a0 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb338800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb330113890 con 0x7fb3301037f0 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301041a0 0x7fb33019c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3301037f0 0x7fb33019bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:01.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301041a0 0x7fb33019c310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48780/0 (socket says 172.21.15.28:48780) 2024-05-20T20:02:01.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 -- 172.21.15.28:0/3905377033 learned_addr learned my addr 172.21.15.28:0/3905377033 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:01.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb336800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3301088f0 0x7fb3301a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 -- 172.21.15.28:0/3905377033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3301037f0 msgr2=0x7fb33019bdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3301037f0 0x7fb33019bdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 -- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3301088f0 msgr2=0x7fb3301a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3301088f0 0x7fb3301a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335400700 1 -- 172.21.15.28:0/3905377033 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb3301a0de0 con 0x7fb3301041a0 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb335e00700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3301037f0 0x7fb33019bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.835+0000 7fb336800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3301088f0 0x7fb3301a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.836+0000 7fb335400700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301041a0 0x7fb33019c310 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fb32000e9a0 tx=0x7fb32000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.836+0000 7fb32aa00700 1 -- 172.21.15.28:0/3905377033 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb32000ca10 con 0x7fb3301041a0 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.836+0000 7fb32aa00700 1 -- 172.21.15.28:0/3905377033 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb32000cbb0 con 0x7fb3301041a0 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.836+0000 7fb32aa00700 1 -- 172.21.15.28:0/3905377033 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb320018680 con 0x7fb3301041a0 2024-05-20T20:02:01.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.836+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb3301a10d0 con 0x7fb3301041a0 2024-05-20T20:02:01.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.836+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb3301a8950 con 0x7fb3301041a0 2024-05-20T20:02:01.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.837+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb33006a230 con 0x7fb3301041a0 2024-05-20T20:02:01.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.840+0000 7fb32aa00700 1 -- 172.21.15.28:0/3905377033 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fb320010430 con 0x7fb3301041a0 2024-05-20T20:02:01.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.841+0000 7fb32aa00700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3180b2990 0x7fb3180b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:01.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.841+0000 7fb32aa00700 1 -- 172.21.15.28:0/3905377033 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(78..78 src has 1..78) ==== 7979+0+0 (secure 0 0 0) 0x7fb3200e23e0 con 0x7fb3301041a0 2024-05-20T20:02:01.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.841+0000 7fb335e00700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3180b2990 0x7fb3180b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:01.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.842+0000 7fb335e00700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3180b2990 0x7fb3180b4e50 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fb324003760 tx=0x7fb324005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:01.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:01.846+0000 7fb32aa00700 1 -- 172.21.15.28:0/3905377033 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb3200aa050 con 0x7fb3301041a0 2024-05-20T20:02:02.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.123+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}) -- 0x7fb33010f7d0 con 0x7fb3180b2990 2024-05-20T20:02:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.135+0000 7fb32aa00700 1 -- 172.21.15.28:0/3905377033 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+54 (secure 0 0 0) 0x7fb33010f7d0 con 0x7fb3180b2990 2024-05-20T20:02:02.137 INFO:teuthology.orchestra.run.smithi028.stdout:/volumes/qa/sv_0/7e702041-c325-42f0-89b5-be58c2b98a26 2024-05-20T20:02:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3180b2990 msgr2=0x7fb3180b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3180b2990 0x7fb3180b4e50 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fb324003760 tx=0x7fb324005fb0 comp rx=0 tx=0).stop 2024-05-20T20:02:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301041a0 msgr2=0x7fb33019c310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301041a0 0x7fb33019c310 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fb32000e9a0 tx=0x7fb32000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:02:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 shutdown_connections 2024-05-20T20:02:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb3180b2990 0x7fb3180b4e50 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3301088f0 0x7fb3301a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3301037f0 0x7fb33019bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 --2- 172.21.15.28:0/3905377033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3301041a0 0x7fb33019c310 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 >> 172.21.15.28:0/3905377033 conn(0x7fb33007a6a0 msgr2=0x7fb3301018f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 shutdown_connections 2024-05-20T20:02:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:02.139+0000 7fb338800700 1 -- 172.21.15.28:0/3905377033 wait complete. 2024-05-20T20:02:02.926 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:02 smithi028 ceph-mon[31178]: pgmap v337: 97 pgs: 97 active+clean; 591 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 1.1 KiB/s wr, 9 op/s 2024-05-20T20:02:03.001 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs subvolume create cephfs sv_1 --group_name qa 2024-05-20T20:02:03.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:02 smithi154 ceph-mon[36314]: pgmap v337: 97 pgs: 97 active+clean; 591 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 1.1 KiB/s wr, 9 op/s 2024-05-20T20:02:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:02 smithi033 ceph-mon[36179]: pgmap v337: 97 pgs: 97 active+clean; 591 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 1.1 KiB/s wr, 9 op/s 2024-05-20T20:02:03.479 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:03.889 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:03.889 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]: dispatch 2024-05-20T20:02:03.889 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]: dispatch 2024-05-20T20:02:03.889 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [2, 11]}]: dispatch 2024-05-20T20:02:03.889 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 6]}]: dispatch 2024-05-20T20:02:03.889 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [5, 7]}]: dispatch 2024-05-20T20:02:03.889 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [5, 9]}]: dispatch 2024-05-20T20:02:03.890 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [3, 9]}]: dispatch 2024-05-20T20:02:03.890 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [4, 9]}]: dispatch 2024-05-20T20:02:03.890 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:03 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 8]}]: dispatch 2024-05-20T20:02:04.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:04.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [2, 11]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 6]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [5, 7]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [5, 9]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [3, 9]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [4, 9]}]: dispatch 2024-05-20T20:02:04.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:03 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 8]}]: dispatch 2024-05-20T20:02:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='client.24632 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]: dispatch 2024-05-20T20:02:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]: dispatch 2024-05-20T20:02:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [2, 11]}]: dispatch 2024-05-20T20:02:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 6]}]: dispatch 2024-05-20T20:02:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [5, 7]}]: dispatch 2024-05-20T20:02:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [5, 9]}]: dispatch 2024-05-20T20:02:04.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [3, 9]}]: dispatch 2024-05-20T20:02:04.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [4, 9]}]: dispatch 2024-05-20T20:02:04.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:03 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 8]}]: dispatch 2024-05-20T20:02:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: pgmap v338: 97 pgs: 97 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2024-05-20T20:02:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]': finished 2024-05-20T20:02:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]': finished 2024-05-20T20:02:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [2, 11]}]': finished 2024-05-20T20:02:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 6]}]': finished 2024-05-20T20:02:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [5, 7]}]': finished 2024-05-20T20:02:04.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [5, 9]}]': finished 2024-05-20T20:02:04.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [3, 9]}]': finished 2024-05-20T20:02:04.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [4, 9]}]': finished 2024-05-20T20:02:04.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 8]}]': finished 2024-05-20T20:02:04.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:04 smithi028 ceph-mon[31178]: osdmap e79: 12 total, 12 up, 12 in 2024-05-20T20:02:05.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: pgmap v338: 97 pgs: 97 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [2, 11]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 6]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [5, 7]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [5, 9]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [3, 9]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [4, 9]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 8]}]': finished 2024-05-20T20:02:05.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:04 smithi154 ceph-mon[36314]: osdmap e79: 12 total, 12 up, 12 in 2024-05-20T20:02:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: pgmap v338: 97 pgs: 97 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2024-05-20T20:02:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]': finished 2024-05-20T20:02:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.10", "id": [2, 4]}]': finished 2024-05-20T20:02:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.13", "id": [2, 11]}]': finished 2024-05-20T20:02:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 6]}]': finished 2024-05-20T20:02:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [5, 7]}]': finished 2024-05-20T20:02:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [5, 9]}]': finished 2024-05-20T20:02:05.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.22", "id": [3, 9]}]': finished 2024-05-20T20:02:05.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.36", "id": [4, 9]}]': finished 2024-05-20T20:02:05.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3a", "id": [4, 8]}]': finished 2024-05-20T20:02:05.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:04 smithi033 ceph-mon[36179]: osdmap e79: 12 total, 12 up, 12 in 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 -- 172.21.15.28:0/490619484 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb520114480 msgr2=0x7fb520116870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 --2- 172.21.15.28:0/490619484 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb520114480 0x7fb520116870 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fb51800b0a0 tx=0x7fb51800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 -- 172.21.15.28:0/490619484 shutdown_connections 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 --2- 172.21.15.28:0/490619484 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb520114480 0x7fb520116870 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 --2- 172.21.15.28:0/490619484 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5200758e0 0x7fb520075cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 --2- 172.21.15.28:0/490619484 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb520076200 0x7fb520113f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 -- 172.21.15.28:0/490619484 >> 172.21.15.28:0/490619484 conn(0x7fb5200fefc0 msgr2=0x7fb5201013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:05.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 -- 172.21.15.28:0/490619484 shutdown_connections 2024-05-20T20:02:05.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.795+0000 7fb51f200700 1 -- 172.21.15.28:0/490619484 wait complete. 2024-05-20T20:02:05.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.797+0000 7fb51f200700 1 Processor -- start 2024-05-20T20:02:05.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.797+0000 7fb51f200700 1 -- start start 2024-05-20T20:02:05.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51f200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5200758e0 0x7fb5201a0120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:05.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51f200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb520076200 0x7fb5201a0660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:05.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51f200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb520114480 0x7fb5201a49f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:05.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51f200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb520119c70 con 0x7fb520076200 2024-05-20T20:02:05.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51f200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb520119af0 con 0x7fb5200758e0 2024-05-20T20:02:05.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51f200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb520119df0 con 0x7fb520114480 2024-05-20T20:02:05.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5200758e0 0x7fb5201a0120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:05.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb520076200 0x7fb5201a0660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:05.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5200758e0 0x7fb5201a0120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48802/0 (socket says 172.21.15.28:48802) 2024-05-20T20:02:05.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51de00700 1 -- 172.21.15.28:0/3280014193 learned_addr learned my addr 172.21.15.28:0/3280014193 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:05.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51e800700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb520114480 0x7fb5201a49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.798+0000 7fb51de00700 1 -- 172.21.15.28:0/3280014193 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb520114480 msgr2=0x7fb5201a49f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51de00700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb520114480 0x7fb5201a49f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51de00700 1 -- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb520076200 msgr2=0x7fb5201a0660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51de00700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb520076200 0x7fb5201a0660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51de00700 1 -- 172.21.15.28:0/3280014193 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb5201a5130 con 0x7fb5200758e0 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51e800700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb520114480 0x7fb5201a49f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51de00700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5200758e0 0x7fb5201a0120 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fb51000e9a0 tx=0x7fb51000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb516a00700 1 -- 172.21.15.28:0/3280014193 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb51000ca10 con 0x7fb5200758e0 2024-05-20T20:02:05.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb516a00700 1 -- 172.21.15.28:0/3280014193 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb51000cbb0 con 0x7fb5200758e0 2024-05-20T20:02:05.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb5201a5420 con 0x7fb5200758e0 2024-05-20T20:02:05.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb516a00700 1 -- 172.21.15.28:0/3280014193 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb510018680 con 0x7fb5200758e0 2024-05-20T20:02:05.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.799+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb5201acca0 con 0x7fb5200758e0 2024-05-20T20:02:05.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.800+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb52006a230 con 0x7fb5200758e0 2024-05-20T20:02:05.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.802+0000 7fb516a00700 1 -- 172.21.15.28:0/3280014193 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fb510010430 con 0x7fb5200758e0 2024-05-20T20:02:05.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.804+0000 7fb516a00700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5040b2990 0x7fb5040b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:05.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.804+0000 7fb516a00700 1 -- 172.21.15.28:0/3280014193 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fb5100e24d0 con 0x7fb5200758e0 2024-05-20T20:02:05.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.804+0000 7fb51d400700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5040b2990 0x7fb5040b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:05.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.805+0000 7fb51d400700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5040b2990 0x7fb5040b4e50 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fb50c009680 tx=0x7fb50c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:05.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:05.808+0000 7fb516a00700 1 -- 172.21.15.28:0/3280014193 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb5100aa050 con 0x7fb5200758e0 2024-05-20T20:02:06.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:05 smithi028 ceph-mon[31178]: osdmap e80: 12 total, 12 up, 12 in 2024-05-20T20:02:06.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.082+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}) -- 0x7fb520063980 con 0x7fb5040b2990 2024-05-20T20:02:06.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.120+0000 7fb516a00700 1 -- 172.21.15.28:0/3280014193 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+0 (secure 0 0 0) 0x7fb520063980 con 0x7fb5040b2990 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5040b2990 msgr2=0x7fb5040b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5040b2990 0x7fb5040b4e50 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fb50c009680 tx=0x7fb50c009260 comp rx=0 tx=0).stop 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5200758e0 msgr2=0x7fb5201a0120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5200758e0 0x7fb5201a0120 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fb51000e9a0 tx=0x7fb51000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 shutdown_connections 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5040b2990 0x7fb5040b4e50 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb520114480 0x7fb5201a49f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5200758e0 0x7fb5201a0120 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 --2- 172.21.15.28:0/3280014193 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb520076200 0x7fb5201a0660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 >> 172.21.15.28:0/3280014193 conn(0x7fb5200fefc0 msgr2=0x7fb520100820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 shutdown_connections 2024-05-20T20:02:06.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:06.123+0000 7fb51f200700 1 -- 172.21.15.28:0/3280014193 wait complete. 2024-05-20T20:02:06.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:05 smithi154 ceph-mon[36314]: osdmap e80: 12 total, 12 up, 12 in 2024-05-20T20:02:06.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:05 smithi033 ceph-mon[36179]: osdmap e80: 12 total, 12 up, 12 in 2024-05-20T20:02:06.932 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs subvolume getpath cephfs --group_name qa sv_1 2024-05-20T20:02:07.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:06 smithi028 ceph-mon[31178]: pgmap v341: 97 pgs: 4 peering, 93 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 3.2 KiB/s wr, 13 op/s 2024-05-20T20:02:07.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:06 smithi154 ceph-mon[36314]: pgmap v341: 97 pgs: 4 peering, 93 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 3.2 KiB/s wr, 13 op/s 2024-05-20T20:02:07.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:06 smithi033 ceph-mon[36179]: pgmap v341: 97 pgs: 4 peering, 93 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 3.2 KiB/s wr, 13 op/s 2024-05-20T20:02:07.369 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:08.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:07 smithi028 ceph-mon[31178]: from='client.24638 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:08.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:07 smithi154 ceph-mon[36314]: from='client.24638 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:08.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:07 smithi033 ceph-mon[36179]: from='client.24638 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:09.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:08 smithi028 ceph-mon[31178]: pgmap v342: 97 pgs: 4 peering, 93 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s; 8 B/s, 1 keys/s, 0 objects/s recovering 2024-05-20T20:02:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:08 smithi154 ceph-mon[36314]: pgmap v342: 97 pgs: 4 peering, 93 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s; 8 B/s, 1 keys/s, 0 objects/s recovering 2024-05-20T20:02:09.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:08 smithi033 ceph-mon[36179]: pgmap v342: 97 pgs: 4 peering, 93 active+clean; 605 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s; 8 B/s, 1 keys/s, 0 objects/s recovering 2024-05-20T20:02:09.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 -- 172.21.15.28:0/3941999387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 msgr2=0x7f727010bb40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:09.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 --2- 172.21.15.28:0/3941999387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 0x7f727010bb40 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f7264009990 tx=0x7f7264009ca0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 -- 172.21.15.28:0/3941999387 shutdown_connections 2024-05-20T20:02:09.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 --2- 172.21.15.28:0/3941999387 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270106120 0x7f7270112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 --2- 172.21.15.28:0/3941999387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 0x7f727010bb40 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 --2- 172.21.15.28:0/3941999387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7270105760 0x7f7270105be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 -- 172.21.15.28:0/3941999387 >> 172.21.15.28:0/3941999387 conn(0x7f7270101250 msgr2=0x7f7270103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:09.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 -- 172.21.15.28:0/3941999387 shutdown_connections 2024-05-20T20:02:09.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.613+0000 7f7277400700 1 -- 172.21.15.28:0/3941999387 wait complete. 2024-05-20T20:02:09.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.614+0000 7f7277400700 1 Processor -- start 2024-05-20T20:02:09.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7277400700 1 -- start start 2024-05-20T20:02:09.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7277400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270105760 0x7f727019be50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:09.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7277400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7270106120 0x7f727019c390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:09.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7277400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 0x7f72701a06d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:09.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7277400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7270115960 con 0x7f727010b760 2024-05-20T20:02:09.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7277400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f72701157e0 con 0x7f7270105760 2024-05-20T20:02:09.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7277400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7270115ae0 con 0x7f7270106120 2024-05-20T20:02:09.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7274a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270105760 0x7f727019be50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:09.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f726fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7270106120 0x7f727019c390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:09.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7275400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 0x7f72701a06d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:09.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7274a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270105760 0x7f727019be50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58354/0 (socket says 172.21.15.28:58354) 2024-05-20T20:02:09.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.615+0000 7f7274a00700 1 -- 172.21.15.28:0/2948766807 learned_addr learned my addr 172.21.15.28:0/2948766807 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:09.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.616+0000 7f7274a00700 1 -- 172.21.15.28:0/2948766807 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7270106120 msgr2=0x7f727019c390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:09.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.616+0000 7f7274a00700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7270106120 0x7f727019c390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.616+0000 7f7274a00700 1 -- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 msgr2=0x7f72701a06d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:09.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.616+0000 7f7274a00700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 0x7f72701a06d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.616+0000 7f7274a00700 1 -- 172.21.15.28:0/2948766807 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f72701a0e10 con 0x7f7270105760 2024-05-20T20:02:09.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.616+0000 7f7275400700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 0x7f72701a06d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:09.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.616+0000 7f7274a00700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270105760 0x7f727019be50 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f7264000c00 tx=0x7f7264009580 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:09.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.617+0000 7f726d600700 1 -- 172.21.15.28:0/2948766807 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7264027070 con 0x7f7270105760 2024-05-20T20:02:09.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.617+0000 7f726d600700 1 -- 172.21.15.28:0/2948766807 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7264022470 con 0x7f7270105760 2024-05-20T20:02:09.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.617+0000 7f726d600700 1 -- 172.21.15.28:0/2948766807 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7264017650 con 0x7f7270105760 2024-05-20T20:02:09.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.617+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f72701a10a0 con 0x7f7270105760 2024-05-20T20:02:09.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.617+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f72701a8980 con 0x7f7270105760 2024-05-20T20:02:09.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.618+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f727010dcf0 con 0x7f7270105760 2024-05-20T20:02:09.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.620+0000 7f726d600700 1 -- 172.21.15.28:0/2948766807 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f726402a070 con 0x7f7270105760 2024-05-20T20:02:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.622+0000 7f726d600700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f72580b2990 0x7f72580b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.622+0000 7f726d600700 1 -- 172.21.15.28:0/2948766807 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f72640e32f0 con 0x7f7270105760 2024-05-20T20:02:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.622+0000 7f726fe00700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f72580b2990 0x7f72580b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:09.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.623+0000 7f726fe00700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f72580b2990 0x7f72580b4e50 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f7270103930 tx=0x7f725c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:09.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.625+0000 7f726d600700 1 -- 172.21.15.28:0/2948766807 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7264013070 con 0x7f7270105760 2024-05-20T20:02:09.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.906+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}) -- 0x7f727019d160 con 0x7f72580b2990 2024-05-20T20:02:09.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.914+0000 7f726d600700 1 -- 172.21.15.28:0/2948766807 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+54 (secure 0 0 0) 0x7f727019d160 con 0x7f72580b2990 2024-05-20T20:02:09.915 INFO:teuthology.orchestra.run.smithi028.stdout:/volumes/qa/sv_1/9e9b2651-9ad5-4ff0-99f5-cbf91c762d93 2024-05-20T20:02:09.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f72580b2990 msgr2=0x7f72580b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:09.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f72580b2990 0x7f72580b4e50 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f7270103930 tx=0x7f725c009260 comp rx=0 tx=0).stop 2024-05-20T20:02:09.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270105760 msgr2=0x7f727019be50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:09.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270105760 0x7f727019be50 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f7264000c00 tx=0x7f7264009580 comp rx=0 tx=0).stop 2024-05-20T20:02:09.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 shutdown_connections 2024-05-20T20:02:09.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f72580b2990 0x7f72580b4e50 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f727010b760 0x7f72701a06d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7270105760 0x7f727019be50 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 --2- 172.21.15.28:0/2948766807 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7270106120 0x7f727019c390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:09.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 >> 172.21.15.28:0/2948766807 conn(0x7f7270101250 msgr2=0x7f7270101aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:09.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 shutdown_connections 2024-05-20T20:02:09.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:09.918+0000 7f7277400700 1 -- 172.21.15.28:0/2948766807 wait complete. 2024-05-20T20:02:10.826 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:02:11.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:10 smithi028 ceph-mon[31178]: pgmap v343: 97 pgs: 2 peering, 95 active+clean; 625 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 4.7 KiB/s wr, 14 op/s; 22 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:10 smithi154 ceph-mon[36314]: pgmap v343: 97 pgs: 2 peering, 95 active+clean; 625 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 4.7 KiB/s wr, 14 op/s; 22 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:11.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:10 smithi033 ceph-mon[36179]: pgmap v343: 97 pgs: 2 peering, 95 active+clean; 625 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 4.7 KiB/s wr, 14 op/s; 22 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:11.315 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:12.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:11 smithi028 ceph-mon[31178]: from='client.24644 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:11 smithi154 ceph-mon[36314]: from='client.24644 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:12.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:11 smithi033 ceph-mon[36179]: from='client.24644 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:02:13.049 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:12 smithi028 ceph-mon[31178]: pgmap v344: 97 pgs: 97 active+clean; 625 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 3.1 KiB/s wr, 9 op/s; 22 B/s, 1 keys/s, 2 objects/s recovering 2024-05-20T20:02:13.049 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:12 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:02:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:12 smithi154 ceph-mon[36314]: pgmap v344: 97 pgs: 97 active+clean; 625 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 3.1 KiB/s wr, 9 op/s; 22 B/s, 1 keys/s, 2 objects/s recovering 2024-05-20T20:02:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:12 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:02:13.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:12 smithi033 ceph-mon[36179]: pgmap v344: 97 pgs: 97 active+clean; 625 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 3.1 KiB/s wr, 9 op/s; 22 B/s, 1 keys/s, 2 objects/s recovering 2024-05-20T20:02:13.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:12 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 -- 172.21.15.28:0/1547579221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501088f0 msgr2=0x7fa350110470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 --2- 172.21.15.28:0/1547579221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501088f0 0x7fa350110470 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fa34800b0d0 tx=0x7fa34800b3e0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 -- 172.21.15.28:0/1547579221 shutdown_connections 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 --2- 172.21.15.28:0/1547579221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501088f0 0x7fa350110470 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 --2- 172.21.15.28:0/1547579221 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa3501037f0 0x7fa350103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 --2- 172.21.15.28:0/1547579221 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501041a0 0x7fa3501081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 -- 172.21.15.28:0/1547579221 >> 172.21.15.28:0/1547579221 conn(0x7fa35007a6a0 msgr2=0x7fa35007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 -- 172.21.15.28:0/1547579221 shutdown_connections 2024-05-20T20:02:13.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.551+0000 7fa355800700 1 -- 172.21.15.28:0/1547579221 wait complete. 2024-05-20T20:02:13.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.552+0000 7fa355800700 1 Processor -- start 2024-05-20T20:02:13.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.552+0000 7fa355800700 1 -- start start 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa355800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501037f0 0x7fa35019be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa355800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501041a0 0x7fa35019c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa355800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa3501088f0 0x7fa3501a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa355800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa350113710 con 0x7fa3501041a0 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa355800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa350113590 con 0x7fa3501088f0 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa355800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa350113890 con 0x7fa3501037f0 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa34ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501037f0 0x7fa35019be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa34ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501037f0 0x7fa35019be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:42448/0 (socket says 172.21.15.28:42448) 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa34ea00700 1 -- 172.21.15.28:0/552200385 learned_addr learned my addr 172.21.15.28:0/552200385 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa34e000700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501041a0 0x7fa35019c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa34f400700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa3501088f0 0x7fa3501a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:13.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.553+0000 7fa34ea00700 1 -- 172.21.15.28:0/552200385 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa3501088f0 msgr2=0x7fa3501a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:13.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa34ea00700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa3501088f0 0x7fa3501a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa34ea00700 1 -- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501041a0 msgr2=0x7fa35019c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:13.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa34ea00700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501041a0 0x7fa35019c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa34ea00700 1 -- 172.21.15.28:0/552200385 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa3501a0de0 con 0x7fa3501037f0 2024-05-20T20:02:13.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa34ea00700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501037f0 0x7fa35019be20 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa34000eb30 tx=0x7fa34000ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:13.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa347400700 1 -- 172.21.15.28:0/552200385 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa34000cbd0 con 0x7fa3501037f0 2024-05-20T20:02:13.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa3501a10d0 con 0x7fa3501037f0 2024-05-20T20:02:13.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa347400700 1 -- 172.21.15.28:0/552200385 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa3400044d0 con 0x7fa3501037f0 2024-05-20T20:02:13.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa347400700 1 -- 172.21.15.28:0/552200385 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa340003e30 con 0x7fa3501037f0 2024-05-20T20:02:13.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.554+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa3501a8950 con 0x7fa3501037f0 2024-05-20T20:02:13.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.555+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa35006a230 con 0x7fa3501037f0 2024-05-20T20:02:13.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.558+0000 7fa347400700 1 -- 172.21.15.28:0/552200385 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fa340004000 con 0x7fa3501037f0 2024-05-20T20:02:13.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.559+0000 7fa347400700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa3340b2990 0x7fa3340b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:13.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.559+0000 7fa347400700 1 -- 172.21.15.28:0/552200385 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fa3400e2740 con 0x7fa3501037f0 2024-05-20T20:02:13.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.560+0000 7fa34e000700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa3340b2990 0x7fa3340b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:13.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.560+0000 7fa34e000700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa3340b2990 0x7fa3340b4e50 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fa33c009680 tx=0x7fa33c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:13.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.563+0000 7fa347400700 1 -- 172.21.15.28:0/552200385 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa3400042f0 con 0x7fa3501037f0 2024-05-20T20:02:13.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.866+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fa35019d130 con 0x7fa3501037f0 2024-05-20T20:02:13.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.867+0000 7fa347400700 1 -- 172.21.15.28:0/552200385 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 28 v28) ==== 94+0+11925 (secure 0 0 0) 0x7fa3400aa2c0 con 0x7fa3501037f0 2024-05-20T20:02:13.868 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:02:13.868 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":28,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":28,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:01:50.441739+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24475,24482,24493,24500,24542]},"id":1}]} 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.870+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa3340b2990 msgr2=0x7fa3340b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.870+0000 7fa355800700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa3340b2990 0x7fa3340b4e50 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fa33c009680 tx=0x7fa33c009260 comp rx=0 tx=0).stop 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.870+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501037f0 msgr2=0x7fa35019be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.870+0000 7fa355800700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501037f0 0x7fa35019be20 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa34000eb30 tx=0x7fa34000ee40 comp rx=0 tx=0).stop 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 shutdown_connections 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa3340b2990 0x7fa3340b4e50 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa3501088f0 0x7fa3501a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa3501037f0 0x7fa35019be20 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 --2- 172.21.15.28:0/552200385 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa3501041a0 0x7fa35019c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 >> 172.21.15.28:0/552200385 conn(0x7fa35007a6a0 msgr2=0x7fa3501017e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 shutdown_connections 2024-05-20T20:02:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:13.871+0000 7fa355800700 1 -- 172.21.15.28:0/552200385 wait complete. 2024-05-20T20:02:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 28 2024-05-20T20:02:15.101 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:14 smithi028 ceph-mon[31178]: pgmap v345: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.8 KiB/s rd, 4.3 KiB/s wr, 12 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:15.101 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:14 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/552200385' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:02:15.102 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.102 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.102 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.102 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:14 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.106 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:02:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:14 smithi154 ceph-mon[36314]: pgmap v345: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.8 KiB/s rd, 4.3 KiB/s wr, 12 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:14 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/552200385' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:02:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:14 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:14 smithi033 ceph-mon[36179]: pgmap v345: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.8 KiB/s rd, 4.3 KiB/s wr, 12 op/s; 18 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:14 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/552200385' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:02:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:14 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:15.599 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:17.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:16 smithi028 ceph-mon[31178]: pgmap v346: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.0 KiB/s rd, 3.8 KiB/s wr, 11 op/s; 16 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:16 smithi154 ceph-mon[36314]: pgmap v346: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.0 KiB/s rd, 3.8 KiB/s wr, 11 op/s; 16 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:16 smithi033 ceph-mon[36179]: pgmap v346: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.0 KiB/s rd, 3.8 KiB/s wr, 11 op/s; 16 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:17.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.926+0000 7fcf5a000700 1 -- 172.21.15.28:0/1561394706 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 msgr2=0x7fcf54109990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:17.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.926+0000 7fcf5a000700 1 --2- 172.21.15.28:0/1561394706 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 0x7fcf54109990 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7fcf44009990 tx=0x7fcf44009ca0 comp rx=0 tx=0).stop 2024-05-20T20:02:17.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.926+0000 7fcf5a000700 1 -- 172.21.15.28:0/1561394706 shutdown_connections 2024-05-20T20:02:17.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.926+0000 7fcf5a000700 1 --2- 172.21.15.28:0/1561394706 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf5406bbe0 0x7fcf541103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:17.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.926+0000 7fcf5a000700 1 --2- 172.21.15.28:0/1561394706 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 0x7fcf54109990 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:17.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.926+0000 7fcf5a000700 1 --2- 172.21.15.28:0/1561394706 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 0x7fcf5406b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:17.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.926+0000 7fcf5a000700 1 -- 172.21.15.28:0/1561394706 >> 172.21.15.28:0/1561394706 conn(0x7fcf540793f0 msgr2=0x7fcf54079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:17.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.927+0000 7fcf5a000700 1 -- 172.21.15.28:0/1561394706 shutdown_connections 2024-05-20T20:02:17.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.927+0000 7fcf5a000700 1 -- 172.21.15.28:0/1561394706 wait complete. 2024-05-20T20:02:17.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 Processor -- start 2024-05-20T20:02:17.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 -- start start 2024-05-20T20:02:17.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 0x7fcf5419bda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:17.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf5406bbe0 0x7fcf5419c2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:17.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 0x7fcf541a0670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:17.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fcf541136c0 con 0x7fcf541095b0 2024-05-20T20:02:17.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fcf54113540 con 0x7fcf5406bbe0 2024-05-20T20:02:17.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.928+0000 7fcf5a000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fcf54113840 con 0x7fcf5406b220 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 0x7fcf5419bda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 0x7fcf5419bda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:46936/0 (socket says 172.21.15.28:46936) 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 -- 172.21.15.28:0/2502176115 learned_addr learned my addr 172.21.15.28:0/2502176115 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53e00700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 0x7fcf541a0670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf52a00700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf5406bbe0 0x7fcf5419c2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 -- 172.21.15.28:0/2502176115 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf5406bbe0 msgr2=0x7fcf5419c2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf5406bbe0 0x7fcf5419c2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 -- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 msgr2=0x7fcf541a0670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 0x7fcf541a0670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.929+0000 7fcf53400700 1 -- 172.21.15.28:0/2502176115 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcf541a0db0 con 0x7fcf5406b220 2024-05-20T20:02:17.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.930+0000 7fcf53400700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 0x7fcf5419bda0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fcf44000c00 tx=0x7fcf4400bfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:17.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.930+0000 7fcf4be00700 1 -- 172.21.15.28:0/2502176115 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf44027070 con 0x7fcf5406b220 2024-05-20T20:02:17.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.930+0000 7fcf4be00700 1 -- 172.21.15.28:0/2502176115 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcf44022470 con 0x7fcf5406b220 2024-05-20T20:02:17.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.930+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcf541a1040 con 0x7fcf5406b220 2024-05-20T20:02:17.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.930+0000 7fcf4be00700 1 -- 172.21.15.28:0/2502176115 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf44017650 con 0x7fcf5406b220 2024-05-20T20:02:17.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.930+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcf541a8920 con 0x7fcf5406b220 2024-05-20T20:02:17.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.931+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcf5406a230 con 0x7fcf5406b220 2024-05-20T20:02:17.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.935+0000 7fcf4be00700 1 -- 172.21.15.28:0/2502176115 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fcf4402a070 con 0x7fcf5406b220 2024-05-20T20:02:17.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.936+0000 7fcf4be00700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf380b2990 0x7fcf380b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:17.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.937+0000 7fcf4be00700 1 -- 172.21.15.28:0/2502176115 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fcf440e2fe0 con 0x7fcf5406b220 2024-05-20T20:02:17.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.937+0000 7fcf52a00700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf380b2990 0x7fcf380b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:17.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.938+0000 7fcf52a00700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf380b2990 0x7fcf380b4e50 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fcf54110810 tx=0x7fcf40009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:17.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:17.941+0000 7fcf4be00700 1 -- 172.21.15.28:0/2502176115 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fcf44013070 con 0x7fcf5406b220 2024-05-20T20:02:18.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.187+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fcf541a8af0 con 0x7fcf5406b220 2024-05-20T20:02:18.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.189+0000 7fcf4be00700 1 -- 172.21.15.28:0/2502176115 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) ==== 74+0+22571 (secure 0 0 0) 0x7fcf440aac10 con 0x7fcf5406b220 2024-05-20T20:02:18.190 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:02:18.190 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":80,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:02:04.742448+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf380b2990 msgr2=0x7fcf380b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf380b2990 0x7fcf380b4e50 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fcf54110810 tx=0x7fcf40009260 comp rx=0 tx=0).stop 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 msgr2=0x7fcf5419bda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 0x7fcf5419bda0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fcf44000c00 tx=0x7fcf4400bfd0 comp rx=0 tx=0).stop 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 shutdown_connections 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf380b2990 0x7fcf380b4e50 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf541095b0 0x7fcf541a0670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf5406b220 0x7fcf5419bda0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 --2- 172.21.15.28:0/2502176115 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf5406bbe0 0x7fcf5419c2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:18.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.192+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 >> 172.21.15.28:0/2502176115 conn(0x7fcf540793f0 msgr2=0x7fcf54100eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:18.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.193+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 shutdown_connections 2024-05-20T20:02:18.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:18.193+0000 7fcf5a000700 1 -- 172.21.15.28:0/2502176115 wait complete. 2024-05-20T20:02:18.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:18 smithi154 ceph-mon[36314]: pgmap v347: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 3.3 KiB/s wr, 10 op/s; 14 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:18.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:18.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:18 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:18.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:18 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2502176115' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:18 smithi033 ceph-mon[36179]: pgmap v347: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 3.3 KiB/s wr, 10 op/s; 14 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:18 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:18 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2502176115' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:19.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:18 smithi028 ceph-mon[31178]: pgmap v347: 97 pgs: 97 active+clean; 639 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 3.3 KiB/s wr, 10 op/s; 14 B/s, 1 keys/s, 1 objects/s recovering 2024-05-20T20:02:19.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:18 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:18 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2502176115' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:19.167 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-20T20:02:19.167 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T20:02:19.635 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:19.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:02:19.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:02:19.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:02:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:02:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:19.923 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:02:19.923 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:02:19.923 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:02:20.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:20 smithi154 ceph-mon[36314]: pgmap v348: 97 pgs: 97 active+clean; 640 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 3.6 KiB/s wr, 13 op/s; 9 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T20:02:20.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:20 smithi033 ceph-mon[36179]: pgmap v348: 97 pgs: 97 active+clean; 640 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 3.6 KiB/s wr, 13 op/s; 9 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T20:02:20.877 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:20 smithi028 ceph-mon[31178]: pgmap v348: 97 pgs: 97 active+clean; 640 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 3.6 KiB/s wr, 13 op/s; 9 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T20:02:21.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 -- 172.21.15.28:0/2006321475 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044106140 msgr2=0x7ff044112650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:21.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 --2- 172.21.15.28:0/2006321475 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044106140 0x7ff044112650 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7ff03c009990 tx=0x7ff03c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:02:21.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 -- 172.21.15.28:0/2006321475 shutdown_connections 2024-05-20T20:02:21.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 --2- 172.21.15.28:0/2006321475 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044106140 0x7ff044112650 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:21.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 --2- 172.21.15.28:0/2006321475 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff04410b780 0x7ff04410bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:21.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 --2- 172.21.15.28:0/2006321475 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff044105780 0x7ff044105c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:21.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 -- 172.21.15.28:0/2006321475 >> 172.21.15.28:0/2006321475 conn(0x7ff044101270 msgr2=0x7ff044103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:21.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.757+0000 7ff048c00700 1 -- 172.21.15.28:0/2006321475 shutdown_connections 2024-05-20T20:02:21.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.759+0000 7ff048c00700 1 -- 172.21.15.28:0/2006321475 wait complete. 2024-05-20T20:02:21.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 Processor -- start 2024-05-20T20:02:21.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 -- start start 2024-05-20T20:02:21.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044105780 0x7ff04419bdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff044106140 0x7ff04419c300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff04410b780 0x7ff0441a0690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff0441159b0 con 0x7ff044105780 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff044115830 con 0x7ff04410b780 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.760+0000 7ff048c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff044115b30 con 0x7ff044106140 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff041e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044105780 0x7ff04419bdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff039400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff044106140 0x7ff04419c300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:21.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff041e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044105780 0x7ff04419bdc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60808/0 (socket says 172.21.15.28:60808) 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff041e00700 1 -- 172.21.15.28:0/70821141 learned_addr learned my addr 172.21.15.28:0/70821141 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff042800700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff04410b780 0x7ff0441a0690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff039400700 1 -- 172.21.15.28:0/70821141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff04410b780 msgr2=0x7ff0441a0690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff039400700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff04410b780 0x7ff0441a0690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff039400700 1 -- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044105780 msgr2=0x7ff04419bdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff039400700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044105780 0x7ff04419bdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff039400700 1 -- 172.21.15.28:0/70821141 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff0441a0dd0 con 0x7ff044106140 2024-05-20T20:02:21.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.761+0000 7ff041e00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044105780 0x7ff04419bdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:02:21.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.762+0000 7ff039400700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff044106140 0x7ff04419c300 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7ff02c00e9f0 tx=0x7ff02c00ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:21.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.762+0000 7ff03b400700 1 -- 172.21.15.28:0/70821141 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff02c00cb70 con 0x7ff044106140 2024-05-20T20:02:21.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.762+0000 7ff03b400700 1 -- 172.21.15.28:0/70821141 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff02c0044d0 con 0x7ff044106140 2024-05-20T20:02:21.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.762+0000 7ff03b400700 1 -- 172.21.15.28:0/70821141 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff02c003e00 con 0x7ff044106140 2024-05-20T20:02:21.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.762+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff0441a10c0 con 0x7ff044106140 2024-05-20T20:02:21.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.762+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff0441a8940 con 0x7ff044106140 2024-05-20T20:02:21.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.763+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff04406a230 con 0x7ff044106140 2024-05-20T20:02:21.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.766+0000 7ff03b400700 1 -- 172.21.15.28:0/70821141 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7ff02c003fa0 con 0x7ff044106140 2024-05-20T20:02:21.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.768+0000 7ff03b400700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff0280b2910 0x7ff0280b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:21.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.768+0000 7ff03b400700 1 -- 172.21.15.28:0/70821141 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7ff02c0e2560 con 0x7ff044106140 2024-05-20T20:02:21.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.769+0000 7ff041e00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff0280b2910 0x7ff0280b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:21.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.770+0000 7ff041e00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff0280b2910 0x7ff0280b4dd0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7ff034009c10 tx=0x7ff03400b040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:21.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:21.771+0000 7ff03b400700 1 -- 172.21.15.28:0/70821141 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7ff02c0aa0e0 con 0x7ff044106140 2024-05-20T20:02:22.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.009+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7ff044063980 con 0x7ff0280b2910 2024-05-20T20:02:22.036 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.036+0000 7ff03b400700 1 -- 172.21.15.28:0/70821141 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258988 (secure 0 0 0) 0x7ff044063980 con 0x7ff0280b2910 2024-05-20T20:02:22.043 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:02:22.045 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff0280b2910 msgr2=0x7ff0280b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:22.045 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff0280b2910 0x7ff0280b4dd0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7ff034009c10 tx=0x7ff03400b040 comp rx=0 tx=0).stop 2024-05-20T20:02:22.045 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff044106140 msgr2=0x7ff04419c300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:22.045 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff044106140 0x7ff04419c300 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7ff02c00e9f0 tx=0x7ff02c00ed00 comp rx=0 tx=0).stop 2024-05-20T20:02:22.045 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 shutdown_connections 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff0280b2910 0x7ff0280b4dd0 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff04410b780 0x7ff0441a0690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff044105780 0x7ff04419bdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 --2- 172.21.15.28:0/70821141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff044106140 0x7ff04419c300 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.042+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 >> 172.21.15.28:0/70821141 conn(0x7ff044101270 msgr2=0x7ff0441017f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.043+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 shutdown_connections 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:22.043+0000 7ff048c00700 1 -- 172.21.15.28:0/70821141 wait complete. 2024-05-20T20:02:22.046 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:02:22.803 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:22 smithi028 ceph-mon[31178]: pgmap v349: 97 pgs: 97 active+clean; 640 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 10 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T20:02:22.863 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":349,"stamp":"2024-05-20T20:02:21.392541+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":655493,"num_objects":102,"num_object_clones":0,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":0,"num_read":434,"num_read_kb":317,"num_write":368,"num_write_kb":2283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":302,"ondisk_log_size":302,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347512,"kb_used_data":10804,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124594312,"statfs":{"total":1151940427776,"available":1151584575488,"internally_reserved":0,"allocated":11063296,"data_stored":7066356,"data_compressed":177837,"data_compressed_allocated":233472,"data_compressed_original":1966080,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":15877,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":106,"num_read_kb":71,"num_write":16,"num_write_kb":21,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004951"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751812+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:02:04.751812+0000","last_peered":"2024-05-20T20:02:04.751812+0000","last_clean":"2024-05-20T20:02:04.751812+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:02:04.751812+0000","last_undegraded":"2024-05-20T20:02:04.751812+0000","last_fullsized":"2024-05-20T20:02:04.751812+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770822+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:02:04.770822+0000","last_peered":"2024-05-20T20:02:04.770822+0000","last_clean":"2024-05-20T20:02:04.770822+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:02:04.770822+0000","last_undegraded":"2024-05-20T20:02:04.770822+0000","last_fullsized":"2024-05-20T20:02:04.770822+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770707+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:02:04.770707+0000","last_peered":"2024-05-20T20:02:04.770707+0000","last_clean":"2024-05-20T20:02:04.770707+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:02:04.770707+0000","last_undegraded":"2024-05-20T20:02:04.770707+0000","last_fullsized":"2024-05-20T20:02:04.770707+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154915+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:02:05.154915+0000","last_peered":"2024-05-20T20:02:05.154915+0000","last_clean":"2024-05-20T20:02:05.154915+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:02:05.154915+0000","last_undegraded":"2024-05-20T20:02:05.154915+0000","last_fullsized":"2024-05-20T20:02:05.154915+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751552+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:02:04.751552+0000","last_peered":"2024-05-20T20:02:04.751552+0000","last_clean":"2024-05-20T20:02:04.751552+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:02:04.751552+0000","last_undegraded":"2024-05-20T20:02:04.751552+0000","last_fullsized":"2024-05-20T20:02:04.751552+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.762828+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:02:04.762828+0000","last_peered":"2024-05-20T20:02:04.762828+0000","last_clean":"2024-05-20T20:02:04.762828+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:02:04.762828+0000","last_undegraded":"2024-05-20T20:02:04.762828+0000","last_fullsized":"2024-05-20T20:02:04.762828+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930220+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:02:04.930220+0000","last_peered":"2024-05-20T20:02:04.930220+0000","last_clean":"2024-05-20T20:02:04.930220+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:02:04.930220+0000","last_undegraded":"2024-05-20T20:02:04.930220+0000","last_fullsized":"2024-05-20T20:02:04.930220+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750682+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:02:04.750682+0000","last_peered":"2024-05-20T20:02:04.750682+0000","last_clean":"2024-05-20T20:02:04.750682+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:02:04.750682+0000","last_undegraded":"2024-05-20T20:02:04.750682+0000","last_fullsized":"2024-05-20T20:02:04.750682+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154551+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:02:05.154551+0000","last_peered":"2024-05-20T20:02:05.154551+0000","last_clean":"2024-05-20T20:02:05.154551+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:02:05.154551+0000","last_undegraded":"2024-05-20T20:02:05.154551+0000","last_fullsized":"2024-05-20T20:02:05.154551+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.763055+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:02:04.763055+0000","last_peered":"2024-05-20T20:02:04.763055+0000","last_clean":"2024-05-20T20:02:04.763055+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:02:04.763055+0000","last_undegraded":"2024-05-20T20:02:04.763055+0000","last_fullsized":"2024-05-20T20:02:04.763055+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930302+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:02:04.930302+0000","last_peered":"2024-05-20T20:02:04.930302+0000","last_clean":"2024-05-20T20:02:04.930302+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:02:04.930302+0000","last_undegraded":"2024-05-20T20:02:04.930302+0000","last_fullsized":"2024-05-20T20:02:04.930302+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154923+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:02:05.154923+0000","last_peered":"2024-05-20T20:02:05.154923+0000","last_clean":"2024-05-20T20:02:05.154923+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:02:05.154923+0000","last_undegraded":"2024-05-20T20:02:05.154923+0000","last_fullsized":"2024-05-20T20:02:05.154923+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153908+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:02:05.153908+0000","last_peered":"2024-05-20T20:02:05.153908+0000","last_clean":"2024-05-20T20:02:05.153908+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:02:05.153908+0000","last_undegraded":"2024-05-20T20:02:05.153908+0000","last_fullsized":"2024-05-20T20:02:05.153908+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.769576+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:02:04.769576+0000","last_peered":"2024-05-20T20:02:04.769576+0000","last_clean":"2024-05-20T20:02:04.769576+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:02:04.769576+0000","last_undegraded":"2024-05-20T20:02:04.769576+0000","last_fullsized":"2024-05-20T20:02:04.769576+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.769541+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:02:04.769541+0000","last_peered":"2024-05-20T20:02:04.769541+0000","last_clean":"2024-05-20T20:02:04.769541+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:02:04.769541+0000","last_undegraded":"2024-05-20T20:02:04.769541+0000","last_fullsized":"2024-05-20T20:02:04.769541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153684+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:02:05.153684+0000","last_peered":"2024-05-20T20:02:05.153684+0000","last_clean":"2024-05-20T20:02:05.153684+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:02:05.153684+0000","last_undegraded":"2024-05-20T20:02:05.153684+0000","last_fullsized":"2024-05-20T20:02:05.153684+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752019+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:02:04.752019+0000","last_peered":"2024-05-20T20:02:04.752019+0000","last_clean":"2024-05-20T20:02:04.752019+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:02:04.752019+0000","last_undegraded":"2024-05-20T20:02:04.752019+0000","last_fullsized":"2024-05-20T20:02:04.752019+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.759389+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:02:04.759389+0000","last_peered":"2024-05-20T20:02:04.759389+0000","last_clean":"2024-05-20T20:02:04.759389+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:02:04.759389+0000","last_undegraded":"2024-05-20T20:02:04.759389+0000","last_fullsized":"2024-05-20T20:02:04.759389+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.754131+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:02:04.754131+0000","last_peered":"2024-05-20T20:02:04.754131+0000","last_clean":"2024-05-20T20:02:04.754131+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:02:04.754131+0000","last_undegraded":"2024-05-20T20:02:04.754131+0000","last_fullsized":"2024-05-20T20:02:04.754131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"78'13","reported_seq":73,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:17.743125+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:02:17.743125+0000","last_peered":"2024-05-20T20:02:17.743125+0000","last_clean":"2024-05-20T20:02:17.743125+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:02:17.743125+0000","last_undegraded":"2024-05-20T20:02:17.743125+0000","last_fullsized":"2024-05-20T20:02:17.743125+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":34,"num_read_kb":34,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752457+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:02:04.752457+0000","last_peered":"2024-05-20T20:02:04.752457+0000","last_clean":"2024-05-20T20:02:04.752457+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:02:04.752457+0000","last_undegraded":"2024-05-20T20:02:04.752457+0000","last_fullsized":"2024-05-20T20:02:04.752457+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753101+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:02:04.753101+0000","last_peered":"2024-05-20T20:02:04.753101+0000","last_clean":"2024-05-20T20:02:04.753101+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:02:04.753101+0000","last_undegraded":"2024-05-20T20:02:04.753101+0000","last_fullsized":"2024-05-20T20:02:04.753101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770833+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:02:04.770833+0000","last_peered":"2024-05-20T20:02:04.770833+0000","last_clean":"2024-05-20T20:02:04.770833+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:02:04.770833+0000","last_undegraded":"2024-05-20T20:02:04.770833+0000","last_fullsized":"2024-05-20T20:02:04.770833+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"78'12","reported_seq":39,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752192+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:02:04.752192+0000","last_peered":"2024-05-20T20:02:04.752192+0000","last_clean":"2024-05-20T20:02:04.752192+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:02:04.752192+0000","last_undegraded":"2024-05-20T20:02:04.752192+0000","last_fullsized":"2024-05-20T20:02:04.752192+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154230+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:02:05.154230+0000","last_peered":"2024-05-20T20:02:05.154230+0000","last_clean":"2024-05-20T20:02:05.154230+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:02:05.154230+0000","last_undegraded":"2024-05-20T20:02:05.154230+0000","last_fullsized":"2024-05-20T20:02:05.154230+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751794+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:02:04.751794+0000","last_peered":"2024-05-20T20:02:04.751794+0000","last_clean":"2024-05-20T20:02:04.751794+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:02:04.751794+0000","last_undegraded":"2024-05-20T20:02:04.751794+0000","last_fullsized":"2024-05-20T20:02:04.751794+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.159343+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:02:05.159343+0000","last_peered":"2024-05-20T20:02:05.159343+0000","last_clean":"2024-05-20T20:02:05.159343+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:02:05.159343+0000","last_undegraded":"2024-05-20T20:02:05.159343+0000","last_fullsized":"2024-05-20T20:02:05.159343+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"78'6","reported_seq":66,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:15.713037+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:02:15.713037+0000","last_peered":"2024-05-20T20:02:15.713037+0000","last_clean":"2024-05-20T20:02:15.713037+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:02:15.713037+0000","last_undegraded":"2024-05-20T20:02:15.713037+0000","last_fullsized":"2024-05-20T20:02:15.713037+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1013,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":34,"num_read_kb":3,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.931600+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:02:04.931600+0000","last_peered":"2024-05-20T20:02:04.931600+0000","last_clean":"2024-05-20T20:02:04.931600+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:02:04.931600+0000","last_undegraded":"2024-05-20T20:02:04.931600+0000","last_fullsized":"2024-05-20T20:02:04.931600+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748817+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:02:04.748817+0000","last_peered":"2024-05-20T20:02:04.748817+0000","last_clean":"2024-05-20T20:02:04.748817+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:02:04.748817+0000","last_undegraded":"2024-05-20T20:02:04.748817+0000","last_fullsized":"2024-05-20T20:02:04.748817+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748647+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:02:04.748647+0000","last_peered":"2024-05-20T20:02:04.748647+0000","last_clean":"2024-05-20T20:02:04.748647+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:02:04.748647+0000","last_undegraded":"2024-05-20T20:02:04.748647+0000","last_fullsized":"2024-05-20T20:02:04.748647+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153602+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:02:05.153602+0000","last_peered":"2024-05-20T20:02:05.153602+0000","last_clean":"2024-05-20T20:02:05.153602+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:02:05.153602+0000","last_undegraded":"2024-05-20T20:02:05.153602+0000","last_fullsized":"2024-05-20T20:02:05.153602+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750852+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:02:04.750852+0000","last_peered":"2024-05-20T20:02:04.750852+0000","last_clean":"2024-05-20T20:02:04.750852+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:02:04.750852+0000","last_undegraded":"2024-05-20T20:02:04.750852+0000","last_fullsized":"2024-05-20T20:02:04.750852+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748365+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:02:04.748365+0000","last_peered":"2024-05-20T20:02:04.748365+0000","last_clean":"2024-05-20T20:02:04.748365+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:02:04.748365+0000","last_undegraded":"2024-05-20T20:02:04.748365+0000","last_fullsized":"2024-05-20T20:02:04.748365+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753519+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:02:04.753519+0000","last_peered":"2024-05-20T20:02:04.753519+0000","last_clean":"2024-05-20T20:02:04.753519+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:02:04.753519+0000","last_undegraded":"2024-05-20T20:02:04.753519+0000","last_fullsized":"2024-05-20T20:02:04.753519+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"78'4","reported_seq":92,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:16.713633+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:02:16.713633+0000","last_peered":"2024-05-20T20:02:16.713633+0000","last_clean":"2024-05-20T20:02:16.713633+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:02:16.713633+0000","last_undegraded":"2024-05-20T20:02:16.713633+0000","last_fullsized":"2024-05-20T20:02:16.713633+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":62,"num_read_kb":62,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154065+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:02:05.154065+0000","last_peered":"2024-05-20T20:02:05.154065+0000","last_clean":"2024-05-20T20:02:05.154065+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:02:05.154065+0000","last_undegraded":"2024-05-20T20:02:05.154065+0000","last_fullsized":"2024-05-20T20:02:05.154065+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930524+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:02:04.930524+0000","last_peered":"2024-05-20T20:02:04.930524+0000","last_clean":"2024-05-20T20:02:04.930524+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:02:04.930524+0000","last_undegraded":"2024-05-20T20:02:04.930524+0000","last_fullsized":"2024-05-20T20:02:04.930524+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.771192+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:02:04.771192+0000","last_peered":"2024-05-20T20:02:04.771192+0000","last_clean":"2024-05-20T20:02:04.771192+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:02:04.771192+0000","last_undegraded":"2024-05-20T20:02:04.771192+0000","last_fullsized":"2024-05-20T20:02:04.771192+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.749018+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:02:04.749018+0000","last_peered":"2024-05-20T20:02:04.749018+0000","last_clean":"2024-05-20T20:02:04.749018+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:02:04.749018+0000","last_undegraded":"2024-05-20T20:02:04.749018+0000","last_fullsized":"2024-05-20T20:02:04.749018+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154455+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:02:05.154455+0000","last_peered":"2024-05-20T20:02:05.154455+0000","last_clean":"2024-05-20T20:02:05.154455+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:02:05.154455+0000","last_undegraded":"2024-05-20T20:02:05.154455+0000","last_fullsized":"2024-05-20T20:02:05.154455+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"80'30","reported_seq":126,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:16.713719+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:02:16.713719+0000","last_peered":"2024-05-20T20:02:16.713719+0000","last_clean":"2024-05-20T20:02:16.713719+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:02:16.713719+0000","last_undegraded":"2024-05-20T20:02:16.713719+0000","last_fullsized":"2024-05-20T20:02:16.713719+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58078,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":70,"num_read_kb":91,"num_write":32,"num_write_kb":66,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751889+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:02:04.751889+0000","last_peered":"2024-05-20T20:02:04.751889+0000","last_clean":"2024-05-20T20:02:04.751889+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:02:04.751889+0000","last_undegraded":"2024-05-20T20:02:04.751889+0000","last_fullsized":"2024-05-20T20:02:04.751889+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.773205+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:02:04.773205+0000","last_peered":"2024-05-20T20:02:04.773205+0000","last_clean":"2024-05-20T20:02:04.773205+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:02:04.773205+0000","last_undegraded":"2024-05-20T20:02:04.773205+0000","last_fullsized":"2024-05-20T20:02:04.773205+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153440+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:02:05.153440+0000","last_peered":"2024-05-20T20:02:05.153440+0000","last_clean":"2024-05-20T20:02:05.153440+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:02:05.153440+0000","last_undegraded":"2024-05-20T20:02:05.153440+0000","last_fullsized":"2024-05-20T20:02:05.153440+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748643+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:02:04.748643+0000","last_peered":"2024-05-20T20:02:04.748643+0000","last_clean":"2024-05-20T20:02:04.748643+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:02:04.748643+0000","last_undegraded":"2024-05-20T20:02:04.748643+0000","last_fullsized":"2024-05-20T20:02:04.748643+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930547+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:02:04.930547+0000","last_peered":"2024-05-20T20:02:04.930547+0000","last_clean":"2024-05-20T20:02:04.930547+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:02:04.930547+0000","last_undegraded":"2024-05-20T20:02:04.930547+0000","last_fullsized":"2024-05-20T20:02:04.930547+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"78'5","reported_seq":64,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:15.740184+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:02:15.740184+0000","last_peered":"2024-05-20T20:02:15.740184+0000","last_clean":"2024-05-20T20:02:15.740184+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:02:15.740184+0000","last_undegraded":"2024-05-20T20:02:15.740184+0000","last_fullsized":"2024-05-20T20:02:15.740184+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":34,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.929973+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:02:04.929973+0000","last_peered":"2024-05-20T20:02:04.929973+0000","last_clean":"2024-05-20T20:02:04.929973+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:02:04.929973+0000","last_undegraded":"2024-05-20T20:02:04.929973+0000","last_fullsized":"2024-05-20T20:02:04.929973+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":68,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.169342+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:02:05.169342+0000","last_peered":"2024-05-20T20:02:05.169342+0000","last_clean":"2024-05-20T20:02:05.169342+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:02:05.169342+0000","last_undegraded":"2024-05-20T20:02:05.169342+0000","last_fullsized":"2024-05-20T20:02:05.169342+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748542+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:02:04.748542+0000","last_peered":"2024-05-20T20:02:04.748542+0000","last_clean":"2024-05-20T20:02:04.748542+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:02:04.748542+0000","last_undegraded":"2024-05-20T20:02:04.748542+0000","last_fullsized":"2024-05-20T20:02:04.748542+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"78'2","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752142+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:02:04.752142+0000","last_peered":"2024-05-20T20:02:04.752142+0000","last_clean":"2024-05-20T20:02:04.752142+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:02:04.752142+0000","last_undegraded":"2024-05-20T20:02:04.752142+0000","last_fullsized":"2024-05-20T20:02:04.752142+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751702+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:02:04.751702+0000","last_peered":"2024-05-20T20:02:04.751702+0000","last_clean":"2024-05-20T20:02:04.751702+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:02:04.751702+0000","last_undegraded":"2024-05-20T20:02:04.751702+0000","last_fullsized":"2024-05-20T20:02:04.751702+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.757043+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:02:04.757043+0000","last_peered":"2024-05-20T20:02:04.757043+0000","last_clean":"2024-05-20T20:02:04.757043+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:02:04.757043+0000","last_undegraded":"2024-05-20T20:02:04.757043+0000","last_fullsized":"2024-05-20T20:02:04.757043+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"78'1","reported_seq":21,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.749048+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:02:04.749048+0000","last_peered":"2024-05-20T20:02:04.749048+0000","last_clean":"2024-05-20T20:02:04.749048+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:02:04.749048+0000","last_undegraded":"2024-05-20T20:02:04.749048+0000","last_fullsized":"2024-05-20T20:02:04.749048+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"78'5","reported_seq":65,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:17.744012+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:02:17.744012+0000","last_peered":"2024-05-20T20:02:17.744012+0000","last_clean":"2024-05-20T20:02:17.744012+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:02:17.744012+0000","last_undegraded":"2024-05-20T20:02:17.744012+0000","last_fullsized":"2024-05-20T20:02:17.744012+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":34,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748898+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:02:04.748898+0000","last_peered":"2024-05-20T20:02:04.748898+0000","last_clean":"2024-05-20T20:02:04.748898+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:02:04.748898+0000","last_undegraded":"2024-05-20T20:02:04.748898+0000","last_fullsized":"2024-05-20T20:02:04.748898+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748775+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:02:04.748775+0000","last_peered":"2024-05-20T20:02:04.748775+0000","last_clean":"2024-05-20T20:02:04.748775+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:02:04.748775+0000","last_undegraded":"2024-05-20T20:02:04.748775+0000","last_fullsized":"2024-05-20T20:02:04.748775+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"78'5","reported_seq":63,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:15.738945+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:02:15.738945+0000","last_peered":"2024-05-20T20:02:15.738945+0000","last_clean":"2024-05-20T20:02:15.738945+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:02:15.738945+0000","last_undegraded":"2024-05-20T20:02:15.738945+0000","last_fullsized":"2024-05-20T20:02:15.738945+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":32,"num_read_kb":32,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751623+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:02:04.751623+0000","last_peered":"2024-05-20T20:02:04.751623+0000","last_clean":"2024-05-20T20:02:04.751623+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:02:04.751623+0000","last_undegraded":"2024-05-20T20:02:04.751623+0000","last_fullsized":"2024-05-20T20:02:04.751623+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":35,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154587+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:02:05.154587+0000","last_peered":"2024-05-20T20:02:05.154587+0000","last_clean":"2024-05-20T20:02:05.154587+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:02:05.154587+0000","last_undegraded":"2024-05-20T20:02:05.154587+0000","last_fullsized":"2024-05-20T20:02:05.154587+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"20'92","reported_seq":100,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930195+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:02:04.930195+0000","last_peered":"2024-05-20T20:02:04.930195+0000","last_clean":"2024-05-20T20:02:04.930195+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:02:04.930195+0000","last_undegraded":"2024-05-20T20:02:04.930195+0000","last_fullsized":"2024-05-20T20:02:04.930195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153378+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:02:05.153378+0000","last_peered":"2024-05-20T20:02:05.153378+0000","last_clean":"2024-05-20T20:02:05.153378+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:02:05.153378+0000","last_undegraded":"2024-05-20T20:02:05.153378+0000","last_fullsized":"2024-05-20T20:02:05.153378+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930276+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:02:04.930276+0000","last_peered":"2024-05-20T20:02:04.930276+0000","last_clean":"2024-05-20T20:02:04.930276+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:02:04.930276+0000","last_undegraded":"2024-05-20T20:02:04.930276+0000","last_fullsized":"2024-05-20T20:02:04.930276+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752011+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:02:04.752011+0000","last_peered":"2024-05-20T20:02:04.752011+0000","last_clean":"2024-05-20T20:02:04.752011+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:02:04.752011+0000","last_undegraded":"2024-05-20T20:02:04.752011+0000","last_fullsized":"2024-05-20T20:02:04.752011+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154276+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:02:05.154276+0000","last_peered":"2024-05-20T20:02:05.154276+0000","last_clean":"2024-05-20T20:02:05.154276+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:02:05.154276+0000","last_undegraded":"2024-05-20T20:02:05.154276+0000","last_fullsized":"2024-05-20T20:02:05.154276+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"80'15","reported_seq":57,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:11.535003+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:02:11.535003+0000","last_peered":"2024-05-20T20:02:11.535003+0000","last_clean":"2024-05-20T20:02:11.535003+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:02:11.535003+0000","last_undegraded":"2024-05-20T20:02:11.535003+0000","last_fullsized":"2024-05-20T20:02:11.535003+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.754047+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:02:04.754047+0000","last_peered":"2024-05-20T20:02:04.754047+0000","last_clean":"2024-05-20T20:02:04.754047+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:02:04.754047+0000","last_undegraded":"2024-05-20T20:02:04.754047+0000","last_fullsized":"2024-05-20T20:02:04.754047+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":34,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.155314+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:02:05.155314+0000","last_peered":"2024-05-20T20:02:05.155314+0000","last_clean":"2024-05-20T20:02:05.155314+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:02:05.155314+0000","last_undegraded":"2024-05-20T20:02:05.155314+0000","last_fullsized":"2024-05-20T20:02:05.155314+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.155654+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:02:05.155654+0000","last_peered":"2024-05-20T20:02:05.155654+0000","last_clean":"2024-05-20T20:02:05.155654+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:02:05.155654+0000","last_undegraded":"2024-05-20T20:02:05.155654+0000","last_fullsized":"2024-05-20T20:02:05.155654+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153826+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:02:05.153826+0000","last_peered":"2024-05-20T20:02:05.153826+0000","last_clean":"2024-05-20T20:02:05.153826+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:02:05.153826+0000","last_undegraded":"2024-05-20T20:02:05.153826+0000","last_fullsized":"2024-05-20T20:02:05.153826+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751497+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:02:04.751497+0000","last_peered":"2024-05-20T20:02:04.751497+0000","last_clean":"2024-05-20T20:02:04.751497+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:02:04.751497+0000","last_undegraded":"2024-05-20T20:02:04.751497+0000","last_fullsized":"2024-05-20T20:02:04.751497+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750932+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:02:04.750932+0000","last_peered":"2024-05-20T20:02:04.750932+0000","last_clean":"2024-05-20T20:02:04.750932+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:02:04.750932+0000","last_undegraded":"2024-05-20T20:02:04.750932+0000","last_fullsized":"2024-05-20T20:02:04.750932+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750668+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:02:04.750668+0000","last_peered":"2024-05-20T20:02:04.750668+0000","last_clean":"2024-05-20T20:02:04.750668+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:02:04.750668+0000","last_undegraded":"2024-05-20T20:02:04.750668+0000","last_fullsized":"2024-05-20T20:02:04.750668+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753643+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:02:04.753643+0000","last_peered":"2024-05-20T20:02:04.753643+0000","last_clean":"2024-05-20T20:02:04.753643+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:02:04.753643+0000","last_undegraded":"2024-05-20T20:02:04.753643+0000","last_fullsized":"2024-05-20T20:02:04.753643+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751959+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:02:04.751959+0000","last_peered":"2024-05-20T20:02:04.751959+0000","last_clean":"2024-05-20T20:02:04.751959+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:02:04.751959+0000","last_undegraded":"2024-05-20T20:02:04.751959+0000","last_fullsized":"2024-05-20T20:02:04.751959+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751646+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:02:04.751646+0000","last_peered":"2024-05-20T20:02:04.751646+0000","last_clean":"2024-05-20T20:02:04.751646+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:02:04.751646+0000","last_undegraded":"2024-05-20T20:02:04.751646+0000","last_fullsized":"2024-05-20T20:02:04.751646+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930403+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:02:04.930403+0000","last_peered":"2024-05-20T20:02:04.930403+0000","last_clean":"2024-05-20T20:02:04.930403+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:02:04.930403+0000","last_undegraded":"2024-05-20T20:02:04.930403+0000","last_fullsized":"2024-05-20T20:02:04.930403+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"78'5","reported_seq":64,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:15.712875+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:02:15.712875+0000","last_peered":"2024-05-20T20:02:15.712875+0000","last_clean":"2024-05-20T20:02:15.712875+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:02:15.712875+0000","last_undegraded":"2024-05-20T20:02:15.712875+0000","last_fullsized":"2024-05-20T20:02:15.712875+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1170,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":33,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930431+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:02:04.930431+0000","last_peered":"2024-05-20T20:02:04.930431+0000","last_clean":"2024-05-20T20:02:04.930431+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:02:04.930431+0000","last_undegraded":"2024-05-20T20:02:04.930431+0000","last_fullsized":"2024-05-20T20:02:04.930431+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748758+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:02:04.748758+0000","last_peered":"2024-05-20T20:02:04.748758+0000","last_clean":"2024-05-20T20:02:04.748758+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:02:04.748758+0000","last_undegraded":"2024-05-20T20:02:04.748758+0000","last_fullsized":"2024-05-20T20:02:04.748758+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'1","reported_seq":21,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930321+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:02:04.930321+0000","last_peered":"2024-05-20T20:02:04.930321+0000","last_clean":"2024-05-20T20:02:04.930321+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:02:04.930321+0000","last_undegraded":"2024-05-20T20:02:04.930321+0000","last_fullsized":"2024-05-20T20:02:04.930321+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154077+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:02:05.154077+0000","last_peered":"2024-05-20T20:02:05.154077+0000","last_clean":"2024-05-20T20:02:05.154077+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:02:05.154077+0000","last_undegraded":"2024-05-20T20:02:05.154077+0000","last_fullsized":"2024-05-20T20:02:05.154077+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930618+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:02:04.930618+0000","last_peered":"2024-05-20T20:02:04.930618+0000","last_clean":"2024-05-20T20:02:04.930618+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:02:04.930618+0000","last_undegraded":"2024-05-20T20:02:04.930618+0000","last_fullsized":"2024-05-20T20:02:04.930618+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750870+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:02:04.750870+0000","last_peered":"2024-05-20T20:02:04.750870+0000","last_clean":"2024-05-20T20:02:04.750870+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:02:04.750870+0000","last_undegraded":"2024-05-20T20:02:04.750870+0000","last_fullsized":"2024-05-20T20:02:04.750870+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753730+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:02:04.753730+0000","last_peered":"2024-05-20T20:02:04.753730+0000","last_clean":"2024-05-20T20:02:04.753730+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:02:04.753730+0000","last_undegraded":"2024-05-20T20:02:04.753730+0000","last_fullsized":"2024-05-20T20:02:04.753730+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770679+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:02:04.770679+0000","last_peered":"2024-05-20T20:02:04.770679+0000","last_clean":"2024-05-20T20:02:04.770679+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:02:04.770679+0000","last_undegraded":"2024-05-20T20:02:04.770679+0000","last_fullsized":"2024-05-20T20:02:04.770679+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770231+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:02:04.770231+0000","last_peered":"2024-05-20T20:02:04.770231+0000","last_clean":"2024-05-20T20:02:04.770231+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:02:04.770231+0000","last_undegraded":"2024-05-20T20:02:04.770231+0000","last_fullsized":"2024-05-20T20:02:04.770231+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154273+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:02:05.154273+0000","last_peered":"2024-05-20T20:02:05.154273+0000","last_clean":"2024-05-20T20:02:05.154273+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:02:05.154273+0000","last_undegraded":"2024-05-20T20:02:05.154273+0000","last_fullsized":"2024-05-20T20:02:05.154273+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751304+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:02:04.751304+0000","last_peered":"2024-05-20T20:02:04.751304+0000","last_clean":"2024-05-20T20:02:04.751304+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:02:04.751304+0000","last_undegraded":"2024-05-20T20:02:04.751304+0000","last_fullsized":"2024-05-20T20:02:04.751304+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.754216+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:02:04.754216+0000","last_peered":"2024-05-20T20:02:04.754216+0000","last_clean":"2024-05-20T20:02:04.754216+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:02:04.754216+0000","last_undegraded":"2024-05-20T20:02:04.754216+0000","last_fullsized":"2024-05-20T20:02:04.754216+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753745+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:02:04.753745+0000","last_peered":"2024-05-20T20:02:04.753745+0000","last_clean":"2024-05-20T20:02:04.753745+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:02:04.753745+0000","last_undegraded":"2024-05-20T20:02:04.753745+0000","last_fullsized":"2024-05-20T20:02:04.753745+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748489+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:02:04.748489+0000","last_peered":"2024-05-20T20:02:04.748489+0000","last_clean":"2024-05-20T20:02:04.748489+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:02:04.748489+0000","last_undegraded":"2024-05-20T20:02:04.748489+0000","last_fullsized":"2024-05-20T20:02:04.748489+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750745+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:02:04.750745+0000","last_peered":"2024-05-20T20:02:04.750745+0000","last_clean":"2024-05-20T20:02:04.750745+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:02:04.750745+0000","last_undegraded":"2024-05-20T20:02:04.750745+0000","last_fullsized":"2024-05-20T20:02:04.750745+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154270+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:02:05.154270+0000","last_peered":"2024-05-20T20:02:05.154270+0000","last_clean":"2024-05-20T20:02:05.154270+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:02:05.154270+0000","last_undegraded":"2024-05-20T20:02:05.154270+0000","last_fullsized":"2024-05-20T20:02:05.154270+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153795+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:02:05.153795+0000","last_peered":"2024-05-20T20:02:05.153795+0000","last_clean":"2024-05-20T20:02:05.153795+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:02:05.153795+0000","last_undegraded":"2024-05-20T20:02:05.153795+0000","last_fullsized":"2024-05-20T20:02:05.153795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751063+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:02:04.751063+0000","last_peered":"2024-05-20T20:02:04.751063+0000","last_clean":"2024-05-20T20:02:04.751063+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:02:04.751063+0000","last_undegraded":"2024-05-20T20:02:04.751063+0000","last_fullsized":"2024-05-20T20:02:04.751063+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":4132576,"data_compressed":248011,"data_compressed_allocated":315392,"data_compressed_original":4079616,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":64891,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":353,"num_read_kb":248,"num_write":243,"num_write_kb":131,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":569344,"data_stored":198930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710748,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":425895,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877906977,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28020,"kb_used_data":1004,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717132,"statfs":{"total":95995035648,"available":95966343168,"internally_reserved":0,"allocated":1028096,"data_stored":1074934,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070503,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32052,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713100,"statfs":{"total":95995035648,"available":95962214400,"internally_reserved":0,"allocated":897024,"data_stored":425757,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266733,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":892928,"data_stored":426135,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863462963,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32100,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713052,"statfs":{"total":95995035648,"available":95962165248,"internally_reserved":0,"allocated":946176,"data_stored":1018653,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626488,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27884,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717268,"statfs":{"total":95995035648,"available":95966482432,"internally_reserved":0,"allocated":880640,"data_stored":425474,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574933,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":488722,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738459,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":901120,"data_stored":426813,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411407,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":484896,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182474,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":897024,"data_stored":425299,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986244,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":868,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":888832,"data_stored":425447,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822718,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32116,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713036,"statfs":{"total":95995035648,"available":95962148864,"internally_reserved":0,"allocated":958464,"data_stored":1018331,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":86016,"data_stored":64087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":86016,"data_stored":60261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":3211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":3533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":59814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:02:22.867 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T20:02:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:22 smithi033 ceph-mon[36179]: pgmap v349: 97 pgs: 97 active+clean; 640 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 10 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T20:02:23.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:22 smithi154 ceph-mon[36314]: pgmap v349: 97 pgs: 97 active+clean; 640 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 1.7 KiB/s wr, 10 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T20:02:23.376 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:23.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:23 smithi028 ceph-mon[31178]: from='client.24643 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:02:24.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:23 smithi154 ceph-mon[36314]: from='client.24643 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:02:24.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:23 smithi033 ceph-mon[36179]: from='client.24643 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:02:25.018 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:24 smithi028 ceph-mon[31178]: pgmap v350: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 2.0 KiB/s wr, 12 op/s 2024-05-20T20:02:25.028 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:24 smithi033 ceph-mon[36179]: pgmap v350: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 2.0 KiB/s wr, 12 op/s 2024-05-20T20:02:25.107 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:24 smithi154 ceph-mon[36314]: pgmap v350: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 2.0 KiB/s wr, 12 op/s 2024-05-20T20:02:25.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.544+0000 7fd1bd000700 1 -- 172.21.15.28:0/2662353763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b810b760 msgr2=0x7fd1b810bb40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:25.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.544+0000 7fd1bd000700 1 --2- 172.21.15.28:0/2662353763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b810b760 0x7fd1b810bb40 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fd1a8009990 tx=0x7fd1a8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.545+0000 7fd1bd000700 1 -- 172.21.15.28:0/2662353763 shutdown_connections 2024-05-20T20:02:25.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.545+0000 7fd1bd000700 1 --2- 172.21.15.28:0/2662353763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd1b8106120 0x7fd1b8112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.545+0000 7fd1bd000700 1 --2- 172.21.15.28:0/2662353763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b810b760 0x7fd1b810bb40 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.545+0000 7fd1bd000700 1 --2- 172.21.15.28:0/2662353763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd1b8105760 0x7fd1b8105be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.545+0000 7fd1bd000700 1 -- 172.21.15.28:0/2662353763 >> 172.21.15.28:0/2662353763 conn(0x7fd1b8101250 msgr2=0x7fd1b8103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:25.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.545+0000 7fd1bd000700 1 -- 172.21.15.28:0/2662353763 shutdown_connections 2024-05-20T20:02:25.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.545+0000 7fd1bd000700 1 -- 172.21.15.28:0/2662353763 wait complete. 2024-05-20T20:02:25.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.547+0000 7fd1bd000700 1 Processor -- start 2024-05-20T20:02:25.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.547+0000 7fd1bd000700 1 -- start start 2024-05-20T20:02:25.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1bd000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b8105760 0x7fd1b819be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:25.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1bd000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd1b8106120 0x7fd1b819c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:25.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1bd000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd1b810b760 0x7fd1b81a0730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:25.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1bd000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd1b81159b0 con 0x7fd1b810b760 2024-05-20T20:02:25.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1bd000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd1b8115830 con 0x7fd1b8106120 2024-05-20T20:02:25.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1bd000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd1b8115b30 con 0x7fd1b8105760 2024-05-20T20:02:25.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1b5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd1b8106120 0x7fd1b819c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:25.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1b5e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b8105760 0x7fd1b819be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:25.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1b5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd1b8106120 0x7fd1b819c3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57366/0 (socket says 172.21.15.28:57366) 2024-05-20T20:02:25.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1b5400700 1 -- 172.21.15.28:0/4007962227 learned_addr learned my addr 172.21.15.28:0/4007962227 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:25.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.548+0000 7fd1b6800700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd1b810b760 0x7fd1b81a0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1b5e00700 1 -- 172.21.15.28:0/4007962227 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd1b8106120 msgr2=0x7fd1b819c3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1b5e00700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd1b8106120 0x7fd1b819c3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1b5e00700 1 -- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd1b810b760 msgr2=0x7fd1b81a0730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1b5e00700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd1b810b760 0x7fd1b81a0730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1b5e00700 1 -- 172.21.15.28:0/4007962227 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd1b81a0e70 con 0x7fd1b8105760 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1b5e00700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b8105760 0x7fd1b819be60 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fd1a80098e0 tx=0x7fd1a80057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1aea00700 1 -- 172.21.15.28:0/4007962227 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd1a8027070 con 0x7fd1b8105760 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd1b81a1100 con 0x7fd1b8105760 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.549+0000 7fd1aea00700 1 -- 172.21.15.28:0/4007962227 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd1a8022470 con 0x7fd1b8105760 2024-05-20T20:02:25.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.550+0000 7fd1aea00700 1 -- 172.21.15.28:0/4007962227 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd1a8017650 con 0x7fd1b8105760 2024-05-20T20:02:25.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.550+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd1b81a89e0 con 0x7fd1b8105760 2024-05-20T20:02:25.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.551+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd1b810dcf0 con 0x7fd1b8105760 2024-05-20T20:02:25.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.553+0000 7fd1aea00700 1 -- 172.21.15.28:0/4007962227 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fd1a802a070 con 0x7fd1b8105760 2024-05-20T20:02:25.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.555+0000 7fd1aea00700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd19c0b2910 0x7fd19c0b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:25.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.555+0000 7fd1aea00700 1 -- 172.21.15.28:0/4007962227 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fd1a80e34d0 con 0x7fd1b8105760 2024-05-20T20:02:25.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.555+0000 7fd1b5400700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd19c0b2910 0x7fd19c0b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:25.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.556+0000 7fd1b5400700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd19c0b2910 0x7fd19c0b4dd0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fd1b81038a0 tx=0x7fd1a4009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:25.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.557+0000 7fd1aea00700 1 -- 172.21.15.28:0/4007962227 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fd1a8031080 con 0x7fd1b8105760 2024-05-20T20:02:25.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.792+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fd1b819d1c0 con 0x7fd19c0b2910 2024-05-20T20:02:25.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.805+0000 7fd1aea00700 1 -- 172.21.15.28:0/4007962227 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258987 (secure 0 0 0) 0x7fd1b819d1c0 con 0x7fd19c0b2910 2024-05-20T20:02:25.810 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd19c0b2910 msgr2=0x7fd19c0b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd19c0b2910 0x7fd19c0b4dd0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fd1b81038a0 tx=0x7fd1a4009260 comp rx=0 tx=0).stop 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b8105760 msgr2=0x7fd1b819be60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b8105760 0x7fd1b819be60 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fd1a80098e0 tx=0x7fd1a80057c0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 shutdown_connections 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd19c0b2910 0x7fd19c0b4dd0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd1b810b760 0x7fd1b81a0730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd1b8105760 0x7fd1b819be60 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 --2- 172.21.15.28:0/4007962227 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd1b8106120 0x7fd1b819c3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.811+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 >> 172.21.15.28:0/4007962227 conn(0x7fd1b8101250 msgr2=0x7fd1b8101a10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.812+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 shutdown_connections 2024-05-20T20:02:25.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:25.812+0000 7fd1bd000700 1 -- 172.21.15.28:0/4007962227 wait complete. 2024-05-20T20:02:25.813 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:02:26.629 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":351,"stamp":"2024-05-20T20:02:25.394575+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":659750,"num_objects":102,"num_object_clones":0,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":0,"num_read":485,"num_read_kb":347,"num_write":369,"num_write_kb":2288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":303,"ondisk_log_size":303,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347520,"kb_used_data":10812,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124594304,"statfs":{"total":1151940427776,"available":1151584567296,"internally_reserved":0,"allocated":11071488,"data_stored":7074870,"data_compressed":177837,"data_compressed_allocated":233472,"data_compressed_original":1966080,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":5395,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":111,"num_read_kb":61,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005411"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751812+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:02:04.751812+0000","last_peered":"2024-05-20T20:02:04.751812+0000","last_clean":"2024-05-20T20:02:04.751812+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:02:04.751812+0000","last_undegraded":"2024-05-20T20:02:04.751812+0000","last_fullsized":"2024-05-20T20:02:04.751812+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770822+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:02:04.770822+0000","last_peered":"2024-05-20T20:02:04.770822+0000","last_clean":"2024-05-20T20:02:04.770822+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:02:04.770822+0000","last_undegraded":"2024-05-20T20:02:04.770822+0000","last_fullsized":"2024-05-20T20:02:04.770822+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770707+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:02:04.770707+0000","last_peered":"2024-05-20T20:02:04.770707+0000","last_clean":"2024-05-20T20:02:04.770707+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:02:04.770707+0000","last_undegraded":"2024-05-20T20:02:04.770707+0000","last_fullsized":"2024-05-20T20:02:04.770707+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154915+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:02:05.154915+0000","last_peered":"2024-05-20T20:02:05.154915+0000","last_clean":"2024-05-20T20:02:05.154915+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:02:05.154915+0000","last_undegraded":"2024-05-20T20:02:05.154915+0000","last_fullsized":"2024-05-20T20:02:05.154915+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751552+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:02:04.751552+0000","last_peered":"2024-05-20T20:02:04.751552+0000","last_clean":"2024-05-20T20:02:04.751552+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:02:04.751552+0000","last_undegraded":"2024-05-20T20:02:04.751552+0000","last_fullsized":"2024-05-20T20:02:04.751552+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.762828+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:02:04.762828+0000","last_peered":"2024-05-20T20:02:04.762828+0000","last_clean":"2024-05-20T20:02:04.762828+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:02:04.762828+0000","last_undegraded":"2024-05-20T20:02:04.762828+0000","last_fullsized":"2024-05-20T20:02:04.762828+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930220+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:02:04.930220+0000","last_peered":"2024-05-20T20:02:04.930220+0000","last_clean":"2024-05-20T20:02:04.930220+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:02:04.930220+0000","last_undegraded":"2024-05-20T20:02:04.930220+0000","last_fullsized":"2024-05-20T20:02:04.930220+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750682+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:02:04.750682+0000","last_peered":"2024-05-20T20:02:04.750682+0000","last_clean":"2024-05-20T20:02:04.750682+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:02:04.750682+0000","last_undegraded":"2024-05-20T20:02:04.750682+0000","last_fullsized":"2024-05-20T20:02:04.750682+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154551+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:02:05.154551+0000","last_peered":"2024-05-20T20:02:05.154551+0000","last_clean":"2024-05-20T20:02:05.154551+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:02:05.154551+0000","last_undegraded":"2024-05-20T20:02:05.154551+0000","last_fullsized":"2024-05-20T20:02:05.154551+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.763055+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:02:04.763055+0000","last_peered":"2024-05-20T20:02:04.763055+0000","last_clean":"2024-05-20T20:02:04.763055+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:02:04.763055+0000","last_undegraded":"2024-05-20T20:02:04.763055+0000","last_fullsized":"2024-05-20T20:02:04.763055+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930302+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:02:04.930302+0000","last_peered":"2024-05-20T20:02:04.930302+0000","last_clean":"2024-05-20T20:02:04.930302+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:02:04.930302+0000","last_undegraded":"2024-05-20T20:02:04.930302+0000","last_fullsized":"2024-05-20T20:02:04.930302+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154923+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:02:05.154923+0000","last_peered":"2024-05-20T20:02:05.154923+0000","last_clean":"2024-05-20T20:02:05.154923+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:02:05.154923+0000","last_undegraded":"2024-05-20T20:02:05.154923+0000","last_fullsized":"2024-05-20T20:02:05.154923+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153908+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:02:05.153908+0000","last_peered":"2024-05-20T20:02:05.153908+0000","last_clean":"2024-05-20T20:02:05.153908+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:02:05.153908+0000","last_undegraded":"2024-05-20T20:02:05.153908+0000","last_fullsized":"2024-05-20T20:02:05.153908+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.769576+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:02:04.769576+0000","last_peered":"2024-05-20T20:02:04.769576+0000","last_clean":"2024-05-20T20:02:04.769576+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:02:04.769576+0000","last_undegraded":"2024-05-20T20:02:04.769576+0000","last_fullsized":"2024-05-20T20:02:04.769576+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.769541+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:02:04.769541+0000","last_peered":"2024-05-20T20:02:04.769541+0000","last_clean":"2024-05-20T20:02:04.769541+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:02:04.769541+0000","last_undegraded":"2024-05-20T20:02:04.769541+0000","last_fullsized":"2024-05-20T20:02:04.769541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153684+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:02:05.153684+0000","last_peered":"2024-05-20T20:02:05.153684+0000","last_clean":"2024-05-20T20:02:05.153684+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:02:05.153684+0000","last_undegraded":"2024-05-20T20:02:05.153684+0000","last_fullsized":"2024-05-20T20:02:05.153684+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752019+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:02:04.752019+0000","last_peered":"2024-05-20T20:02:04.752019+0000","last_clean":"2024-05-20T20:02:04.752019+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:02:04.752019+0000","last_undegraded":"2024-05-20T20:02:04.752019+0000","last_fullsized":"2024-05-20T20:02:04.752019+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.759389+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:02:04.759389+0000","last_peered":"2024-05-20T20:02:04.759389+0000","last_clean":"2024-05-20T20:02:04.759389+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:02:04.759389+0000","last_undegraded":"2024-05-20T20:02:04.759389+0000","last_fullsized":"2024-05-20T20:02:04.759389+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.754131+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:02:04.754131+0000","last_peered":"2024-05-20T20:02:04.754131+0000","last_clean":"2024-05-20T20:02:04.754131+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:02:04.754131+0000","last_undegraded":"2024-05-20T20:02:04.754131+0000","last_fullsized":"2024-05-20T20:02:04.754131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"78'13","reported_seq":78,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:22.754685+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:02:22.754685+0000","last_peered":"2024-05-20T20:02:22.754685+0000","last_clean":"2024-05-20T20:02:22.754685+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:02:22.754685+0000","last_undegraded":"2024-05-20T20:02:22.754685+0000","last_fullsized":"2024-05-20T20:02:22.754685+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":39,"num_read_kb":39,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752457+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:02:04.752457+0000","last_peered":"2024-05-20T20:02:04.752457+0000","last_clean":"2024-05-20T20:02:04.752457+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:02:04.752457+0000","last_undegraded":"2024-05-20T20:02:04.752457+0000","last_fullsized":"2024-05-20T20:02:04.752457+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753101+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:02:04.753101+0000","last_peered":"2024-05-20T20:02:04.753101+0000","last_clean":"2024-05-20T20:02:04.753101+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:02:04.753101+0000","last_undegraded":"2024-05-20T20:02:04.753101+0000","last_fullsized":"2024-05-20T20:02:04.753101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770833+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:02:04.770833+0000","last_peered":"2024-05-20T20:02:04.770833+0000","last_clean":"2024-05-20T20:02:04.770833+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:02:04.770833+0000","last_undegraded":"2024-05-20T20:02:04.770833+0000","last_fullsized":"2024-05-20T20:02:04.770833+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"78'12","reported_seq":39,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752192+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:02:04.752192+0000","last_peered":"2024-05-20T20:02:04.752192+0000","last_clean":"2024-05-20T20:02:04.752192+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:02:04.752192+0000","last_undegraded":"2024-05-20T20:02:04.752192+0000","last_fullsized":"2024-05-20T20:02:04.752192+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154230+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:02:05.154230+0000","last_peered":"2024-05-20T20:02:05.154230+0000","last_clean":"2024-05-20T20:02:05.154230+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:02:05.154230+0000","last_undegraded":"2024-05-20T20:02:05.154230+0000","last_fullsized":"2024-05-20T20:02:05.154230+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751794+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:02:04.751794+0000","last_peered":"2024-05-20T20:02:04.751794+0000","last_clean":"2024-05-20T20:02:04.751794+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:02:04.751794+0000","last_undegraded":"2024-05-20T20:02:04.751794+0000","last_fullsized":"2024-05-20T20:02:04.751794+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.159343+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:02:05.159343+0000","last_peered":"2024-05-20T20:02:05.159343+0000","last_clean":"2024-05-20T20:02:05.159343+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:02:05.159343+0000","last_undegraded":"2024-05-20T20:02:05.159343+0000","last_fullsized":"2024-05-20T20:02:05.159343+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"78'6","reported_seq":71,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:20.721107+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:02:20.721107+0000","last_peered":"2024-05-20T20:02:20.721107+0000","last_clean":"2024-05-20T20:02:20.721107+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:02:20.721107+0000","last_undegraded":"2024-05-20T20:02:20.721107+0000","last_fullsized":"2024-05-20T20:02:20.721107+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1013,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":3,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.931600+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:02:04.931600+0000","last_peered":"2024-05-20T20:02:04.931600+0000","last_clean":"2024-05-20T20:02:04.931600+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:02:04.931600+0000","last_undegraded":"2024-05-20T20:02:04.931600+0000","last_fullsized":"2024-05-20T20:02:04.931600+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748817+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:02:04.748817+0000","last_peered":"2024-05-20T20:02:04.748817+0000","last_clean":"2024-05-20T20:02:04.748817+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:02:04.748817+0000","last_undegraded":"2024-05-20T20:02:04.748817+0000","last_fullsized":"2024-05-20T20:02:04.748817+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748647+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:02:04.748647+0000","last_peered":"2024-05-20T20:02:04.748647+0000","last_clean":"2024-05-20T20:02:04.748647+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:02:04.748647+0000","last_undegraded":"2024-05-20T20:02:04.748647+0000","last_fullsized":"2024-05-20T20:02:04.748647+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153602+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:02:05.153602+0000","last_peered":"2024-05-20T20:02:05.153602+0000","last_clean":"2024-05-20T20:02:05.153602+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:02:05.153602+0000","last_undegraded":"2024-05-20T20:02:05.153602+0000","last_fullsized":"2024-05-20T20:02:05.153602+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750852+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:02:04.750852+0000","last_peered":"2024-05-20T20:02:04.750852+0000","last_clean":"2024-05-20T20:02:04.750852+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:02:04.750852+0000","last_undegraded":"2024-05-20T20:02:04.750852+0000","last_fullsized":"2024-05-20T20:02:04.750852+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748365+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:02:04.748365+0000","last_peered":"2024-05-20T20:02:04.748365+0000","last_clean":"2024-05-20T20:02:04.748365+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:02:04.748365+0000","last_undegraded":"2024-05-20T20:02:04.748365+0000","last_fullsized":"2024-05-20T20:02:04.748365+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753519+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:02:04.753519+0000","last_peered":"2024-05-20T20:02:04.753519+0000","last_clean":"2024-05-20T20:02:04.753519+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:02:04.753519+0000","last_undegraded":"2024-05-20T20:02:04.753519+0000","last_fullsized":"2024-05-20T20:02:04.753519+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"78'4","reported_seq":102,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:21.721829+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:02:21.721829+0000","last_peered":"2024-05-20T20:02:21.721829+0000","last_clean":"2024-05-20T20:02:21.721829+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:02:21.721829+0000","last_undegraded":"2024-05-20T20:02:21.721829+0000","last_fullsized":"2024-05-20T20:02:21.721829+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":72,"num_read_kb":72,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154065+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:02:05.154065+0000","last_peered":"2024-05-20T20:02:05.154065+0000","last_clean":"2024-05-20T20:02:05.154065+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:02:05.154065+0000","last_undegraded":"2024-05-20T20:02:05.154065+0000","last_fullsized":"2024-05-20T20:02:05.154065+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930524+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:02:04.930524+0000","last_peered":"2024-05-20T20:02:04.930524+0000","last_clean":"2024-05-20T20:02:04.930524+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:02:04.930524+0000","last_undegraded":"2024-05-20T20:02:04.930524+0000","last_fullsized":"2024-05-20T20:02:04.930524+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.771192+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:02:04.771192+0000","last_peered":"2024-05-20T20:02:04.771192+0000","last_clean":"2024-05-20T20:02:04.771192+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:02:04.771192+0000","last_undegraded":"2024-05-20T20:02:04.771192+0000","last_fullsized":"2024-05-20T20:02:04.771192+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.749018+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:02:04.749018+0000","last_peered":"2024-05-20T20:02:04.749018+0000","last_clean":"2024-05-20T20:02:04.749018+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:02:04.749018+0000","last_undegraded":"2024-05-20T20:02:04.749018+0000","last_fullsized":"2024-05-20T20:02:04.749018+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154455+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:02:05.154455+0000","last_peered":"2024-05-20T20:02:05.154455+0000","last_clean":"2024-05-20T20:02:05.154455+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:02:05.154455+0000","last_undegraded":"2024-05-20T20:02:05.154455+0000","last_fullsized":"2024-05-20T20:02:05.154455+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"80'31","reported_seq":138,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:21.721828+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:02:21.721828+0000","last_peered":"2024-05-20T20:02:21.721828+0000","last_clean":"2024-05-20T20:02:21.721828+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:02:21.721828+0000","last_undegraded":"2024-05-20T20:02:21.721828+0000","last_fullsized":"2024-05-20T20:02:21.721828+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":31,"log_dups_size":0,"ondisk_log_size":31,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62335,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":81,"num_read_kb":101,"num_write":33,"num_write_kb":71,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751889+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:02:04.751889+0000","last_peered":"2024-05-20T20:02:04.751889+0000","last_clean":"2024-05-20T20:02:04.751889+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:02:04.751889+0000","last_undegraded":"2024-05-20T20:02:04.751889+0000","last_fullsized":"2024-05-20T20:02:04.751889+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.773205+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:02:04.773205+0000","last_peered":"2024-05-20T20:02:04.773205+0000","last_clean":"2024-05-20T20:02:04.773205+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:02:04.773205+0000","last_undegraded":"2024-05-20T20:02:04.773205+0000","last_fullsized":"2024-05-20T20:02:04.773205+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153440+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:02:05.153440+0000","last_peered":"2024-05-20T20:02:05.153440+0000","last_clean":"2024-05-20T20:02:05.153440+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:02:05.153440+0000","last_undegraded":"2024-05-20T20:02:05.153440+0000","last_fullsized":"2024-05-20T20:02:05.153440+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748643+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:02:04.748643+0000","last_peered":"2024-05-20T20:02:04.748643+0000","last_clean":"2024-05-20T20:02:04.748643+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:02:04.748643+0000","last_undegraded":"2024-05-20T20:02:04.748643+0000","last_fullsized":"2024-05-20T20:02:04.748643+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930547+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:02:04.930547+0000","last_peered":"2024-05-20T20:02:04.930547+0000","last_clean":"2024-05-20T20:02:04.930547+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:02:04.930547+0000","last_undegraded":"2024-05-20T20:02:04.930547+0000","last_fullsized":"2024-05-20T20:02:04.930547+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"78'5","reported_seq":69,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:20.751394+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:02:20.751394+0000","last_peered":"2024-05-20T20:02:20.751394+0000","last_clean":"2024-05-20T20:02:20.751394+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:02:20.751394+0000","last_undegraded":"2024-05-20T20:02:20.751394+0000","last_fullsized":"2024-05-20T20:02:20.751394+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":39,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.929973+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:02:04.929973+0000","last_peered":"2024-05-20T20:02:04.929973+0000","last_clean":"2024-05-20T20:02:04.929973+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:02:04.929973+0000","last_undegraded":"2024-05-20T20:02:04.929973+0000","last_fullsized":"2024-05-20T20:02:04.929973+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":68,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.169342+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:02:05.169342+0000","last_peered":"2024-05-20T20:02:05.169342+0000","last_clean":"2024-05-20T20:02:05.169342+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:02:05.169342+0000","last_undegraded":"2024-05-20T20:02:05.169342+0000","last_fullsized":"2024-05-20T20:02:05.169342+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748542+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:02:04.748542+0000","last_peered":"2024-05-20T20:02:04.748542+0000","last_clean":"2024-05-20T20:02:04.748542+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:02:04.748542+0000","last_undegraded":"2024-05-20T20:02:04.748542+0000","last_fullsized":"2024-05-20T20:02:04.748542+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"78'2","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752142+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:02:04.752142+0000","last_peered":"2024-05-20T20:02:04.752142+0000","last_clean":"2024-05-20T20:02:04.752142+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:02:04.752142+0000","last_undegraded":"2024-05-20T20:02:04.752142+0000","last_fullsized":"2024-05-20T20:02:04.752142+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751702+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:02:04.751702+0000","last_peered":"2024-05-20T20:02:04.751702+0000","last_clean":"2024-05-20T20:02:04.751702+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:02:04.751702+0000","last_undegraded":"2024-05-20T20:02:04.751702+0000","last_fullsized":"2024-05-20T20:02:04.751702+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.757043+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:02:04.757043+0000","last_peered":"2024-05-20T20:02:04.757043+0000","last_clean":"2024-05-20T20:02:04.757043+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:02:04.757043+0000","last_undegraded":"2024-05-20T20:02:04.757043+0000","last_fullsized":"2024-05-20T20:02:04.757043+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"78'1","reported_seq":21,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.749048+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:02:04.749048+0000","last_peered":"2024-05-20T20:02:04.749048+0000","last_clean":"2024-05-20T20:02:04.749048+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:02:04.749048+0000","last_undegraded":"2024-05-20T20:02:04.749048+0000","last_fullsized":"2024-05-20T20:02:04.749048+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"78'5","reported_seq":70,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:22.755847+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:02:22.755847+0000","last_peered":"2024-05-20T20:02:22.755847+0000","last_clean":"2024-05-20T20:02:22.755847+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:02:22.755847+0000","last_undegraded":"2024-05-20T20:02:22.755847+0000","last_fullsized":"2024-05-20T20:02:22.755847+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748898+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:02:04.748898+0000","last_peered":"2024-05-20T20:02:04.748898+0000","last_clean":"2024-05-20T20:02:04.748898+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:02:04.748898+0000","last_undegraded":"2024-05-20T20:02:04.748898+0000","last_fullsized":"2024-05-20T20:02:04.748898+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748775+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:02:04.748775+0000","last_peered":"2024-05-20T20:02:04.748775+0000","last_clean":"2024-05-20T20:02:04.748775+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:02:04.748775+0000","last_undegraded":"2024-05-20T20:02:04.748775+0000","last_fullsized":"2024-05-20T20:02:04.748775+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"78'5","reported_seq":68,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:20.750289+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:02:20.750289+0000","last_peered":"2024-05-20T20:02:20.750289+0000","last_clean":"2024-05-20T20:02:20.750289+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:02:20.750289+0000","last_undegraded":"2024-05-20T20:02:20.750289+0000","last_fullsized":"2024-05-20T20:02:20.750289+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":37,"num_read_kb":37,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751623+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:02:04.751623+0000","last_peered":"2024-05-20T20:02:04.751623+0000","last_clean":"2024-05-20T20:02:04.751623+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:02:04.751623+0000","last_undegraded":"2024-05-20T20:02:04.751623+0000","last_fullsized":"2024-05-20T20:02:04.751623+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":35,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154587+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:02:05.154587+0000","last_peered":"2024-05-20T20:02:05.154587+0000","last_clean":"2024-05-20T20:02:05.154587+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:02:05.154587+0000","last_undegraded":"2024-05-20T20:02:05.154587+0000","last_fullsized":"2024-05-20T20:02:05.154587+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"20'92","reported_seq":100,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930195+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:02:04.930195+0000","last_peered":"2024-05-20T20:02:04.930195+0000","last_clean":"2024-05-20T20:02:04.930195+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:02:04.930195+0000","last_undegraded":"2024-05-20T20:02:04.930195+0000","last_fullsized":"2024-05-20T20:02:04.930195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153378+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:02:05.153378+0000","last_peered":"2024-05-20T20:02:05.153378+0000","last_clean":"2024-05-20T20:02:05.153378+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:02:05.153378+0000","last_undegraded":"2024-05-20T20:02:05.153378+0000","last_fullsized":"2024-05-20T20:02:05.153378+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930276+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:02:04.930276+0000","last_peered":"2024-05-20T20:02:04.930276+0000","last_clean":"2024-05-20T20:02:04.930276+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:02:04.930276+0000","last_undegraded":"2024-05-20T20:02:04.930276+0000","last_fullsized":"2024-05-20T20:02:04.930276+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752011+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:02:04.752011+0000","last_peered":"2024-05-20T20:02:04.752011+0000","last_clean":"2024-05-20T20:02:04.752011+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:02:04.752011+0000","last_undegraded":"2024-05-20T20:02:04.752011+0000","last_fullsized":"2024-05-20T20:02:04.752011+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154276+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:02:05.154276+0000","last_peered":"2024-05-20T20:02:05.154276+0000","last_clean":"2024-05-20T20:02:05.154276+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:02:05.154276+0000","last_undegraded":"2024-05-20T20:02:05.154276+0000","last_fullsized":"2024-05-20T20:02:05.154276+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"80'15","reported_seq":57,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:11.535003+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:02:11.535003+0000","last_peered":"2024-05-20T20:02:11.535003+0000","last_clean":"2024-05-20T20:02:11.535003+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:02:11.535003+0000","last_undegraded":"2024-05-20T20:02:11.535003+0000","last_fullsized":"2024-05-20T20:02:11.535003+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.754047+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:02:04.754047+0000","last_peered":"2024-05-20T20:02:04.754047+0000","last_clean":"2024-05-20T20:02:04.754047+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:02:04.754047+0000","last_undegraded":"2024-05-20T20:02:04.754047+0000","last_fullsized":"2024-05-20T20:02:04.754047+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":34,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.155314+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:02:05.155314+0000","last_peered":"2024-05-20T20:02:05.155314+0000","last_clean":"2024-05-20T20:02:05.155314+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:02:05.155314+0000","last_undegraded":"2024-05-20T20:02:05.155314+0000","last_fullsized":"2024-05-20T20:02:05.155314+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.155654+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:02:05.155654+0000","last_peered":"2024-05-20T20:02:05.155654+0000","last_clean":"2024-05-20T20:02:05.155654+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:02:05.155654+0000","last_undegraded":"2024-05-20T20:02:05.155654+0000","last_fullsized":"2024-05-20T20:02:05.155654+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153826+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:02:05.153826+0000","last_peered":"2024-05-20T20:02:05.153826+0000","last_clean":"2024-05-20T20:02:05.153826+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:02:05.153826+0000","last_undegraded":"2024-05-20T20:02:05.153826+0000","last_fullsized":"2024-05-20T20:02:05.153826+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751497+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:02:04.751497+0000","last_peered":"2024-05-20T20:02:04.751497+0000","last_clean":"2024-05-20T20:02:04.751497+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:02:04.751497+0000","last_undegraded":"2024-05-20T20:02:04.751497+0000","last_fullsized":"2024-05-20T20:02:04.751497+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750932+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:02:04.750932+0000","last_peered":"2024-05-20T20:02:04.750932+0000","last_clean":"2024-05-20T20:02:04.750932+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:02:04.750932+0000","last_undegraded":"2024-05-20T20:02:04.750932+0000","last_fullsized":"2024-05-20T20:02:04.750932+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750668+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:02:04.750668+0000","last_peered":"2024-05-20T20:02:04.750668+0000","last_clean":"2024-05-20T20:02:04.750668+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:02:04.750668+0000","last_undegraded":"2024-05-20T20:02:04.750668+0000","last_fullsized":"2024-05-20T20:02:04.750668+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753643+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:02:04.753643+0000","last_peered":"2024-05-20T20:02:04.753643+0000","last_clean":"2024-05-20T20:02:04.753643+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:02:04.753643+0000","last_undegraded":"2024-05-20T20:02:04.753643+0000","last_fullsized":"2024-05-20T20:02:04.753643+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751959+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:02:04.751959+0000","last_peered":"2024-05-20T20:02:04.751959+0000","last_clean":"2024-05-20T20:02:04.751959+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:02:04.751959+0000","last_undegraded":"2024-05-20T20:02:04.751959+0000","last_fullsized":"2024-05-20T20:02:04.751959+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751646+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:02:04.751646+0000","last_peered":"2024-05-20T20:02:04.751646+0000","last_clean":"2024-05-20T20:02:04.751646+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:02:04.751646+0000","last_undegraded":"2024-05-20T20:02:04.751646+0000","last_fullsized":"2024-05-20T20:02:04.751646+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930403+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:02:04.930403+0000","last_peered":"2024-05-20T20:02:04.930403+0000","last_clean":"2024-05-20T20:02:04.930403+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:02:04.930403+0000","last_undegraded":"2024-05-20T20:02:04.930403+0000","last_fullsized":"2024-05-20T20:02:04.930403+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"78'5","reported_seq":69,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:20.720994+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:02:20.720994+0000","last_peered":"2024-05-20T20:02:20.720994+0000","last_clean":"2024-05-20T20:02:20.720994+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:02:20.720994+0000","last_undegraded":"2024-05-20T20:02:20.720994+0000","last_fullsized":"2024-05-20T20:02:20.720994+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1170,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":38,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930431+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:02:04.930431+0000","last_peered":"2024-05-20T20:02:04.930431+0000","last_clean":"2024-05-20T20:02:04.930431+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:02:04.930431+0000","last_undegraded":"2024-05-20T20:02:04.930431+0000","last_fullsized":"2024-05-20T20:02:04.930431+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748758+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:02:04.748758+0000","last_peered":"2024-05-20T20:02:04.748758+0000","last_clean":"2024-05-20T20:02:04.748758+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:02:04.748758+0000","last_undegraded":"2024-05-20T20:02:04.748758+0000","last_fullsized":"2024-05-20T20:02:04.748758+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'1","reported_seq":21,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930321+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:02:04.930321+0000","last_peered":"2024-05-20T20:02:04.930321+0000","last_clean":"2024-05-20T20:02:04.930321+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:02:04.930321+0000","last_undegraded":"2024-05-20T20:02:04.930321+0000","last_fullsized":"2024-05-20T20:02:04.930321+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154077+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:02:05.154077+0000","last_peered":"2024-05-20T20:02:05.154077+0000","last_clean":"2024-05-20T20:02:05.154077+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:02:05.154077+0000","last_undegraded":"2024-05-20T20:02:05.154077+0000","last_fullsized":"2024-05-20T20:02:05.154077+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930618+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:02:04.930618+0000","last_peered":"2024-05-20T20:02:04.930618+0000","last_clean":"2024-05-20T20:02:04.930618+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:02:04.930618+0000","last_undegraded":"2024-05-20T20:02:04.930618+0000","last_fullsized":"2024-05-20T20:02:04.930618+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750870+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:02:04.750870+0000","last_peered":"2024-05-20T20:02:04.750870+0000","last_clean":"2024-05-20T20:02:04.750870+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:02:04.750870+0000","last_undegraded":"2024-05-20T20:02:04.750870+0000","last_fullsized":"2024-05-20T20:02:04.750870+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753730+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:02:04.753730+0000","last_peered":"2024-05-20T20:02:04.753730+0000","last_clean":"2024-05-20T20:02:04.753730+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:02:04.753730+0000","last_undegraded":"2024-05-20T20:02:04.753730+0000","last_fullsized":"2024-05-20T20:02:04.753730+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770679+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:02:04.770679+0000","last_peered":"2024-05-20T20:02:04.770679+0000","last_clean":"2024-05-20T20:02:04.770679+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:02:04.770679+0000","last_undegraded":"2024-05-20T20:02:04.770679+0000","last_fullsized":"2024-05-20T20:02:04.770679+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.770231+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:02:04.770231+0000","last_peered":"2024-05-20T20:02:04.770231+0000","last_clean":"2024-05-20T20:02:04.770231+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:02:04.770231+0000","last_undegraded":"2024-05-20T20:02:04.770231+0000","last_fullsized":"2024-05-20T20:02:04.770231+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154273+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:02:05.154273+0000","last_peered":"2024-05-20T20:02:05.154273+0000","last_clean":"2024-05-20T20:02:05.154273+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:02:05.154273+0000","last_undegraded":"2024-05-20T20:02:05.154273+0000","last_fullsized":"2024-05-20T20:02:05.154273+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751304+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:02:04.751304+0000","last_peered":"2024-05-20T20:02:04.751304+0000","last_clean":"2024-05-20T20:02:04.751304+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:02:04.751304+0000","last_undegraded":"2024-05-20T20:02:04.751304+0000","last_fullsized":"2024-05-20T20:02:04.751304+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.754216+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:02:04.754216+0000","last_peered":"2024-05-20T20:02:04.754216+0000","last_clean":"2024-05-20T20:02:04.754216+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:02:04.754216+0000","last_undegraded":"2024-05-20T20:02:04.754216+0000","last_fullsized":"2024-05-20T20:02:04.754216+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753745+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:02:04.753745+0000","last_peered":"2024-05-20T20:02:04.753745+0000","last_clean":"2024-05-20T20:02:04.753745+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:02:04.753745+0000","last_undegraded":"2024-05-20T20:02:04.753745+0000","last_fullsized":"2024-05-20T20:02:04.753745+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748489+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:02:04.748489+0000","last_peered":"2024-05-20T20:02:04.748489+0000","last_clean":"2024-05-20T20:02:04.748489+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:02:04.748489+0000","last_undegraded":"2024-05-20T20:02:04.748489+0000","last_fullsized":"2024-05-20T20:02:04.748489+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.750745+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:02:04.750745+0000","last_peered":"2024-05-20T20:02:04.750745+0000","last_clean":"2024-05-20T20:02:04.750745+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:02:04.750745+0000","last_undegraded":"2024-05-20T20:02:04.750745+0000","last_fullsized":"2024-05-20T20:02:04.750745+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154270+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:02:05.154270+0000","last_peered":"2024-05-20T20:02:05.154270+0000","last_clean":"2024-05-20T20:02:05.154270+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:02:05.154270+0000","last_undegraded":"2024-05-20T20:02:05.154270+0000","last_fullsized":"2024-05-20T20:02:05.154270+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153795+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:02:05.153795+0000","last_peered":"2024-05-20T20:02:05.153795+0000","last_clean":"2024-05-20T20:02:05.153795+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:02:05.153795+0000","last_undegraded":"2024-05-20T20:02:05.153795+0000","last_fullsized":"2024-05-20T20:02:05.153795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":20,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751063+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:02:04.751063+0000","last_peered":"2024-05-20T20:02:04.751063+0000","last_clean":"2024-05-20T20:02:04.751063+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:02:04.751063+0000","last_undegraded":"2024-05-20T20:02:04.751063+0000","last_fullsized":"2024-05-20T20:02:04.751063+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":4132576,"data_compressed":248011,"data_compressed_allocated":315392,"data_compressed_original":4079616,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":69148,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":404,"num_read_kb":278,"num_write":244,"num_write_kb":136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":577536,"data_stored":207444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":209,"ondisk_log_size":209,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710749,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":425895,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877906978,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28024,"kb_used_data":1008,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717128,"statfs":{"total":95995035648,"available":95966339072,"internally_reserved":0,"allocated":1032192,"data_stored":1079191,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070504,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32052,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713100,"statfs":{"total":95995035648,"available":95962214400,"internally_reserved":0,"allocated":897024,"data_stored":425757,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266734,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":892928,"data_stored":426135,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863462964,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32100,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713052,"statfs":{"total":95995035648,"available":95962165248,"internally_reserved":0,"allocated":946176,"data_stored":1018653,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626489,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27884,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717268,"statfs":{"total":95995035648,"available":95966482432,"internally_reserved":0,"allocated":880640,"data_stored":425474,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574934,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":488722,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738460,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":901120,"data_stored":426813,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411408,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":928,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":950272,"data_stored":489153,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182475,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":897024,"data_stored":425299,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986245,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":868,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":888832,"data_stored":425447,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822719,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32116,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713036,"statfs":{"total":95995035648,"available":95962148864,"internally_reserved":0,"allocated":958464,"data_stored":1018331,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":86016,"data_stored":64087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":64518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":3211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":3533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":64071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:02:26.633 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-20T20:02:26.633 INFO:tasks.cephadm:Setting up client nodes... 2024-05-20T20:02:26.634 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-20T20:02:27.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:26 smithi028 ceph-mon[31178]: pgmap v351: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 682 B/s wr, 9 op/s 2024-05-20T20:02:27.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:26 smithi028 ceph-mon[31178]: from='client.24649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:02:27.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:26 smithi154 ceph-mon[36314]: pgmap v351: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 682 B/s wr, 9 op/s 2024-05-20T20:02:27.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:26 smithi154 ceph-mon[36314]: from='client.24649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:02:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:26 smithi033 ceph-mon[36179]: pgmap v351: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 682 B/s wr, 9 op/s 2024-05-20T20:02:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:26 smithi033 ceph-mon[36179]: from='client.24649 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:02:27.162 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:29.011 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:28 smithi028 ceph-mon[31178]: pgmap v352: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 682 B/s wr, 9 op/s 2024-05-20T20:02:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:28 smithi154 ceph-mon[36314]: pgmap v352: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 682 B/s wr, 9 op/s 2024-05-20T20:02:29.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:28 smithi033 ceph-mon[36179]: pgmap v352: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 682 B/s wr, 9 op/s 2024-05-20T20:02:29.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.588+0000 7f8474a00700 1 -- 172.21.15.28:0/2701246370 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f84701090f0 msgr2=0x7f847010b560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:29.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.588+0000 7f8474a00700 1 --2- 172.21.15.28:0/2701246370 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f84701090f0 0x7f847010b560 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f846800b0a0 tx=0x7f846800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.589+0000 7f8474a00700 1 -- 172.21.15.28:0/2701246370 shutdown_connections 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.589+0000 7f8474a00700 1 --2- 172.21.15.28:0/2701246370 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f84701090f0 0x7f847010b560 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.589+0000 7f8474a00700 1 --2- 172.21.15.28:0/2701246370 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8470103e90 0x7f8470106280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.589+0000 7f8474a00700 1 --2- 172.21.15.28:0/2701246370 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 0x7f8470108bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.589+0000 7f8474a00700 1 -- 172.21.15.28:0/2701246370 >> 172.21.15.28:0/2701246370 conn(0x7f84700fd8c0 msgr2=0x7f84700ffce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.589+0000 7f8474a00700 1 -- 172.21.15.28:0/2701246370 shutdown_connections 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.589+0000 7f8474a00700 1 -- 172.21.15.28:0/2701246370 wait complete. 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.590+0000 7f8474a00700 1 Processor -- start 2024-05-20T20:02:29.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.591+0000 7f8474a00700 1 -- start start 2024-05-20T20:02:29.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.591+0000 7f8474a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8470103e90 0x7f8470199420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:29.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.591+0000 7f8474a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 0x7f8470199960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:29.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.591+0000 7f8474a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f84701090f0 0x7f84701a09e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:29.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.591+0000 7f8474a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f847010ee70 con 0x7f8470103e90 2024-05-20T20:02:29.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.591+0000 7f8474a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f847010ecf0 con 0x7f84701090f0 2024-05-20T20:02:29.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.591+0000 7f8474a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f847010eff0 con 0x7f84701067c0 2024-05-20T20:02:29.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 0x7f8470199960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:29.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 0x7f8470199960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:36308/0 (socket says 172.21.15.28:36308) 2024-05-20T20:02:29.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f84701090f0 0x7f84701a09e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 -- 172.21.15.28:0/1475689267 learned_addr learned my addr 172.21.15.28:0/1475689267 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846f400700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8470103e90 0x7f8470199420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 -- 172.21.15.28:0/1475689267 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f84701090f0 msgr2=0x7f84701a09e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f84701090f0 0x7f84701a09e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 -- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8470103e90 msgr2=0x7f8470199420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8470103e90 0x7f8470199420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 -- 172.21.15.28:0/1475689267 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f84701a1080 con 0x7f84701067c0 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846fe00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f84701090f0 0x7f84701a09e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:29.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.592+0000 7f846ea00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 0x7f8470199960 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f845c00e9d0 tx=0x7f845c00ece0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:29.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.593+0000 7f8467e00700 1 -- 172.21.15.28:0/1475689267 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f845c00cbd0 con 0x7f84701067c0 2024-05-20T20:02:29.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.593+0000 7f8467e00700 1 -- 172.21.15.28:0/1475689267 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f845c00ee70 con 0x7f84701067c0 2024-05-20T20:02:29.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.593+0000 7f8467e00700 1 -- 172.21.15.28:0/1475689267 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f845c018680 con 0x7f84701067c0 2024-05-20T20:02:29.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.593+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f84701a1370 con 0x7f84701067c0 2024-05-20T20:02:29.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.593+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f84701a1860 con 0x7f84701067c0 2024-05-20T20:02:29.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.594+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8470064ba0 con 0x7f84701067c0 2024-05-20T20:02:29.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.598+0000 7f8467e00700 1 -- 172.21.15.28:0/1475689267 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f845c010430 con 0x7f84701067c0 2024-05-20T20:02:29.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.599+0000 7f8467e00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84540b2a30 0x7f84540b4ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:29.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.599+0000 7f8467e00700 1 -- 172.21.15.28:0/1475689267 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f845c0e2860 con 0x7f84701067c0 2024-05-20T20:02:29.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.599+0000 7f846f400700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84540b2a30 0x7f84540b4ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:29.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.600+0000 7f8467e00700 1 -- 172.21.15.28:0/1475689267 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f845c0aa3e0 con 0x7f84701067c0 2024-05-20T20:02:29.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.600+0000 7f846f400700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84540b2a30 0x7f84540b4ef0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f8460009c60 tx=0x7f8460009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:29.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.939+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]} v 0) -- 0x7f8470064150 con 0x7f84701067c0 2024-05-20T20:02:29.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.948+0000 7f8467e00700 1 -- 172.21.15.28:0/1475689267 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]=0 v32) ==== 170+0+59 (secure 0 0 0) 0x7f845c0a9ce0 con 0x7f84701067c0 2024-05-20T20:02:29.949 INFO:teuthology.orchestra.run.smithi028.stdout:[client.0] 2024-05-20T20:02:29.949 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQBVrEtmLD8vOBAAQrTKDKwzWUjVozNkyYjvNQ== 2024-05-20T20:02:29.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84540b2a30 msgr2=0x7f84540b4ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:29.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84540b2a30 0x7f84540b4ef0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f8460009c60 tx=0x7f8460009260 comp rx=0 tx=0).stop 2024-05-20T20:02:29.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 msgr2=0x7f8470199960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:29.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 0x7f8470199960 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f845c00e9d0 tx=0x7f845c00ece0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 shutdown_connections 2024-05-20T20:02:29.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84540b2a30 0x7f84540b4ef0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f84701090f0 0x7f84701a09e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8470103e90 0x7f8470199420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 --2- 172.21.15.28:0/1475689267 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84701067c0 0x7f8470199960 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:29.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.951+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 >> 172.21.15.28:0/1475689267 conn(0x7f84700fd8c0 msgr2=0x7f84701047b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:29.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.952+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 shutdown_connections 2024-05-20T20:02:29.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:29.952+0000 7f8474a00700 1 -- 172.21.15.28:0/1475689267 wait complete. 2024-05-20T20:02:30.849 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T20:02:30.849 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-20T20:02:30.849 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-20T20:02:30.932 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:30 smithi028 ceph-mon[31178]: pgmap v353: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 682 B/s wr, 12 op/s 2024-05-20T20:02:30.932 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:30 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1475689267' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T20:02:30.932 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:30 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T20:02:30.932 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:30 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T20:02:30.997 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-20T20:02:30.997 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-20T20:02:30.997 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph mgr dump --format=json 2024-05-20T20:02:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:30 smithi154 ceph-mon[36314]: pgmap v353: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 682 B/s wr, 12 op/s 2024-05-20T20:02:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:30 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1475689267' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T20:02:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:30 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T20:02:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:30 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T20:02:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:30 smithi033 ceph-mon[36179]: pgmap v353: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 682 B/s wr, 12 op/s 2024-05-20T20:02:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:30 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1475689267' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T20:02:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:30 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T20:02:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:30 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T20:02:31.513 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:32.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:32 smithi028 ceph-mon[31178]: pgmap v354: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 426 B/s wr, 8 op/s 2024-05-20T20:02:33.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:32 smithi154 ceph-mon[36314]: pgmap v354: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 426 B/s wr, 8 op/s 2024-05-20T20:02:33.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:32 smithi033 ceph-mon[36179]: pgmap v354: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 426 B/s wr, 8 op/s 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.895+0000 7ff1a6600700 1 -- 172.21.15.28:0/3577518913 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0109670 msgr2=0x7ff1a0109a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.895+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3577518913 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0109670 0x7ff1a0109a50 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7ff1900098f0 tx=0x7ff190009c00 comp rx=0 tx=0).stop 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.895+0000 7ff1a6600700 1 -- 172.21.15.28:0/3577518913 shutdown_connections 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.895+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3577518913 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1a0103f90 0x7ff1a0110440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.895+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3577518913 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0109670 0x7ff1a0109a50 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.895+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3577518913 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 0x7ff1a0103a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.895+0000 7ff1a6600700 1 -- 172.21.15.28:0/3577518913 >> 172.21.15.28:0/3577518913 conn(0x7ff1a007a730 msgr2=0x7ff1a007ab40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:33.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.896+0000 7ff1a6600700 1 -- 172.21.15.28:0/3577518913 shutdown_connections 2024-05-20T20:02:33.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.896+0000 7ff1a6600700 1 -- 172.21.15.28:0/3577518913 wait complete. 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 Processor -- start 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 -- start start 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 0x7ff1a0199d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0103f90 0x7ff1a019a250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1a0109670 0x7ff1a019e5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff1a01137e0 con 0x7ff1a01035d0 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff1a0113660 con 0x7ff1a0109670 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.897+0000 7ff1a6600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff1a0113960 con 0x7ff1a0103f90 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 0x7ff1a0199d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 0x7ff1a0199d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45972/0 (socket says 172.21.15.28:45972) 2024-05-20T20:02:33.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0103f90 0x7ff1a019a250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:33.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 -- 172.21.15.28:0/3542003876 learned_addr learned my addr 172.21.15.28:0/3542003876 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:33.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19fe00700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1a0109670 0x7ff1a019e5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 -- 172.21.15.28:0/3542003876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0103f90 msgr2=0x7ff1a019a250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0103f90 0x7ff1a019a250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 -- 172.21.15.28:0/3542003876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1a0109670 msgr2=0x7ff1a019e5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1a0109670 0x7ff1a019e5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19f400700 1 -- 172.21.15.28:0/3542003876 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff1a019ed20 con 0x7ff1a01035d0 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19ea00700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0103f90 0x7ff1a019a250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.898+0000 7ff19fe00700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1a0109670 0x7ff1a019e5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.899+0000 7ff19f400700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 0x7ff1a0199d10 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7ff190009230 tx=0x7ff19000fa70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.899+0000 7ff197e00700 1 -- 172.21.15.28:0/3542003876 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff1900050c0 con 0x7ff1a01035d0 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.899+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff1a019efb0 con 0x7ff1a01035d0 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.899+0000 7ff197e00700 1 -- 172.21.15.28:0/3542003876 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff19000b560 con 0x7ff1a01035d0 2024-05-20T20:02:33.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.899+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff1a01a6890 con 0x7ff1a01035d0 2024-05-20T20:02:33.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.899+0000 7ff197e00700 1 -- 172.21.15.28:0/3542003876 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff1900039a0 con 0x7ff1a01035d0 2024-05-20T20:02:33.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.900+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff1a010bb00 con 0x7ff1a01035d0 2024-05-20T20:02:33.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.903+0000 7ff197e00700 1 -- 172.21.15.28:0/3542003876 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7ff19000b710 con 0x7ff1a01035d0 2024-05-20T20:02:33.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.904+0000 7ff197e00700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1840b2910 0x7ff1840b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:33.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.904+0000 7ff197e00700 1 -- 172.21.15.28:0/3542003876 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7ff1900e3440 con 0x7ff1a01035d0 2024-05-20T20:02:33.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.905+0000 7ff19ea00700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1840b2910 0x7ff1840b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:33.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.905+0000 7ff19ea00700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1840b2910 0x7ff1840b4dd0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ff18c009750 tx=0x7ff18c009310 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:33.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:33.910+0000 7ff197e00700 1 -- 172.21.15.28:0/3542003876 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7ff190013030 con 0x7ff1a01035d0 2024-05-20T20:02:34.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.217+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7ff1a019b070 con 0x7ff1a01035d0 2024-05-20T20:02:34.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.235+0000 7ff197e00700 1 -- 172.21.15.28:0/3542003876 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v17) ==== 74+0+288645 (secure 0 0 0) 0x7ff1900ab050 con 0x7ff1a01035d0 2024-05-20T20:02:34.242 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:02:34.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.242+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1840b2910 msgr2=0x7ff1840b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:34.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.242+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1840b2910 0x7ff1840b4dd0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ff18c009750 tx=0x7ff18c009310 comp rx=0 tx=0).stop 2024-05-20T20:02:34.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.242+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 msgr2=0x7ff1a0199d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:34.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.242+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 0x7ff1a0199d10 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7ff190009230 tx=0x7ff19000fa70 comp rx=0 tx=0).stop 2024-05-20T20:02:34.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 shutdown_connections 2024-05-20T20:02:34.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1840b2910 0x7ff1840b4dd0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:34.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1a0109670 0x7ff1a019e5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:34.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1a01035d0 0x7ff1a0199d10 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:34.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 --2- 172.21.15.28:0/3542003876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1a0103f90 0x7ff1a019a250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:34.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 >> 172.21.15.28:0/3542003876 conn(0x7ff1a007a730 msgr2=0x7ff1a01023a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:34.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 shutdown_connections 2024-05-20T20:02:34.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:34.243+0000 7ff1a6600700 1 -- 172.21.15.28:0/3542003876 wait complete. 2024-05-20T20:02:35.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:34 smithi028 ceph-mon[31178]: pgmap v355: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 426 B/s wr, 11 op/s 2024-05-20T20:02:35.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:34 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3542003876' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T20:02:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:34 smithi154 ceph-mon[36314]: pgmap v355: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 426 B/s wr, 11 op/s 2024-05-20T20:02:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:34 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3542003876' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T20:02:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:34 smithi033 ceph-mon[36179]: pgmap v355: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 426 B/s wr, 11 op/s 2024-05-20T20:02:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:34 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3542003876' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T20:02:35.180 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":17,"flags":0,"active_gid":14152,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6800","nonce":2019900521},{"type":"v1","addr":"172.21.15.28:6801","nonce":2019900521}]},"active_addr":"172.21.15.28:6801/2019900521","active_change":"2024-05-20T19:53:03.221374+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":24109,"name":"z","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]},{"gid":24113,"name":"y","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.28:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":1496067961}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":19609679}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":1975849601}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":2759010565}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":2837676556}]}]} 2024-05-20T20:02:35.187 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-20T20:02:35.187 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-20T20:02:35.187 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:02:35.700 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:37.053 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:36 smithi028 ceph-mon[31178]: pgmap v356: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:36 smithi154 ceph-mon[36314]: pgmap v356: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:37.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:36 smithi033 ceph-mon[36179]: pgmap v356: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 -- 172.21.15.28:0/1587107814 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410eeb0 msgr2=0x7f4af4116a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 --2- 172.21.15.28:0/1587107814 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410eeb0 0x7f4af4116a80 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f4aec00b0a0 tx=0x7f4aec00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 -- 172.21.15.28:0/1587107814 shutdown_connections 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 --2- 172.21.15.28:0/1587107814 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410eeb0 0x7f4af4116a80 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 --2- 172.21.15.28:0/1587107814 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 0x7f4af410a1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 --2- 172.21.15.28:0/1587107814 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4af410a780 0x7f4af410e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 -- 172.21.15.28:0/1587107814 >> 172.21.15.28:0/1587107814 conn(0x7f4af4076d10 msgr2=0x7f4af4108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 -- 172.21.15.28:0/1587107814 shutdown_connections 2024-05-20T20:02:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.847+0000 7f4afc200700 1 -- 172.21.15.28:0/1587107814 wait complete. 2024-05-20T20:02:37.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.848+0000 7f4afc200700 1 Processor -- start 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4afc200700 1 -- start start 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4afc200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 0x7f4af41a0200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4afc200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410a780 0x7f4af41a0740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4afc200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4af410eeb0 0x7f4af41a4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4afc200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4af4119db0 con 0x7f4af4109dd0 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4afc200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4af4119c30 con 0x7f4af410a780 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4afc200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4af4119f30 con 0x7f4af410eeb0 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4af9800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 0x7f4af41a0200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4af9800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 0x7f4af41a0200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47740/0 (socket says 172.21.15.28:47740) 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4af8e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410a780 0x7f4af41a0740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.849+0000 7f4af9800700 1 -- 172.21.15.28:0/737539273 learned_addr learned my addr 172.21.15.28:0/737539273 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4afa200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4af410eeb0 0x7f4af41a4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4af9800700 1 -- 172.21.15.28:0/737539273 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4af410eeb0 msgr2=0x7f4af41a4ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4af9800700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4af410eeb0 0x7f4af41a4ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4af9800700 1 -- 172.21.15.28:0/737539273 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410a780 msgr2=0x7f4af41a0740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4af9800700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410a780 0x7f4af41a0740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4af9800700 1 -- 172.21.15.28:0/737539273 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4af41a5210 con 0x7f4af4109dd0 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4afa200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4af410eeb0 0x7f4af41a4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4af8e00700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410a780 0x7f4af41a0740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.850+0000 7f4af9800700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 0x7f4af41a0200 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f4ae800e9a0 tx=0x7f4ae800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.851+0000 7f4af2000700 1 -- 172.21.15.28:0/737539273 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ae800ca10 con 0x7f4af4109dd0 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.851+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4af41a5500 con 0x7f4af4109dd0 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.851+0000 7f4af2000700 1 -- 172.21.15.28:0/737539273 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4ae800cbb0 con 0x7f4af4109dd0 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.851+0000 7f4af2000700 1 -- 172.21.15.28:0/737539273 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4ae8018680 con 0x7f4af4109dd0 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.851+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4af41acd80 con 0x7f4af4109dd0 2024-05-20T20:02:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.852+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4af406a230 con 0x7f4af4109dd0 2024-05-20T20:02:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.854+0000 7f4af2000700 1 -- 172.21.15.28:0/737539273 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f4ae8010430 con 0x7f4af4109dd0 2024-05-20T20:02:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.856+0000 7f4af2000700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4adc0b2910 0x7f4adc0b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.856+0000 7f4af2000700 1 -- 172.21.15.28:0/737539273 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f4ae80e2530 con 0x7f4af4109dd0 2024-05-20T20:02:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.856+0000 7f4af8e00700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4adc0b2910 0x7f4adc0b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:37.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.857+0000 7f4af8e00700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4adc0b2910 0x7f4adc0b4dd0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f4af41092c0 tx=0x7f4ae0009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:37.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:37.860+0000 7f4af2000700 1 -- 172.21.15.28:0/737539273 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f4ae80aa0b0 con 0x7f4af4109dd0 2024-05-20T20:02:38.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.103+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f4af402e560 con 0x7f4af4109dd0 2024-05-20T20:02:38.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.106+0000 7f4af2000700 1 -- 172.21.15.28:0/737539273 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) ==== 74+0+22571 (secure 0 0 0) 0x7f4ae80b2130 con 0x7f4af4109dd0 2024-05-20T20:02:38.107 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:02:38.108 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":80,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:02:04.742448+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:02:38.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4adc0b2910 msgr2=0x7f4adc0b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:38.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4adc0b2910 0x7f4adc0b4dd0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f4af41092c0 tx=0x7f4ae0009260 comp rx=0 tx=0).stop 2024-05-20T20:02:38.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 msgr2=0x7f4af41a0200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 0x7f4af41a0200 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f4ae800e9a0 tx=0x7f4ae800ecb0 comp rx=0 tx=0).stop 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 shutdown_connections 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4adc0b2910 0x7f4adc0b4dd0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4af410eeb0 0x7f4af41a4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4af4109dd0 0x7f4af41a0200 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 --2- 172.21.15.28:0/737539273 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4af410a780 0x7f4af41a0740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 >> 172.21.15.28:0/737539273 conn(0x7f4af4076d10 msgr2=0x7f4af4077250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 shutdown_connections 2024-05-20T20:02:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:38.109+0000 7f4afc200700 1 -- 172.21.15.28:0/737539273 wait complete. 2024-05-20T20:02:38.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:38 smithi028 ceph-mon[31178]: pgmap v357: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:38.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:38 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/737539273' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:39.011 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-20T20:02:39.011 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:02:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:38 smithi154 ceph-mon[36314]: pgmap v357: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:38 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/737539273' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:39.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:38 smithi033 ceph-mon[36179]: pgmap v357: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:39.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:38 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/737539273' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:39.484 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:40 smithi028 ceph-mon[31178]: pgmap v358: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:41.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:40 smithi154 ceph-mon[36314]: pgmap v358: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:40 smithi033 ceph-mon[36179]: pgmap v358: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:41.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 -- 172.21.15.28:0/813347218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f4102f80 msgr2=0x7fb5f4106fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:41.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/813347218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f4102f80 0x7fb5f4106fd0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fb5dc0099c0 tx=0x7fb5dc009cd0 comp rx=0 tx=0).stop 2024-05-20T20:02:41.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 -- 172.21.15.28:0/813347218 shutdown_connections 2024-05-20T20:02:41.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/813347218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5f41076b0 0x7fb5f410f280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:41.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/813347218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5f41025d0 0x7fb5f41029b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:41.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/813347218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f4102f80 0x7fb5f4106fd0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:41.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 -- 172.21.15.28:0/813347218 >> 172.21.15.28:0/813347218 conn(0x7fb5f40fde60 msgr2=0x7fb5f4100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:41.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 -- 172.21.15.28:0/813347218 shutdown_connections 2024-05-20T20:02:41.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.773+0000 7fb5f3e00700 1 -- 172.21.15.28:0/813347218 wait complete. 2024-05-20T20:02:41.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.774+0000 7fb5f3e00700 1 Processor -- start 2024-05-20T20:02:41.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.774+0000 7fb5f3e00700 1 -- start start 2024-05-20T20:02:41.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.774+0000 7fb5f3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5f41025d0 0x7fb5f419bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:41.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.774+0000 7fb5f3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5f4102f80 0x7fb5f419c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:41.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f41076b0 0x7fb5f41a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:41.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f3e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb5f4112570 con 0x7fb5f4102f80 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f3e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb5f41123f0 con 0x7fb5f41025d0 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f3e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb5f41126f0 con 0x7fb5f41076b0 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5ea000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5f4102f80 0x7fb5f419c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5ea000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5f4102f80 0x7fb5f419c310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47756/0 (socket says 172.21.15.28:47756) 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5f41025d0 0x7fb5f419bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5ea000700 1 -- 172.21.15.28:0/1420265574 learned_addr learned my addr 172.21.15.28:0/1420265574 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f3400700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f41076b0 0x7fb5f41a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f2a00700 1 -- 172.21.15.28:0/1420265574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f41076b0 msgr2=0x7fb5f41a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f2a00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f41076b0 0x7fb5f41a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f2a00700 1 -- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5f4102f80 msgr2=0x7fb5f419c310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f2a00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5f4102f80 0x7fb5f419c310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.775+0000 7fb5f2a00700 1 -- 172.21.15.28:0/1420265574 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb5f41a0de0 con 0x7fb5f41025d0 2024-05-20T20:02:41.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.776+0000 7fb5f2a00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5f41025d0 0x7fb5f419bdd0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb5e400e9f0 tx=0x7fb5e400ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:41.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.776+0000 7fb5ebe00700 1 -- 172.21.15.28:0/1420265574 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb5e400c9c0 con 0x7fb5f41025d0 2024-05-20T20:02:41.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.776+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb5f41a10d0 con 0x7fb5f41025d0 2024-05-20T20:02:41.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.776+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb5f41a8950 con 0x7fb5f41025d0 2024-05-20T20:02:41.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.776+0000 7fb5ebe00700 1 -- 172.21.15.28:0/1420265574 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb5e400cb60 con 0x7fb5f41025d0 2024-05-20T20:02:41.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.776+0000 7fb5ebe00700 1 -- 172.21.15.28:0/1420265574 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb5e4018680 con 0x7fb5f41025d0 2024-05-20T20:02:41.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.778+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb5d0005330 con 0x7fb5f41025d0 2024-05-20T20:02:41.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.780+0000 7fb5ebe00700 1 -- 172.21.15.28:0/1420265574 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fb5e404aae0 con 0x7fb5f41025d0 2024-05-20T20:02:41.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.781+0000 7fb5ebe00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5d80b2990 0x7fb5d80b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:41.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.782+0000 7fb5ebe00700 1 -- 172.21.15.28:0/1420265574 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fb5e40e2550 con 0x7fb5f41025d0 2024-05-20T20:02:41.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.782+0000 7fb5ea000700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5d80b2990 0x7fb5d80b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:41.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.782+0000 7fb5ea000700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5d80b2990 0x7fb5d80b4e50 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fb5dc003760 tx=0x7fb5dc005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:41.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:41.784+0000 7fb5ebe00700 1 -- 172.21.15.28:0/1420265574 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb5e40aa0d0 con 0x7fb5f41025d0 2024-05-20T20:02:42.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.021+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fb5d00058f0 con 0x7fb5f41025d0 2024-05-20T20:02:42.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.023+0000 7fb5ebe00700 1 -- 172.21.15.28:0/1420265574 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) ==== 74+0+22571 (secure 0 0 0) 0x7fb5e40b2150 con 0x7fb5f41025d0 2024-05-20T20:02:42.024 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:02:42.025 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":80,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:02:04.742448+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:02:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.027+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5d80b2990 msgr2=0x7fb5d80b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.027+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5d80b2990 0x7fb5d80b4e50 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fb5dc003760 tx=0x7fb5dc005fb0 comp rx=0 tx=0).stop 2024-05-20T20:02:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.027+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5f41025d0 msgr2=0x7fb5f419bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:42.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.027+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5f41025d0 0x7fb5f419bdd0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb5e400e9f0 tx=0x7fb5e400ed00 comp rx=0 tx=0).stop 2024-05-20T20:02:42.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.027+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 shutdown_connections 2024-05-20T20:02:42.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.027+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb5d80b2990 0x7fb5d80b4e50 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:42.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.028+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5f41076b0 0x7fb5f41a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:42.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.028+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5f41025d0 0x7fb5f419bdd0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:42.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.028+0000 7fb5f3e00700 1 --2- 172.21.15.28:0/1420265574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5f4102f80 0x7fb5f419c310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:42.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.028+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 >> 172.21.15.28:0/1420265574 conn(0x7fb5f40fde60 msgr2=0x7fb5f406b700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:42.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.028+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 shutdown_connections 2024-05-20T20:02:42.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:42.028+0000 7fb5f3e00700 1 -- 172.21.15.28:0/1420265574 wait complete. 2024-05-20T20:02:42.911 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-20T20:02:42.912 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-20T20:02:42.912 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-20T20:02:42.913 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-20T20:02:42.913 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-20T20:02:42.914 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-20T20:02:42.914 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-20T20:02:42.914 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-20T20:02:42.915 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.8 flush_pg_stats 2024-05-20T20:02:42.915 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.9 flush_pg_stats 2024-05-20T20:02:42.916 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.10 flush_pg_stats 2024-05-20T20:02:42.916 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell osd.11 flush_pg_stats 2024-05-20T20:02:43.031 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:42 smithi028 ceph-mon[31178]: pgmap v359: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:02:43.031 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:42 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1420265574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:43.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:42 smithi154 ceph-mon[36314]: pgmap v359: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:02:43.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:42 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1420265574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:42 smithi033 ceph-mon[36179]: pgmap v359: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:02:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:42 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1420265574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:02:44.049 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.404 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.410 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.413 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.413 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.434 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.434 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.441 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.450 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.450 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.451 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:44.926 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:02:45.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:44 smithi028 ceph-mon[31178]: pgmap v360: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:45.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:44 smithi154 ceph-mon[36314]: pgmap v360: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:44 smithi033 ceph-mon[36179]: pgmap v360: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:46 smithi028 ceph-mon[31178]: pgmap v361: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:46 smithi154 ceph-mon[36314]: pgmap v361: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:46 smithi033 ceph-mon[36179]: pgmap v361: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:48 smithi028 ceph-mon[31178]: pgmap v362: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:49.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:48 smithi154 ceph-mon[36314]: pgmap v362: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:48 smithi033 ceph-mon[36179]: pgmap v362: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:50.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:50 smithi028 ceph-mon[31178]: pgmap v363: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:51.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:50 smithi154 ceph-mon[36314]: pgmap v363: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:51.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:50 smithi033 ceph-mon[36179]: pgmap v363: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:51.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.668+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3646114887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc10b730 msgr2=0x7f1dbc10bb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:51.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.668+0000 7f1dc3e00700 1 --2- 172.21.15.28:0/3646114887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc10b730 0x7f1dbc10bb10 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f1db0009990 tx=0x7f1db0009ca0 comp rx=0 tx=0).stop 2024-05-20T20:02:51.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.668+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3646114887 shutdown_connections 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.668+0000 7f1dc3e00700 1 --2- 172.21.15.28:0/3646114887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1dbc1060f0 0x7f1dbc112600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.668+0000 7f1dc3e00700 1 --2- 172.21.15.28:0/3646114887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc10b730 0x7f1dbc10bb10 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.668+0000 7f1dc3e00700 1 --2- 172.21.15.28:0/3646114887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1dbc105730 0x7f1dbc105bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.668+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3646114887 >> 172.21.15.28:0/3646114887 conn(0x7f1dbc101220 msgr2=0x7f1dbc103640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.669+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3646114887 shutdown_connections 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.669+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3646114887 wait complete. 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 Processor -- start 2024-05-20T20:02:51.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 -- start start 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc105730 0x7f1dbc19be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1dbc1060f0 0x7f1dbc19c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1dbc10b730 0x7f1dbc1a0700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1dbc115980 con 0x7f1dbc105730 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1dbc115800 con 0x7f1dbc1060f0 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc3e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1dbc115b00 con 0x7f1dbc10b730 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc1400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc105730 0x7f1dbc19be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc1400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc105730 0x7f1dbc19be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42542/0 (socket says 172.21.15.28:42542) 2024-05-20T20:02:51.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc1400700 1 -- 172.21.15.28:0/3093517896 learned_addr learned my addr 172.21.15.28:0/3093517896 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.670+0000 7f1dc0a00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1dbc1060f0 0x7f1dbc19c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.671+0000 7f1dc1e00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1dbc10b730 0x7f1dbc1a0700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.671+0000 7f1dc1400700 1 -- 172.21.15.28:0/3093517896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1dbc10b730 msgr2=0x7f1dbc1a0700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.671+0000 7f1dc1400700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1dbc10b730 0x7f1dbc1a0700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.671+0000 7f1dc1400700 1 -- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1dbc1060f0 msgr2=0x7f1dbc19c370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.671+0000 7f1dc1400700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1dbc1060f0 0x7f1dbc19c370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.671+0000 7f1dc1400700 1 -- 172.21.15.28:0/3093517896 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1dbc1a0e40 con 0x7f1dbc105730 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.671+0000 7f1dc1400700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc105730 0x7f1dbc19be30 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f1db0005910 tx=0x7f1db000b670 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:51.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.672+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1db0027070 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.672+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1db0022470 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.672+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1db0017650 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.672+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1dbc1a10d0 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.672+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1dbc1a89b0 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.673+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f1dbc063cd0 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.676+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f1db002a070 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.677+0000 7f1dba000700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1da40b2910 0x7f1da40b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.677+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f1db00e35e0 con 0x7f1dbc105730 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.677+0000 7f1dc0a00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1da40b2910 0x7f1da40b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.678+0000 7f1dba000700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] conn(0x7f1da40bc4d0 0x7f1da40be970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.678+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 --> [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f1db000b0b0 con 0x7f1da40bc4d0 2024-05-20T20:02:51.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.678+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f1db00e3990 con 0x7f1dbc105730 2024-05-20T20:02:51.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.678+0000 7f1dc1e00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] conn(0x7f1da40bc4d0 0x7f1da40be970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:51.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.678+0000 7f1dc0a00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1da40b2910 0x7f1da40b4dd0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f1dbc103840 tx=0x7f1da8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:51.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.679+0000 7f1dc1e00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] conn(0x7f1da40bc4d0 0x7f1da40be970 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:51.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.680+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== osd.1 v2:172.21.15.33:6800/3302429642 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f1db000b0b0 con 0x7f1da40bc4d0 2024-05-20T20:02:51.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.698+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 --> [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f1dbc064150 con 0x7f1da40bc4d0 2024-05-20T20:02:51.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.699+0000 7f1dba000700 1 -- 172.21.15.28:0/3093517896 <== osd.1 v2:172.21.15.33:6800/3302429642 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7f1dbc064150 con 0x7f1da40bc4d0 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.699+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] conn(0x7f1da40bc4d0 msgr2=0x7f1da40be970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.699+0000 7f1dc3e00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] conn(0x7f1da40bc4d0 0x7f1da40be970 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1da40b2910 msgr2=0x7f1da40b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f1da40b2910 0x7f1da40b4dd0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f1dbc103840 tx=0x7f1da8009260 comp rx=0 tx=0).stop 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc105730 msgr2=0x7f1dbc19be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 --2- 172.21.15.28:0/3093517896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1dbc105730 0x7f1dbc19be30 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f1db0005910 tx=0x7f1db000b670 comp rx=0 tx=0).stop 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc1e00700 1 -- 172.21.15.28:0/3093517896 reap_dead start 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 shutdown_connections 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 >> 172.21.15.28:0/3093517896 conn(0x7f1dbc101220 msgr2=0x7f1dbc1018d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 shutdown_connections 2024-05-20T20:02:51.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:51.700+0000 7f1dc3e00700 1 -- 172.21.15.28:0/3093517896 wait complete. 2024-05-20T20:02:53.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:52 smithi028 ceph-mon[31178]: pgmap v364: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:02:53.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:52 smithi154 ceph-mon[36314]: pgmap v364: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:02:53.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:52 smithi033 ceph-mon[36179]: pgmap v364: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:02:54.800 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:54 smithi028 ceph-mon[31178]: pgmap v365: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:55.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:54 smithi154 ceph-mon[36314]: pgmap v365: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:55.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:54 smithi033 ceph-mon[36179]: pgmap v365: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 -- 172.21.15.28:0/3282185520 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14114710 msgr2=0x7fca14116b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 --2- 172.21.15.28:0/3282185520 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14114710 0x7fca14116b00 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fca0c00b0a0 tx=0x7fca0c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 -- 172.21.15.28:0/3282185520 shutdown_connections 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 --2- 172.21.15.28:0/3282185520 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14114710 0x7fca14116b00 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 --2- 172.21.15.28:0/3282185520 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca14075770 0x7fca14075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 --2- 172.21.15.28:0/3282185520 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca14076090 0x7fca141141d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 -- 172.21.15.28:0/3282185520 >> 172.21.15.28:0/3282185520 conn(0x7fca1407a6a0 msgr2=0x7fca1407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:55.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 -- 172.21.15.28:0/3282185520 shutdown_connections 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.024+0000 7fca19a00700 1 -- 172.21.15.28:0/3282185520 wait complete. 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.025+0000 7fca19a00700 1 Processor -- start 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.025+0000 7fca19a00700 1 -- start start 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca19a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14075770 0x7fca141a02e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca19a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca14076090 0x7fca141a0820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca19a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca14114710 0x7fca141a4bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca19a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fca14119d30 con 0x7fca14075770 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca19a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fca14119bb0 con 0x7fca14076090 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca19a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fca14119eb0 con 0x7fca14114710 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14075770 0x7fca141a02e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14075770 0x7fca141a02e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42548/0 (socket says 172.21.15.28:42548) 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 -- 172.21.15.28:0/2167654631 learned_addr learned my addr 172.21.15.28:0/2167654631 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:55.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12000700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca14076090 0x7fca141a0820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca13400700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca14114710 0x7fca141a4bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 -- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca14114710 msgr2=0x7fca141a4bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca14114710 0x7fca141a4bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 -- 172.21.15.28:0/2167654631 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca14076090 msgr2=0x7fca141a0820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca14076090 0x7fca141a0820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.026+0000 7fca12a00700 1 -- 172.21.15.28:0/2167654631 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca141a52f0 con 0x7fca14075770 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.027+0000 7fca12a00700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14075770 0x7fca141a02e0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fca0400e9a0 tx=0x7fca0400ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.027+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca0400cbd0 con 0x7fca14075770 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.027+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fca0400cd70 con 0x7fca14075770 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.027+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca141a55e0 con 0x7fca14075770 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.027+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca04018680 con 0x7fca14075770 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.027+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca141ace60 con 0x7fca14075770 2024-05-20T20:02:55.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.028+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fca14113ea0 con 0x7fca14075770 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.031+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fca04010430 con 0x7fca14075770 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.032+0000 7fca0b400700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9f80b2990 0x7fc9f80b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.032+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fca040e2dd0 con 0x7fca14075770 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.033+0000 7fca12000700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9f80b2990 0x7fc9f80b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.033+0000 7fca0b400700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] conn(0x7fc9f80bc550 0x7fc9f80be9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.033+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 --> [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fc9f80bf0c0 con 0x7fc9f80bc550 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.033+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7fca040e3180 con 0x7fca14075770 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.033+0000 7fca13400700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] conn(0x7fc9f80bc550 0x7fc9f80be9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.033+0000 7fca12000700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9f80b2990 0x7fc9f80b4e50 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fca00009680 tx=0x7fca00009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:55.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.034+0000 7fca13400700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] conn(0x7fc9f80bc550 0x7fc9f80be9f0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.035+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== osd.5 v2:172.21.15.154:6808/666306624 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7fc9f80bf0c0 con 0x7fc9f80bc550 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.055+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 --> [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fca14076e60 con 0x7fc9f80bc550 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.055+0000 7fca0b400700 1 -- 172.21.15.28:0/2167654631 <== osd.5 v2:172.21.15.154:6808/666306624 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7fca14076e60 con 0x7fc9f80bc550 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] conn(0x7fc9f80bc550 msgr2=0x7fc9f80be9f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] conn(0x7fc9f80bc550 0x7fc9f80be9f0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9f80b2990 msgr2=0x7fc9f80b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9f80b2990 0x7fc9f80b4e50 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fca00009680 tx=0x7fca00009260 comp rx=0 tx=0).stop 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14075770 msgr2=0x7fca141a02e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 --2- 172.21.15.28:0/2167654631 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca14075770 0x7fca141a02e0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fca0400e9a0 tx=0x7fca0400ecb0 comp rx=0 tx=0).stop 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca13400700 1 -- 172.21.15.28:0/2167654631 reap_dead start 2024-05-20T20:02:55.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 shutdown_connections 2024-05-20T20:02:55.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.056+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 >> 172.21.15.28:0/2167654631 conn(0x7fca1407a6a0 msgr2=0x7fca14113540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:55.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.057+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 shutdown_connections 2024-05-20T20:02:55.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:55.057+0000 7fca19a00700 1 -- 172.21.15.28:0/2167654631 wait complete. 2024-05-20T20:02:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:56 smithi028 ceph-mon[31178]: pgmap v366: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:57.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:56 smithi154 ceph-mon[36314]: pgmap v366: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:57.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:56 smithi033 ceph-mon[36179]: pgmap v366: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:02:57.265 INFO:teuthology.orchestra.run.smithi028.stdout:55834574940 2024-05-20T20:02:57.266 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-20T20:02:58.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 -- 172.21.15.28:0/1233953797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 msgr2=0x7f2d10113e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:58.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 --2- 172.21.15.28:0/1233953797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 0x7f2d10113e80 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f2cfc009990 tx=0x7f2cfc009ca0 comp rx=0 tx=0).stop 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 -- 172.21.15.28:0/1233953797 shutdown_connections 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 --2- 172.21.15.28:0/1233953797 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2d101143c0 0x7f2d101167b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 --2- 172.21.15.28:0/1233953797 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2d10077a70 0x7f2d10077e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 --2- 172.21.15.28:0/1233953797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 0x7f2d10113e80 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 -- 172.21.15.28:0/1233953797 >> 172.21.15.28:0/1233953797 conn(0x7f2d101011f0 msgr2=0x7f2d10103610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 -- 172.21.15.28:0/1233953797 shutdown_connections 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.892+0000 7f2d17a00700 1 -- 172.21.15.28:0/1233953797 wait complete. 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.893+0000 7f2d17a00700 1 Processor -- start 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.893+0000 7f2d17a00700 1 -- start start 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d17a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2d10077a70 0x7f2d101a02f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:58.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d17a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 0x7f2d101a0830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d17a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2d101143c0 0x7f2d101a4bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d17a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2d10119da0 con 0x7f2d10078390 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d17a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2d10119c20 con 0x7f2d10077a70 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d17a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2d10119f20 con 0x7f2d101143c0 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d0fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 0x7f2d101a0830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d0fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 0x7f2d101a0830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47742/0 (socket says 172.21.15.28:47742) 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d0fe00700 1 -- 172.21.15.28:0/47007972 learned_addr learned my addr 172.21.15.28:0/47007972 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.894+0000 7f2d15a00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2d101143c0 0x7f2d101a4bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0fe00700 1 -- 172.21.15.28:0/47007972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2d101143c0 msgr2=0x7f2d101a4bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0fe00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2d101143c0 0x7f2d101a4bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0fe00700 1 -- 172.21.15.28:0/47007972 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2d10077a70 msgr2=0x7f2d101a02f0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0fe00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2d10077a70 0x7f2d101a02f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0fe00700 1 -- 172.21.15.28:0/47007972 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2d101a5300 con 0x7f2d10078390 2024-05-20T20:02:58.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d15a00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2d101143c0 0x7f2d101a4bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0fe00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 0x7f2d101a0830 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f2cfc0047b0 tx=0x7f2cfc009550 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cfc027070 con 0x7f2d10078390 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2cfc022470 con 0x7f2d10078390 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.895+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2d101a5590 con 0x7f2d10078390 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.896+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cfc017650 con 0x7f2d10078390 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.896+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2d101ace70 con 0x7f2d10078390 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.897+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f2d10063c90 con 0x7f2d10078390 2024-05-20T20:02:58.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.899+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f2cfc02a070 con 0x7f2d10078390 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.901+0000 7f2d0d600700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2cf80b2910 0x7f2cf80b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.901+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f2cfc0e3c20 con 0x7f2d10078390 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.901+0000 7f2d15000700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2cf80b2910 0x7f2cf80b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.901+0000 7f2d0d600700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] conn(0x7f2cf80bc3a0 0x7f2cf80be840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.901+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 --> [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f2cf80bef10 con 0x7f2cf80bc3a0 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.901+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f2cfc0e3fd0 con 0x7f2d10078390 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.902+0000 7f2d15a00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] conn(0x7f2cf80bc3a0 0x7f2cf80be840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.902+0000 7f2d15000700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2cf80b2910 0x7f2cf80b4dd0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f2d0400e350 tx=0x7f2d0400b040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.902+0000 7f2d15a00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] conn(0x7f2cf80bc3a0 0x7f2cf80be840 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.903+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== osd.6 v2:172.21.15.28:6818/163025282 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f2cf80bef10 con 0x7f2cf80bc3a0 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.933+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 --> [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f2d10064110 con 0x7f2cf80bc3a0 2024-05-20T20:02:58.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.933+0000 7f2d0d600700 1 -- 172.21.15.28:0/47007972 <== osd.6 v2:172.21.15.28:6818/163025282 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f2d10064110 con 0x7f2cf80bc3a0 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] conn(0x7f2cf80bc3a0 msgr2=0x7f2cf80be840 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] conn(0x7f2cf80bc3a0 0x7f2cf80be840 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2cf80b2910 msgr2=0x7f2cf80b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f2cf80b2910 0x7f2cf80b4dd0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f2d0400e350 tx=0x7f2d0400b040 comp rx=0 tx=0).stop 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 msgr2=0x7f2d101a0830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 --2- 172.21.15.28:0/47007972 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2d10078390 0x7f2d101a0830 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f2cfc0047b0 tx=0x7f2cfc009550 comp rx=0 tx=0).stop 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d15a00700 1 -- 172.21.15.28:0/47007972 reap_dead start 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 shutdown_connections 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 >> 172.21.15.28:0/47007972 conn(0x7f2d101011f0 msgr2=0x7f2d101035e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 shutdown_connections 2024-05-20T20:02:58.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:02:57.934+0000 7f2d17a00700 1 -- 172.21.15.28:0/47007972 wait complete. 2024-05-20T20:02:59.014 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:02:58 smithi028 ceph-mon[31178]: pgmap v367: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:59.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:02:58 smithi154 ceph-mon[36314]: pgmap v367: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:59.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:02:58 smithi033 ceph-mon[36179]: pgmap v367: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:02:59.786 INFO:teuthology.orchestra.run.smithi028.stdout:180388626497 2024-05-20T20:02:59.787 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 -- 172.21.15.28:0/2386299128 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8de8109580 msgr2=0x7f8de8109960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 --2- 172.21.15.28:0/2386299128 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8de8109580 0x7f8de8109960 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f8ddc0099c0 tx=0x7f8ddc009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 -- 172.21.15.28:0/2386299128 shutdown_connections 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 --2- 172.21.15.28:0/2386299128 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8de806c470 0x7f8de8110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 --2- 172.21.15.28:0/2386299128 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8de8109580 0x7f8de8109960 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 --2- 172.21.15.28:0/2386299128 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8de806bab0 0x7f8de806bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 -- 172.21.15.28:0/2386299128 >> 172.21.15.28:0/2386299128 conn(0x7f8de8079280 msgr2=0x7f8de8079690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:00.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.360+0000 7f8dee400700 1 -- 172.21.15.28:0/2386299128 shutdown_connections 2024-05-20T20:03:00.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.361+0000 7f8dee400700 1 -- 172.21.15.28:0/2386299128 wait complete. 2024-05-20T20:03:00.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 Processor -- start 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 -- start start 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8de806bab0 0x7f8de81a1a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8de806c470 0x7f8de81a1f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8de8109580 0x7f8de819bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8de8113630 con 0x7f8de806bab0 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8de81134b0 con 0x7f8de806c470 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.362+0000 7f8dee400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8de81137b0 con 0x7f8de8109580 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8de8109580 0x7f8de819bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8de7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8de806c470 0x7f8de81a1f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8de8109580 0x7f8de819bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50344/0 (socket says 172.21.15.28:50344) 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8de7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8de806c470 0x7f8de81a1f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52152/0 (socket says 172.21.15.28:52152) 2024-05-20T20:03:00.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 -- 172.21.15.28:0/3100419957 learned_addr learned my addr 172.21.15.28:0/3100419957 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:00.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 -- 172.21.15.28:0/3100419957 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8de806c470 msgr2=0x7f8de81a1f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:00.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8de806c470 0x7f8de81a1f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 -- 172.21.15.28:0/3100419957 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8de806bab0 msgr2=0x7f8de81a1a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8de806bab0 0x7f8de81a1a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 -- 172.21.15.28:0/3100419957 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8de819c3f0 con 0x7f8de8109580 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8de7e00700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8de806c470 0x7f8de81a1f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.363+0000 7f8deda00700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8de8109580 0x7f8de819bb10 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f8de000efc0 tx=0x7f8de000c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.364+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8de00092e0 con 0x7f8de8109580 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.364+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8de819c680 con 0x7f8de8109580 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.364+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8de0004bb0 con 0x7f8de8109580 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.364+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8de0004020 con 0x7f8de8109580 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.364+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8de81a87f0 con 0x7f8de8109580 2024-05-20T20:03:00.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.365+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f8de8101a00 con 0x7f8de8109580 2024-05-20T20:03:00.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.367+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f8de001b030 con 0x7f8de8109580 2024-05-20T20:03:00.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.369+0000 7f8de5600700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8dd00b2910 0x7f8dd00b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:00.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.369+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f8de00e26a0 con 0x7f8de8109580 2024-05-20T20:03:00.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.369+0000 7f8ded000700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8dd00b2910 0x7f8dd00b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:00.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.369+0000 7f8de5600700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] conn(0x7f8dd00bc4d0 0x7f8dd00be970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:00.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.370+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 --> [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f8de0013ec0 con 0x7f8dd00bc4d0 2024-05-20T20:03:00.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.370+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f8de00e2a50 con 0x7f8de8109580 2024-05-20T20:03:00.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.370+0000 7f8ded000700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8dd00b2910 0x7f8dd00b4dd0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8ddc0098e0 tx=0x7f8ddc000bc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:00.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.370+0000 7f8de7e00700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] conn(0x7f8dd00bc4d0 0x7f8dd00be970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:00.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.371+0000 7f8de7e00700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] conn(0x7f8dd00bc4d0 0x7f8dd00be970 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.8 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:00.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.372+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== osd.8 v2:172.21.15.154:6816/1056926817 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f8de0013ec0 con 0x7f8dd00bc4d0 2024-05-20T20:03:00.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.390+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 --> [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f8de8104bb0 con 0x7f8dd00bc4d0 2024-05-20T20:03:00.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.391+0000 7f8de5600700 1 -- 172.21.15.28:0/3100419957 <== osd.8 v2:172.21.15.154:6816/1056926817 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f8de8104bb0 con 0x7f8dd00bc4d0 2024-05-20T20:03:00.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] conn(0x7f8dd00bc4d0 msgr2=0x7f8dd00be970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:00.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] conn(0x7f8dd00bc4d0 0x7f8dd00be970 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8dd00b2910 msgr2=0x7f8dd00b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:00.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8dd00b2910 0x7f8dd00b4dd0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8ddc0098e0 tx=0x7f8ddc000bc0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8de8109580 msgr2=0x7f8de819bb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:00.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 --2- 172.21.15.28:0/3100419957 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8de8109580 0x7f8de819bb10 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f8de000efc0 tx=0x7f8de000c3a0 comp rx=0 tx=0).stop 2024-05-20T20:03:00.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8deda00700 1 -- 172.21.15.28:0/3100419957 reap_dead start 2024-05-20T20:03:00.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 shutdown_connections 2024-05-20T20:03:00.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.392+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 >> 172.21.15.28:0/3100419957 conn(0x7f8de8079280 msgr2=0x7f8de8101020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:00.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.393+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 shutdown_connections 2024-05-20T20:03:00.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.393+0000 7f8dee400700 1 -- 172.21.15.28:0/3100419957 wait complete. 2024-05-20T20:03:01.025 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:00 smithi028 ceph-mon[31178]: pgmap v368: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 11 op/s 2024-05-20T20:03:01.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.092+0000 7fec40200700 1 -- 172.21.15.28:0/471677613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 msgr2=0x7fec38105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:01.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.092+0000 7fec40200700 1 --2- 172.21.15.28:0/471677613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 0x7fec38105be0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fec24009990 tx=0x7fec24009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.092+0000 7fec40200700 1 -- 172.21.15.28:0/471677613 shutdown_connections 2024-05-20T20:03:01.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.092+0000 7fec40200700 1 --2- 172.21.15.28:0/471677613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fec38106120 0x7fec38112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.092+0000 7fec40200700 1 --2- 172.21.15.28:0/471677613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fec3810b760 0x7fec3810bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.092+0000 7fec40200700 1 --2- 172.21.15.28:0/471677613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 0x7fec38105be0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.092+0000 7fec40200700 1 -- 172.21.15.28:0/471677613 >> 172.21.15.28:0/471677613 conn(0x7fec38101250 msgr2=0x7fec38103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:01.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.093+0000 7fec40200700 1 -- 172.21.15.28:0/471677613 shutdown_connections 2024-05-20T20:03:01.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.093+0000 7fec40200700 1 -- 172.21.15.28:0/471677613 wait complete. 2024-05-20T20:03:01.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 Processor -- start 2024-05-20T20:03:01.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 -- start start 2024-05-20T20:03:01.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 0x7fec3819be40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:01.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fec38106120 0x7fec3819c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:01.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fec3810b760 0x7fec381a0710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:01.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fec38115990 con 0x7fec38105760 2024-05-20T20:03:01.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fec38115810 con 0x7fec38106120 2024-05-20T20:03:01.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.094+0000 7fec40200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fec38115b10 con 0x7fec3810b760 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 0x7fec3819be40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 0x7fec3819be40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47786/0 (socket says 172.21.15.28:47786) 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 -- 172.21.15.28:0/449720147 learned_addr learned my addr 172.21.15.28:0/449720147 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3ce00700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fec38106120 0x7fec3819c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3e200700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fec3810b760 0x7fec381a0710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 -- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fec3810b760 msgr2=0x7fec381a0710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fec3810b760 0x7fec381a0710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 -- 172.21.15.28:0/449720147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fec38106120 msgr2=0x7fec3819c380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fec38106120 0x7fec3819c380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3d800700 1 -- 172.21.15.28:0/449720147 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fec381a0e50 con 0x7fec38105760 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.095+0000 7fec3ce00700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fec38106120 0x7fec3819c380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:01.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.096+0000 7fec3d800700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 0x7fec3819be40 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fec2c00e9a0 tx=0x7fec2c00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:01.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.096+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec2c00ca10 con 0x7fec38105760 2024-05-20T20:03:01.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.096+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fec381a1140 con 0x7fec38105760 2024-05-20T20:03:01.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.096+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fec2c00cbb0 con 0x7fec38105760 2024-05-20T20:03:01.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.096+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fec2c018680 con 0x7fec38105760 2024-05-20T20:03:01.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.096+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fec381a89c0 con 0x7fec38105760 2024-05-20T20:03:01.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.097+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fec38063cd0 con 0x7fec38105760 2024-05-20T20:03:01.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.100+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fec2c010430 con 0x7fec38105760 2024-05-20T20:03:01.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.101+0000 7fec36000700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fec200b2910 0x7fec200b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:01.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.102+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fec2c0e2ab0 con 0x7fec38105760 2024-05-20T20:03:01.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.102+0000 7fec3ce00700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fec200b2910 0x7fec200b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:01.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.102+0000 7fec36000700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] conn(0x7fec200bc4d0 0x7fec200be970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:01.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.102+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 --> [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fec2c024e70 con 0x7fec200bc4d0 2024-05-20T20:03:01.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.102+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7fec2c0e2e60 con 0x7fec38105760 2024-05-20T20:03:01.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.102+0000 7fec3e200700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] conn(0x7fec200bc4d0 0x7fec200be970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:01.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.103+0000 7fec3ce00700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fec200b2910 0x7fec200b4dd0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fec38103820 tx=0x7fec2400b260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:01.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.103+0000 7fec3e200700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] conn(0x7fec200bc4d0 0x7fec200be970 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.11 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:01.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.104+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== osd.11 v2:172.21.15.154:6824/2796531640 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7fec2c024e70 con 0x7fec200bc4d0 2024-05-20T20:03:01.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.123+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 --> [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fec38064100 con 0x7fec200bc4d0 2024-05-20T20:03:01.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.123+0000 7fec36000700 1 -- 172.21.15.28:0/449720147 <== osd.11 v2:172.21.15.154:6824/2796531640 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7fec38064100 con 0x7fec200bc4d0 2024-05-20T20:03:01.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] conn(0x7fec200bc4d0 msgr2=0x7fec200be970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:01.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec40200700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] conn(0x7fec200bc4d0 0x7fec200be970 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fec200b2910 msgr2=0x7fec200b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec40200700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fec200b2910 0x7fec200b4dd0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fec38103820 tx=0x7fec2400b260 comp rx=0 tx=0).stop 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 msgr2=0x7fec3819be40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec40200700 1 --2- 172.21.15.28:0/449720147 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fec38105760 0x7fec3819be40 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fec2c00e9a0 tx=0x7fec2c00ecb0 comp rx=0 tx=0).stop 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec3e200700 1 -- 172.21.15.28:0/449720147 reap_dead start 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.124+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 shutdown_connections 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.125+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 >> 172.21.15.28:0/449720147 conn(0x7fec38101250 msgr2=0x7fec38101920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.125+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 shutdown_connections 2024-05-20T20:03:01.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:01.125+0000 7fec40200700 1 -- 172.21.15.28:0/449720147 wait complete. 2024-05-20T20:03:01.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:00 smithi154 ceph-mon[36314]: pgmap v368: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 11 op/s 2024-05-20T20:03:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:00 smithi033 ceph-mon[36179]: pgmap v368: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 11 op/s 2024-05-20T20:03:01.397 INFO:teuthology.orchestra.run.smithi028.stdout:227633266742 2024-05-20T20:03:01.397 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.8 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 -- 172.21.15.28:0/2476210046 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 msgr2=0x7f7d8006c090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 --2- 172.21.15.28:0/2476210046 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 0x7f7d8006c090 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f7d6c009990 tx=0x7f7d6c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 -- 172.21.15.28:0/2476210046 shutdown_connections 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 --2- 172.21.15.28:0/2476210046 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d8006c5d0 0x7f7d801103e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 --2- 172.21.15.28:0/2476210046 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d801095c0 0x7f7d801099a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 --2- 172.21.15.28:0/2476210046 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 0x7f7d8006c090 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 -- 172.21.15.28:0/2476210046 >> 172.21.15.28:0/2476210046 conn(0x7f7d800793f0 msgr2=0x7f7d80079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 -- 172.21.15.28:0/2476210046 shutdown_connections 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.170+0000 7f7d87e00700 1 -- 172.21.15.28:0/2476210046 wait complete. 2024-05-20T20:03:02.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.171+0000 7f7d87e00700 1 Processor -- start 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.171+0000 7f7d87e00700 1 -- start start 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d87e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 0x7f7d8019bee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d87e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d8006c5d0 0x7f7d8019c420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d87e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d801095c0 0x7f7d801a0710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d87e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7d80113750 con 0x7f7d8006bc10 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d87e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7d801135d0 con 0x7f7d801095c0 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d87e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7d801138d0 con 0x7f7d8006c5d0 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 0x7f7d8019bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 0x7f7d8019bee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47754/0 (socket says 172.21.15.28:47754) 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 -- 172.21.15.28:0/2403458138 learned_addr learned my addr 172.21.15.28:0/2403458138 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d84a00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d8006c5d0 0x7f7d8019c420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85e00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d801095c0 0x7f7d801a0710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 -- 172.21.15.28:0/2403458138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d8006c5d0 msgr2=0x7f7d8019c420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:02.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d8006c5d0 0x7f7d8019c420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 -- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d801095c0 msgr2=0x7f7d801a0710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d801095c0 0x7f7d801a0710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d85400700 1 -- 172.21.15.28:0/2403458138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7d801a0e50 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.172+0000 7f7d84a00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d8006c5d0 0x7f7d8019c420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.173+0000 7f7d85400700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 0x7f7d8019bee0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f7d7400eb30 tx=0x7f7d7400ee40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.173+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d7400cbd0 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.173+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7d740044d0 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.173+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7d801a1140 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.173+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d74003e30 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.173+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7d801a89c0 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.174+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7d80101df0 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.177+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f7d74004000 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.178+0000 7f7d7e000700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d680b2910 0x7f7d680b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.178+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f7d740e2b40 con 0x7f7d8006bc10 2024-05-20T20:03:02.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.178+0000 7f7d84a00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d680b2910 0x7f7d680b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.179+0000 7f7d7e000700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] conn(0x7f7d680bc4d0 0x7f7d680be970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.179+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 --> [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f7d680bf040 con 0x7f7d680bc4d0 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.179+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f7d740e2ef0 con 0x7f7d8006bc10 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.179+0000 7f7d85e00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] conn(0x7f7d680bc4d0 0x7f7d680be970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.179+0000 7f7d84a00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d680b2910 0x7f7d680b4dd0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f7d80110800 tx=0x7f7d6c00b260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.180+0000 7f7d85e00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] conn(0x7f7d680bc4d0 0x7f7d680be970 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.10 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.181+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== osd.10 v2:172.21.15.33:6824/924561268 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f7d680bf040 con 0x7f7d680bc4d0 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.200+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 --> [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f7d8006b4c0 con 0x7f7d680bc4d0 2024-05-20T20:03:02.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.200+0000 7f7d7e000700 1 -- 172.21.15.28:0/2403458138 <== osd.10 v2:172.21.15.33:6824/924561268 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f7d8006b4c0 con 0x7f7d680bc4d0 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] conn(0x7f7d680bc4d0 msgr2=0x7f7d680be970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] conn(0x7f7d680bc4d0 0x7f7d680be970 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d680b2910 msgr2=0x7f7d680b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d680b2910 0x7f7d680b4dd0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f7d80110800 tx=0x7f7d6c00b260 comp rx=0 tx=0).stop 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 msgr2=0x7f7d8019bee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 --2- 172.21.15.28:0/2403458138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d8006bc10 0x7f7d8019bee0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f7d7400eb30 tx=0x7f7d7400ee40 comp rx=0 tx=0).stop 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d85e00700 1 -- 172.21.15.28:0/2403458138 reap_dead start 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 shutdown_connections 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.201+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 >> 172.21.15.28:0/2403458138 conn(0x7f7d800793f0 msgr2=0x7f7d80101390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.202+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 shutdown_connections 2024-05-20T20:03:02.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:00.202+0000 7f7d87e00700 1 -- 172.21.15.28:0/2403458138 wait complete. 2024-05-20T20:03:02.917 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:02 smithi028 ceph-mon[31178]: pgmap v369: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 9 op/s 2024-05-20T20:03:03.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:02 smithi154 ceph-mon[36314]: pgmap v369: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 9 op/s 2024-05-20T20:03:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:02 smithi033 ceph-mon[36179]: pgmap v369: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 9 op/s 2024-05-20T20:03:03.418 INFO:teuthology.orchestra.run.smithi028.stdout:274877906986 2024-05-20T20:03:03.418 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.10 2024-05-20T20:03:03.903 INFO:teuthology.orchestra.run.smithi028.stdout:300647710757 2024-05-20T20:03:03.903 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.11 2024-05-20T20:03:04.567 INFO:teuthology.orchestra.run.smithi028.stdout:154618822726 2024-05-20T20:03:04.567 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-20T20:03:04.751 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:04 smithi028 ceph-mon[31178]: pgmap v370: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:05.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:04 smithi154 ceph-mon[36314]: pgmap v370: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:04 smithi033 ceph-mon[36179]: pgmap v370: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 -- 172.21.15.28:0/1907652306 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8ee010a780 msgr2=0x7f8ee010e7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 --2- 172.21.15.28:0/1907652306 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8ee010a780 0x7f8ee010e7d0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8ecc0099c0 tx=0x7f8ecc009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 -- 172.21.15.28:0/1907652306 shutdown_connections 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 --2- 172.21.15.28:0/1907652306 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8ee010eeb0 0x7f8ee0116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 --2- 172.21.15.28:0/1907652306 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8ee0109dd0 0x7f8ee010a1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 --2- 172.21.15.28:0/1907652306 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8ee010a780 0x7f8ee010e7d0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 -- 172.21.15.28:0/1907652306 >> 172.21.15.28:0/1907652306 conn(0x7f8ee0076d10 msgr2=0x7f8ee0108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 -- 172.21.15.28:0/1907652306 shutdown_connections 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.024+0000 7f8ee5800700 1 -- 172.21.15.28:0/1907652306 wait complete. 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.025+0000 7f8ee5800700 1 Processor -- start 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.025+0000 7f8ee5800700 1 -- start start 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8ee5800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8ee0109dd0 0x7f8ee01a02c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8ee5800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8ee010a780 0x7f8ee01a0800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8ee5800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8ee010eeb0 0x7f8ee01a4b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8ee5800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8ee0119d90 con 0x7f8ee010eeb0 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8ee5800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8ee0119c10 con 0x7f8ee0109dd0 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8ee5800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8ee0119f10 con 0x7f8ee010a780 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8edea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8ee0109dd0 0x7f8ee01a02c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8edf400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8ee010eeb0 0x7f8ee01a4b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8edea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8ee0109dd0 0x7f8ee01a02c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52180/0 (socket says 172.21.15.28:52180) 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8edea00700 1 -- 172.21.15.28:0/2194708678 learned_addr learned my addr 172.21.15.28:0/2194708678 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.026+0000 7f8ede000700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8ee010a780 0x7f8ee01a0800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8edea00700 1 -- 172.21.15.28:0/2194708678 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8ee010a780 msgr2=0x7f8ee01a0800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8edea00700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8ee010a780 0x7f8ee01a0800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8edea00700 1 -- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8ee010eeb0 msgr2=0x7f8ee01a4b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8edea00700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8ee010eeb0 0x7f8ee01a4b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8edea00700 1 -- 172.21.15.28:0/2194708678 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8ee01a52d0 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8edea00700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8ee0109dd0 0x7f8ee01a02c0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f8ed000e9c0 tx=0x7f8ed000ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ed000cbd0 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8ee01a55c0 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8ed000ee60 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ed00055c0 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.027+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8ee01ace40 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.028+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f8ee0063bd0 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.031+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f8ed0004750 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.032+0000 7f8ed7400700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ec40b2990 0x7f8ec40b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.032+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f8ed00e2a00 con 0x7f8ee0109dd0 2024-05-20T20:03:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.032+0000 7f8ede000700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ec40b2990 0x7f8ec40b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.033+0000 7f8ed7400700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] conn(0x7f8ec40bc550 0x7f8ec40be9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.033+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 --> [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f8ed000be50 con 0x7f8ec40bc550 2024-05-20T20:03:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.033+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f8ed00e2db0 con 0x7f8ee0109dd0 2024-05-20T20:03:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.033+0000 7f8edf400700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] conn(0x7f8ec40bc550 0x7f8ec40be9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.033+0000 7f8ede000700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ec40b2990 0x7f8ec40b4e50 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f8ecc000c00 tx=0x7f8ecc005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.034+0000 7f8edf400700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] conn(0x7f8ec40bc550 0x7f8ec40be9f0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.035+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== osd.3 v2:172.21.15.28:6810/1329515601 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f8ed000be50 con 0x7f8ec40bc550 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.056+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 --> [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f8ee0064050 con 0x7f8ec40bc550 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.057+0000 7f8ed7400700 1 -- 172.21.15.28:0/2194708678 <== osd.3 v2:172.21.15.28:6810/1329515601 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f8ee0064050 con 0x7f8ec40bc550 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.057+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] conn(0x7f8ec40bc550 msgr2=0x7f8ec40be9f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.057+0000 7f8ee5800700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] conn(0x7f8ec40bc550 0x7f8ec40be9f0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ec40b2990 msgr2=0x7f8ec40b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ec40b2990 0x7f8ec40b4e50 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f8ecc000c00 tx=0x7f8ecc005fb0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8ee0109dd0 msgr2=0x7f8ee01a02c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 --2- 172.21.15.28:0/2194708678 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8ee0109dd0 0x7f8ee01a02c0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f8ed000e9c0 tx=0x7f8ed000ecd0 comp rx=0 tx=0).stop 2024-05-20T20:03:05.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8edf400700 1 -- 172.21.15.28:0/2194708678 reap_dead start 2024-05-20T20:03:05.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 shutdown_connections 2024-05-20T20:03:05.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 >> 172.21.15.28:0/2194708678 conn(0x7f8ee0076d10 msgr2=0x7f8ee00773d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:05.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 shutdown_connections 2024-05-20T20:03:05.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:05.058+0000 7f8ee5800700 1 -- 172.21.15.28:0/2194708678 wait complete. 2024-05-20T20:03:06.873 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:06 smithi028 ceph-mon[31178]: pgmap v371: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:07.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:06 smithi154 ceph-mon[36314]: pgmap v371: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:07.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:06 smithi033 ceph-mon[36179]: pgmap v371: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.512+0000 7f7dd0c00700 1 -- 172.21.15.28:0/4094619887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7dcc1096a0 msgr2=0x7f7dcc109a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.512+0000 7f7dd0c00700 1 --2- 172.21.15.28:0/4094619887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7dcc1096a0 0x7f7dcc109a80 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f7dbc009990 tx=0x7f7dbc009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.512+0000 7f7dd0c00700 1 -- 172.21.15.28:0/4094619887 shutdown_connections 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.512+0000 7f7dd0c00700 1 --2- 172.21.15.28:0/4094619887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7dcc103f50 0x7f7dcc110430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.512+0000 7f7dd0c00700 1 --2- 172.21.15.28:0/4094619887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7dcc1096a0 0x7f7dcc109a80 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.512+0000 7f7dd0c00700 1 --2- 172.21.15.28:0/4094619887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7dcc103590 0x7f7dcc103a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.512+0000 7f7dd0c00700 1 -- 172.21.15.28:0/4094619887 >> 172.21.15.28:0/4094619887 conn(0x7f7dcc07a6f0 msgr2=0x7f7dcc07ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.513+0000 7f7dd0c00700 1 -- 172.21.15.28:0/4094619887 shutdown_connections 2024-05-20T20:03:07.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.513+0000 7f7dd0c00700 1 -- 172.21.15.28:0/4094619887 wait complete. 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 Processor -- start 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 -- start start 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7dcc103590 0x7f7dcc19bdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7dcc103f50 0x7f7dcc19c2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7dcc1096a0 0x7f7dcc1a0680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7dcc113730 con 0x7f7dcc103f50 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7dcc1135b0 con 0x7f7dcc1096a0 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.514+0000 7f7dd0c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7dcc1138b0 con 0x7f7dcc103590 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dc9400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7dcc103f50 0x7f7dcc19c2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dc9e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7dcc103590 0x7f7dcc19bdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dca800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7dcc1096a0 0x7f7dcc1a0680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dc9e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7dcc103590 0x7f7dcc19bdb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50402/0 (socket says 172.21.15.28:50402) 2024-05-20T20:03:07.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dc9e00700 1 -- 172.21.15.28:0/1489469524 learned_addr learned my addr 172.21.15.28:0/1489469524 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dca800700 1 -- 172.21.15.28:0/1489469524 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7dcc103590 msgr2=0x7f7dcc19bdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dca800700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7dcc103590 0x7f7dcc19bdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dca800700 1 -- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7dcc103f50 msgr2=0x7f7dcc19c2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dca800700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7dcc103f50 0x7f7dcc19c2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dca800700 1 -- 172.21.15.28:0/1489469524 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7dcc1a0dc0 con 0x7f7dcc1096a0 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dc9e00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7dcc103590 0x7f7dcc19bdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.515+0000 7f7dca800700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7dcc1096a0 0x7f7dcc1a0680 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f7dc400efc0 tx=0x7f7dc400c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:07.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.516+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7dc40092e0 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.516+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7dc4004bb0 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.516+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7dc4004020 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.516+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7dcc1a1050 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.516+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7dcc1a8930 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.517+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7dcc110470 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.519+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f7dc401b030 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.521+0000 7f7dc2a00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7db00b2990 0x7f7db00b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.521+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f7dc40e2730 con 0x7f7dcc1096a0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.521+0000 7f7dc9e00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7db00b2990 0x7f7db00b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.521+0000 7f7dc2a00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] conn(0x7f7db00bc550 0x7f7db00be9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:07.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.521+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 --> [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f7dc4008ec0 con 0x7f7db00bc550 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.521+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f7dc40e2ae0 con 0x7f7dcc1096a0 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.522+0000 7f7dc9400700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] conn(0x7f7db00bc550 0x7f7db00be9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.522+0000 7f7dc9e00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7db00b2990 0x7f7db00b4e50 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f7dbc009750 tx=0x7f7dbc01a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.522+0000 7f7dc9400700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] conn(0x7f7db00bc550 0x7f7db00be9f0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.9 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.523+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== osd.9 v2:172.21.15.28:6826/3479914961 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f7dc4008ec0 con 0x7f7db00bc550 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.542+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 --> [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f7dcc104cb0 con 0x7f7db00bc550 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.542+0000 7f7dc2a00700 1 -- 172.21.15.28:0/1489469524 <== osd.9 v2:172.21.15.28:6826/3479914961 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f7dcc104cb0 con 0x7f7db00bc550 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] conn(0x7f7db00bc550 msgr2=0x7f7db00be9f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] conn(0x7f7db00bc550 0x7f7db00be9f0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7db00b2990 msgr2=0x7f7db00b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7db00b2990 0x7f7db00b4e50 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f7dbc009750 tx=0x7f7dbc01a040 comp rx=0 tx=0).stop 2024-05-20T20:03:07.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7dcc1096a0 msgr2=0x7f7dcc1a0680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:07.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 --2- 172.21.15.28:0/1489469524 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7dcc1096a0 0x7f7dcc1a0680 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f7dc400efc0 tx=0x7f7dc400c3a0 comp rx=0 tx=0).stop 2024-05-20T20:03:07.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dca800700 1 -- 172.21.15.28:0/1489469524 reap_dead start 2024-05-20T20:03:07.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 shutdown_connections 2024-05-20T20:03:07.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.543+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 >> 172.21.15.28:0/1489469524 conn(0x7f7dcc07a6f0 msgr2=0x7f7dcc10ea00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:07.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.544+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 shutdown_connections 2024-05-20T20:03:07.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:06.544+0000 7f7dd0c00700 1 -- 172.21.15.28:0/1489469524 wait complete. 2024-05-20T20:03:08.101 INFO:teuthology.orchestra.run.smithi028.stdout:253403070513 2024-05-20T20:03:08.101 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.9 2024-05-20T20:03:08.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.866+0000 7f1802800700 1 -- 172.21.15.28:0/2030601593 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc109580 msgr2=0x7f17fc109960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:08.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.866+0000 7f1802800700 1 --2- 172.21.15.28:0/2030601593 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc109580 0x7f17fc109960 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f17f00099c0 tx=0x7f17f0009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.867+0000 7f1802800700 1 -- 172.21.15.28:0/2030601593 shutdown_connections 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.867+0000 7f1802800700 1 --2- 172.21.15.28:0/2030601593 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17fc06c470 0x7f17fc110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.867+0000 7f1802800700 1 --2- 172.21.15.28:0/2030601593 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc109580 0x7f17fc109960 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.867+0000 7f1802800700 1 --2- 172.21.15.28:0/2030601593 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17fc06bab0 0x7f17fc06bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.867+0000 7f1802800700 1 -- 172.21.15.28:0/2030601593 >> 172.21.15.28:0/2030601593 conn(0x7f17fc079280 msgr2=0x7f17fc079690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.867+0000 7f1802800700 1 -- 172.21.15.28:0/2030601593 shutdown_connections 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.867+0000 7f1802800700 1 -- 172.21.15.28:0/2030601593 wait complete. 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.868+0000 7f1802800700 1 Processor -- start 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.869+0000 7f1802800700 1 -- start start 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.869+0000 7f1802800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc06bab0 0x7f17fc19f6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.869+0000 7f1802800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17fc06c470 0x7f17fc19fbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:08.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.869+0000 7f1802800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17fc109580 0x7f17fc1998c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.869+0000 7f1802800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f17fc113600 con 0x7f17fc06c470 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.869+0000 7f1802800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f17fc113480 con 0x7f17fc06bab0 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.869+0000 7f1802800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f17fc113780 con 0x7f17fc109580 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc06bab0 0x7f17fc19f6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1800a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17fc06c470 0x7f17fc19fbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc06bab0 0x7f17fc19f6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:35554/0 (socket says 172.21.15.28:35554) 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 -- 172.21.15.28:0/848000051 learned_addr learned my addr 172.21.15.28:0/848000051 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801e00700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17fc109580 0x7f17fc1998c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 -- 172.21.15.28:0/848000051 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17fc109580 msgr2=0x7f17fc1998c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17fc109580 0x7f17fc1998c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:08.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 -- 172.21.15.28:0/848000051 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17fc06c470 msgr2=0x7f17fc19fbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17fc06c470 0x7f17fc19fbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 -- 172.21.15.28:0/848000051 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f17fc19a020 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.870+0000 7f1801400700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc06bab0 0x7f17fc19f6b0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f17f0006010 tx=0x7f17f0005280 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.871+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17f0026070 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.871+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f17f0005680 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.871+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f17fc19a2b0 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.871+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17f000fb20 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.871+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17fc1a6580 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.872+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f17fc063cd0 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.874+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f17f0029070 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.876+0000 7f17fa000700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f17e40b2990 0x7f17e40b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.876+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f17f00e3250 con 0x7f17fc06bab0 2024-05-20T20:03:08.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.876+0000 7f1800a00700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f17e40b2990 0x7f17e40b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.876+0000 7f17fa000700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] conn(0x7f17e40bc550 0x7f17e40be9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.876+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 --> [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f17f0004ea0 con 0x7f17e40bc550 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.876+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f17f00e3600 con 0x7f17fc06bab0 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.877+0000 7f1801e00700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] conn(0x7f17e40bc550 0x7f17e40be9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.877+0000 7f1800a00700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f17e40b2990 0x7f17e40b4e50 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f17fc079880 tx=0x7f17e8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.877+0000 7f1801e00700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] conn(0x7f17e40bc550 0x7f17e40be9f0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.878+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== osd.7 v2:172.21.15.33:6816/867501728 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f17f0004ea0 con 0x7f17e40bc550 2024-05-20T20:03:08.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.897+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 --> [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f17fc0640d0 con 0x7f17e40bc550 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.898+0000 7f17fa000700 1 -- 172.21.15.28:0/848000051 <== osd.7 v2:172.21.15.33:6816/867501728 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f17fc0640d0 con 0x7f17e40bc550 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.898+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] conn(0x7f17e40bc550 msgr2=0x7f17e40be9f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.898+0000 7f1802800700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] conn(0x7f17e40bc550 0x7f17e40be9f0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f17e40b2990 msgr2=0x7f17e40b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f17e40b2990 0x7f17e40b4e50 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f17fc079880 tx=0x7f17e8009260 comp rx=0 tx=0).stop 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc06bab0 msgr2=0x7f17fc19f6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 --2- 172.21.15.28:0/848000051 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17fc06bab0 0x7f17fc19f6b0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f17f0006010 tx=0x7f17f0005280 comp rx=0 tx=0).stop 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1801e00700 1 -- 172.21.15.28:0/848000051 reap_dead start 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 shutdown_connections 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 >> 172.21.15.28:0/848000051 conn(0x7f17fc079280 msgr2=0x7f17fc101230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 shutdown_connections 2024-05-20T20:03:08.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:07.899+0000 7f1802800700 1 -- 172.21.15.28:0/848000051 wait complete. 2024-05-20T20:03:09.039 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:08 smithi028 ceph-mon[31178]: pgmap v372: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 8 op/s 2024-05-20T20:03:09.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 -- 172.21.15.28:0/1797542121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 msgr2=0x7f9fb8105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 --2- 172.21.15.28:0/1797542121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 0x7f9fb8105be0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40098f0 tx=0x7f9fa4009c00 comp rx=0 tx=0).stop 2024-05-20T20:03:09.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 -- 172.21.15.28:0/1797542121 shutdown_connections 2024-05-20T20:03:09.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 --2- 172.21.15.28:0/1797542121 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb8106120 0x7f9fb8112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 --2- 172.21.15.28:0/1797542121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb810b760 0x7f9fb810bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 --2- 172.21.15.28:0/1797542121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 0x7f9fb8105be0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 -- 172.21.15.28:0/1797542121 >> 172.21.15.28:0/1797542121 conn(0x7f9fb8101250 msgr2=0x7f9fb8103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:09.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 -- 172.21.15.28:0/1797542121 shutdown_connections 2024-05-20T20:03:09.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.116+0000 7f9fbea00700 1 -- 172.21.15.28:0/1797542121 wait complete. 2024-05-20T20:03:09.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.117+0000 7f9fbea00700 1 Processor -- start 2024-05-20T20:03:09.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.117+0000 7f9fbea00700 1 -- start start 2024-05-20T20:03:09.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fbea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 0x7f9fb819be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fbea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb8106120 0x7f9fb819c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fbea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb810b760 0x7f9fb81a0730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fbea00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb81159f0 con 0x7f9fb8106120 2024-05-20T20:03:09.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fbea00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb8115870 con 0x7f9fb810b760 2024-05-20T20:03:09.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fbea00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb8115b70 con 0x7f9fb8105760 2024-05-20T20:03:09.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 0x7f9fb819be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 0x7f9fb819be60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54000/0 (socket says 172.21.15.28:54000) 2024-05-20T20:03:09.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 -- 172.21.15.28:0/4120750312 learned_addr learned my addr 172.21.15.28:0/4120750312 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:09.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7400700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb8106120 0x7f9fb819c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fbca00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb810b760 0x7f9fb81a0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 -- 172.21.15.28:0/4120750312 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb810b760 msgr2=0x7f9fb81a0730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb810b760 0x7f9fb81a0730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 -- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb8106120 msgr2=0x7f9fb819c3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb8106120 0x7f9fb819c3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.118+0000 7f9fb7e00700 1 -- 172.21.15.28:0/4120750312 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fb81a0e70 con 0x7f9fb8105760 2024-05-20T20:03:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.119+0000 7f9fb7e00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 0x7f9fb819be60 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9fa800e9c0 tx=0x7f9fa800ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.119+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa8004a30 con 0x7f9fb8105760 2024-05-20T20:03:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.119+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fb81a1160 con 0x7f9fb8105760 2024-05-20T20:03:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.119+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9fa800cc70 con 0x7f9fb8105760 2024-05-20T20:03:09.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.119+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa8005490 con 0x7f9fb8105760 2024-05-20T20:03:09.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.119+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fb81a89e0 con 0x7f9fb8105760 2024-05-20T20:03:09.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.120+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f9fb8063c90 con 0x7f9fb8105760 2024-05-20T20:03:09.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.123+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f9fa8004bd0 con 0x7f9fb8105760 2024-05-20T20:03:09.130 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:08 smithi033 ceph-mon[36179]: pgmap v372: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 8 op/s 2024-05-20T20:03:09.130 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:08 smithi154 ceph-mon[36314]: pgmap v372: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 8 op/s 2024-05-20T20:03:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.124+0000 7f9fb4c00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9f9c0b2910 0x7f9f9c0b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.124+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f9fa80e2850 con 0x7f9fb8105760 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.124+0000 7f9fb7400700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9f9c0b2910 0x7f9f9c0b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.125+0000 7f9fb4c00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] conn(0x7f9f9c0bc4d0 0x7f9f9c0be970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.125+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 --> [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f9fa801ee50 con 0x7f9f9c0bc4d0 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.125+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f9fa80e2c00 con 0x7f9fb8105760 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.125+0000 7f9fbca00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] conn(0x7f9f9c0bc4d0 0x7f9f9c0be970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.125+0000 7f9fb7400700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9f9c0b2910 0x7f9f9c0b4dd0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40098c0 tx=0x7f9fa4019040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.126+0000 7f9fbca00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] conn(0x7f9f9c0bc4d0 0x7f9f9c0be970 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.127+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== osd.0 v2:172.21.15.28:6802/3612620015 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f9fa801ee50 con 0x7f9f9c0bc4d0 2024-05-20T20:03:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.145+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 --> [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f9fb8064110 con 0x7f9f9c0bc4d0 2024-05-20T20:03:09.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.146+0000 7f9fb4c00700 1 -- 172.21.15.28:0/4120750312 <== osd.0 v2:172.21.15.28:6802/3612620015 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7f9fb8064110 con 0x7f9f9c0bc4d0 2024-05-20T20:03:09.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.146+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] conn(0x7f9f9c0bc4d0 msgr2=0x7f9f9c0be970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.146+0000 7f9fbea00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] conn(0x7f9f9c0bc4d0 0x7f9f9c0be970 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9f9c0b2910 msgr2=0x7f9f9c0b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f9f9c0b2910 0x7f9f9c0b4dd0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40098c0 tx=0x7f9fa4019040 comp rx=0 tx=0).stop 2024-05-20T20:03:09.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 msgr2=0x7f9fb819be60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 --2- 172.21.15.28:0/4120750312 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb8105760 0x7f9fb819be60 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9fa800e9c0 tx=0x7f9fa800ecd0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbca00700 1 -- 172.21.15.28:0/4120750312 reap_dead start 2024-05-20T20:03:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 shutdown_connections 2024-05-20T20:03:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 >> 172.21.15.28:0/4120750312 conn(0x7f9fb8101250 msgr2=0x7f9fb8101920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 shutdown_connections 2024-05-20T20:03:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.147+0000 7f9fbea00700 1 -- 172.21.15.28:0/4120750312 wait complete. 2024-05-20T20:03:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.668+0000 7f821cc00700 1 -- 172.21.15.28:0/280001499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8218101d70 msgr2=0x7f82181021f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.668+0000 7f821cc00700 1 --2- 172.21.15.28:0/280001499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8218101d70 0x7f82181021f0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f8204009990 tx=0x7f8204009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.668+0000 7f821cc00700 1 -- 172.21.15.28:0/280001499 shutdown_connections 2024-05-20T20:03:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.668+0000 7f821cc00700 1 --2- 172.21.15.28:0/280001499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8218102730 0x7f8218112640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.668+0000 7f821cc00700 1 --2- 172.21.15.28:0/280001499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8218104250 0x7f8218104630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.668+0000 7f821cc00700 1 --2- 172.21.15.28:0/280001499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8218101d70 0x7f82181021f0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.668+0000 7f821cc00700 1 -- 172.21.15.28:0/280001499 >> 172.21.15.28:0/280001499 conn(0x7f82180776f0 msgr2=0x7f8218077b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.669+0000 7f821cc00700 1 -- 172.21.15.28:0/280001499 shutdown_connections 2024-05-20T20:03:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.669+0000 7f821cc00700 1 -- 172.21.15.28:0/280001499 wait complete. 2024-05-20T20:03:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.670+0000 7f821cc00700 1 Processor -- start 2024-05-20T20:03:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.670+0000 7f821cc00700 1 -- start start 2024-05-20T20:03:09.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f821cc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8218101d70 0x7f821819bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f821cc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8218102730 0x7f821819c460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f821cc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8218104250 0x7f82181a07f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f821cc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8218115a10 con 0x7f8218101d70 2024-05-20T20:03:09.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f821cc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8218115890 con 0x7f8218104250 2024-05-20T20:03:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f821cc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8218115b90 con 0x7f8218102730 2024-05-20T20:03:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f8217400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8218101d70 0x7f821819bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f8217400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8218101d70 0x7f821819bf20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:49100/0 (socket says 172.21.15.28:49100) 2024-05-20T20:03:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f8217400700 1 -- 172.21.15.28:0/2242226529 learned_addr learned my addr 172.21.15.28:0/2242226529 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f8217e00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8218104250 0x7f82181a07f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.671+0000 7f8216a00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8218102730 0x7f821819c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f8217e00700 1 -- 172.21.15.28:0/2242226529 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8218102730 msgr2=0x7f821819c460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f8217e00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8218102730 0x7f821819c460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f8217e00700 1 -- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8218101d70 msgr2=0x7f821819bf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f8217e00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8218101d70 0x7f821819bf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f8217e00700 1 -- 172.21.15.28:0/2242226529 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f82181a0f30 con 0x7f8218104250 2024-05-20T20:03:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f8217400700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8218101d70 0x7f821819bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f8217e00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8218104250 0x7f82181a07f0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f821000efc0 tx=0x7f821000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82100092e0 con 0x7f8218104250 2024-05-20T20:03:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f82181a11c0 con 0x7f8218104250 2024-05-20T20:03:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8210004bb0 con 0x7f8218104250 2024-05-20T20:03:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f82181a8aa0 con 0x7f8218104250 2024-05-20T20:03:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.672+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8210004020 con 0x7f8218104250 2024-05-20T20:03:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.673+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f8218063c50 con 0x7f8218104250 2024-05-20T20:03:09.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.676+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f821001b030 con 0x7f8218104250 2024-05-20T20:03:09.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.677+0000 7f820fe00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81fc0b2910 0x7f81fc0b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.678+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f82100e2860 con 0x7f8218104250 2024-05-20T20:03:09.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.678+0000 7f8217400700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81fc0b2910 0x7f81fc0b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.678+0000 7f820fe00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] conn(0x7f81fc0bc4d0 0x7f81fc0be970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:09.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.678+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 --> [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f821000be50 con 0x7f81fc0bc4d0 2024-05-20T20:03:09.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.678+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f82100e2c10 con 0x7f8218104250 2024-05-20T20:03:09.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.678+0000 7f8216a00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] conn(0x7f81fc0bc4d0 0x7f81fc0be970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:09.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.679+0000 7f8217400700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81fc0b2910 0x7f81fc0b4dd0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f8208011550 tx=0x7f8208009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:09.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.679+0000 7f8216a00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] conn(0x7f81fc0bc4d0 0x7f81fc0be970 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:09.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.680+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== osd.4 v2:172.21.15.33:6808/3112020300 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f821000be50 con 0x7f81fc0bc4d0 2024-05-20T20:03:09.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.698+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 --> [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f8218064050 con 0x7f81fc0bc4d0 2024-05-20T20:03:09.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f820fe00700 1 -- 172.21.15.28:0/2242226529 <== osd.4 v2:172.21.15.33:6808/3112020300 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (crc 0 0 0) 0x7f8218064050 con 0x7f81fc0bc4d0 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] conn(0x7f81fc0bc4d0 msgr2=0x7f81fc0be970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] conn(0x7f81fc0bc4d0 0x7f81fc0be970 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81fc0b2910 msgr2=0x7f81fc0b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f81fc0b2910 0x7f81fc0b4dd0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f8208011550 tx=0x7f8208009260 comp rx=0 tx=0).stop 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8218104250 msgr2=0x7f82181a07f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 --2- 172.21.15.28:0/2242226529 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8218104250 0x7f82181a07f0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f821000efc0 tx=0x7f821000c3a0 comp rx=0 tx=0).stop 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f8217e00700 1 -- 172.21.15.28:0/2242226529 reap_dead start 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 shutdown_connections 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.699+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 >> 172.21.15.28:0/2242226529 conn(0x7f82180776f0 msgr2=0x7f82180ffe20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.700+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 shutdown_connections 2024-05-20T20:03:09.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:09.700+0000 7f821cc00700 1 -- 172.21.15.28:0/2242226529 wait complete. 2024-05-20T20:03:10.019 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:10.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.232+0000 7f175b600700 1 -- 172.21.15.28:0/1035739551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1754109dd0 msgr2=0x7f175410a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:10.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.232+0000 7f175b600700 1 --2- 172.21.15.28:0/1035739551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1754109dd0 0x7f175410a1b0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f17480099c0 tx=0x7f1748009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:10.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.232+0000 7f175b600700 1 -- 172.21.15.28:0/1035739551 shutdown_connections 2024-05-20T20:03:10.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.232+0000 7f175b600700 1 --2- 172.21.15.28:0/1035739551 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f175410eeb0 0x7f1754116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:10.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.232+0000 7f175b600700 1 --2- 172.21.15.28:0/1035739551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1754109dd0 0x7f175410a1b0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:10.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.232+0000 7f175b600700 1 --2- 172.21.15.28:0/1035739551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f175410a780 0x7f175410e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:10.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.232+0000 7f175b600700 1 -- 172.21.15.28:0/1035739551 >> 172.21.15.28:0/1035739551 conn(0x7f1754076d10 msgr2=0x7f1754108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:10.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.233+0000 7f175b600700 1 -- 172.21.15.28:0/1035739551 shutdown_connections 2024-05-20T20:03:10.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.233+0000 7f175b600700 1 -- 172.21.15.28:0/1035739551 wait complete. 2024-05-20T20:03:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 Processor -- start 2024-05-20T20:03:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 -- start start 2024-05-20T20:03:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1754109dd0 0x7f17541a0220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f175410a780 0x7f17541a0760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f175410eeb0 0x7f17541a4af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1754119d50 con 0x7f1754109dd0 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1754119bd0 con 0x7f175410a780 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f175b600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1754119ed0 con 0x7f175410eeb0 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.234+0000 7f1758c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1754109dd0 0x7f17541a0220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1754109dd0 0x7f17541a0220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:49118/0 (socket says 172.21.15.28:49118) 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 -- 172.21.15.28:0/1261519584 learned_addr learned my addr 172.21.15.28:0/1261519584 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1759600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f175410eeb0 0x7f17541a4af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1753e00700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f175410a780 0x7f17541a0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 -- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f175410eeb0 msgr2=0x7f17541a4af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f175410eeb0 0x7f17541a4af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 -- 172.21.15.28:0/1261519584 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f175410a780 msgr2=0x7f17541a0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f175410a780 0x7f17541a0760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 -- 172.21.15.28:0/1261519584 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f17541a5230 con 0x7f1754109dd0 2024-05-20T20:03:10.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.235+0000 7f1758c00700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1754109dd0 0x7f17541a0220 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f1748006010 tx=0x7f1748005280 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:10.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.236+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1748026070 con 0x7f1754109dd0 2024-05-20T20:03:10.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.236+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1748003c70 con 0x7f1754109dd0 2024-05-20T20:03:10.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.236+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f174800fbb0 con 0x7f1754109dd0 2024-05-20T20:03:10.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.236+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f17541a54c0 con 0x7f1754109dd0 2024-05-20T20:03:10.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.236+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17541acda0 con 0x7f1754109dd0 2024-05-20T20:03:10.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.237+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f17541a5950 con 0x7f1754109dd0 2024-05-20T20:03:10.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.240+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f1748029070 con 0x7f1754109dd0 2024-05-20T20:03:10.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.241+0000 7f1751600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f173c0b2990 0x7f173c0b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:10.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.241+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f17480e3190 con 0x7f1754109dd0 2024-05-20T20:03:10.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.242+0000 7f1753e00700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f173c0b2990 0x7f173c0b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.242+0000 7f1751600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] conn(0x7f173c0bc550 0x7f173c0be9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.242+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 --> [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f173c0bf0c0 con 0x7f173c0bc550 2024-05-20T20:03:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.242+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_get_version_reply(handle=1 version=80) ==== 24+0+0 (secure 0 0 0) 0x7f17480e3540 con 0x7f1754109dd0 2024-05-20T20:03:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.242+0000 7f1759600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] conn(0x7f173c0bc550 0x7f173c0be9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.242+0000 7f1753e00700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f173c0b2990 0x7f173c0b4e50 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f1754077b90 tx=0x7f1740009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:10.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.243+0000 7f1759600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] conn(0x7f173c0bc550 0x7f173c0be9f0 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:10.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.244+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== osd.2 v2:172.21.15.154:6800/152922770 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (crc 0 0 0) 0x7f173c0bf0c0 con 0x7f173c0bc550 2024-05-20T20:03:10.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.262+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 --> [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f1754063ff0 con 0x7f173c0bc550 2024-05-20T20:03:10.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.262+0000 7f1751600700 1 -- 172.21.15.28:0/1261519584 <== osd.2 v2:172.21.15.154:6800/152922770 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (crc 0 0 0) 0x7f1754063ff0 con 0x7f173c0bc550 2024-05-20T20:03:10.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.263+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] conn(0x7f173c0bc550 msgr2=0x7f173c0be9f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:10.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.263+0000 7f175b600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] conn(0x7f173c0bc550 0x7f173c0be9f0 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.263+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f173c0b2990 msgr2=0x7f173c0b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.263+0000 7f175b600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f173c0b2990 0x7f173c0b4e50 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f1754077b90 tx=0x7f1740009260 comp rx=0 tx=0).stop 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.263+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1754109dd0 msgr2=0x7f17541a0220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.263+0000 7f175b600700 1 --2- 172.21.15.28:0/1261519584 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1754109dd0 0x7f17541a0220 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f1748006010 tx=0x7f1748005280 comp rx=0 tx=0).stop 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.263+0000 7f1759600700 1 -- 172.21.15.28:0/1261519584 reap_dead start 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.264+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 shutdown_connections 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.264+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 >> 172.21.15.28:0/1261519584 conn(0x7f1754076d10 msgr2=0x7f17540783a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.264+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 shutdown_connections 2024-05-20T20:03:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:10.264+0000 7f175b600700 1 -- 172.21.15.28:0/1261519584 wait complete. 2024-05-20T20:03:10.759 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:10 smithi028 ceph-mon[31178]: pgmap v373: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 11 op/s 2024-05-20T20:03:10.831 INFO:teuthology.orchestra.run.smithi028.stdout:201863462973 2024-05-20T20:03:10.831 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-20T20:03:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:10 smithi154 ceph-mon[36314]: pgmap v373: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 11 op/s 2024-05-20T20:03:11.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:10 smithi033 ceph-mon[36179]: pgmap v373: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 11 op/s 2024-05-20T20:03:11.202 INFO:teuthology.orchestra.run.smithi028.stdout:107374182483 2024-05-20T20:03:11.202 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-20T20:03:12.342 INFO:teuthology.orchestra.run.smithi028.stdout:34359738470 2024-05-20T20:03:12.343 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-20T20:03:12.940 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:12 smithi028 ceph-mon[31178]: pgmap v374: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2024-05-20T20:03:12.991 INFO:teuthology.orchestra.run.smithi028.stdout:77309411418 2024-05-20T20:03:12.992 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-20T20:03:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:12 smithi154 ceph-mon[36314]: pgmap v374: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2024-05-20T20:03:13.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:12 smithi033 ceph-mon[36179]: pgmap v374: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2024-05-20T20:03:13.816 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:13.854 INFO:teuthology.orchestra.run.smithi028.stdout:133143986255 2024-05-20T20:03:13.854 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-20T20:03:14.369 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:14.371 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:14.372 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:14.374 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:14.383 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:14.937 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:14 smithi028 ceph-mon[31178]: pgmap v375: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 10 op/s 2024-05-20T20:03:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:14 smithi154 ceph-mon[36314]: pgmap v375: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 10 op/s 2024-05-20T20:03:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:14 smithi033 ceph-mon[36179]: pgmap v375: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 10 op/s 2024-05-20T20:03:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:16 smithi028 ceph-mon[31178]: pgmap v376: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:16 smithi154 ceph-mon[36314]: pgmap v376: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:16 smithi033 ceph-mon[36179]: pgmap v376: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:18.997 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:18 smithi028 ceph-mon[31178]: pgmap v377: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:18 smithi154 ceph-mon[36314]: pgmap v377: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:19.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:18 smithi033 ceph-mon[36179]: pgmap v377: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:19.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.684+0000 7f381f200700 1 -- 172.21.15.28:0/3854989080 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f38181035f0 msgr2=0x7f3818103a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:19.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.684+0000 7f381f200700 1 --2- 172.21.15.28:0/3854989080 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f38181035f0 0x7f3818103a50 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f38040098f0 tx=0x7f3804009c00 comp rx=0 tx=0).stop 2024-05-20T20:03:19.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.684+0000 7f381f200700 1 -- 172.21.15.28:0/3854989080 shutdown_connections 2024-05-20T20:03:19.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.684+0000 7f381f200700 1 --2- 172.21.15.28:0/3854989080 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3818103f90 0x7f3818110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.685+0000 7f381f200700 1 --2- 172.21.15.28:0/3854989080 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3818108500 0x7f38181088e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.685+0000 7f381f200700 1 --2- 172.21.15.28:0/3854989080 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f38181035f0 0x7f3818103a50 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.685+0000 7f381f200700 1 -- 172.21.15.28:0/3854989080 >> 172.21.15.28:0/3854989080 conn(0x7f381807a6a0 msgr2=0x7f381807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.685+0000 7f381f200700 1 -- 172.21.15.28:0/3854989080 shutdown_connections 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.685+0000 7f381f200700 1 -- 172.21.15.28:0/3854989080 wait complete. 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 Processor -- start 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 -- start start 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f38181035f0 0x7f381819be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3818103f90 0x7f381819c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3818108500 0x7f38181a06e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f38181137e0 con 0x7f3818108500 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3818113660 con 0x7f38181035f0 2024-05-20T20:03:19.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f381f200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3818113960 con 0x7f3818103f90 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.686+0000 7f3817e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f38181035f0 0x7f381819be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f38181035f0 0x7f381819be60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42042/0 (socket says 172.21.15.28:42042) 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 -- 172.21.15.28:0/26785473 learned_addr learned my addr 172.21.15.28:0/26785473 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817400700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3818103f90 0x7f381819c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f381d200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3818108500 0x7f38181a06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 -- 172.21.15.28:0/26785473 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3818103f90 msgr2=0x7f381819c3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3818103f90 0x7f381819c3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 -- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3818108500 msgr2=0x7f38181a06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3818108500 0x7f38181a06e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 -- 172.21.15.28:0/26785473 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f38181a0e20 con 0x7f38181035f0 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f381d200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3818108500 0x7f38181a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:03:19.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817400700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3818103f90 0x7f381819c3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3817e00700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f38181035f0 0x7f381819be60 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f380800ebf0 tx=0x7f380800ef00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.687+0000 7f3814c00700 1 -- 172.21.15.28:0/26785473 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f380800cd40 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.688+0000 7f3814c00700 1 -- 172.21.15.28:0/26785473 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f38080108e0 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.688+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f38181a1110 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.688+0000 7f3814c00700 1 -- 172.21.15.28:0/26785473 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3808018840 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.688+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f38181a8990 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.689+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f381806a230 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.691+0000 7f3814c00700 1 -- 172.21.15.28:0/26785473 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f380801f020 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.692+0000 7f3814c00700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37fc0b2990 0x7f37fc0b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.693+0000 7f3814c00700 1 -- 172.21.15.28:0/26785473 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f38080e23d0 con 0x7f38181035f0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.693+0000 7f3817400700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37fc0b2990 0x7f37fc0b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:19.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.694+0000 7f3817400700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37fc0b2990 0x7f37fc0b4e50 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f3804009fd0 tx=0x7f3804019040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:19.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.697+0000 7f3814c00700 1 -- 172.21.15.28:0/26785473 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f38080a9f50 con 0x7f38181035f0 2024-05-20T20:03:19.949 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:19 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:03:19.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.945+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 6} v 0) -- 0x7f38181a8bc0 con 0x7f38181035f0 2024-05-20T20:03:19.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.945+0000 7f3814c00700 1 -- 172.21.15.28:0/26785473 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 6}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f38080a9850 con 0x7f38181035f0 2024-05-20T20:03:19.950 INFO:teuthology.orchestra.run.smithi028.stdout:180388626501 2024-05-20T20:03:19.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37fc0b2990 msgr2=0x7f37fc0b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:19.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37fc0b2990 0x7f37fc0b4e50 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f3804009fd0 tx=0x7f3804019040 comp rx=0 tx=0).stop 2024-05-20T20:03:19.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f38181035f0 msgr2=0x7f381819be60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:19.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f38181035f0 0x7f381819be60 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f380800ebf0 tx=0x7f380800ef00 comp rx=0 tx=0).stop 2024-05-20T20:03:19.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 shutdown_connections 2024-05-20T20:03:19.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f37fc0b2990 0x7f37fc0b4e50 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3818108500 0x7f38181a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f38181035f0 0x7f381819be60 secure :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f380800ebf0 tx=0x7f380800ef00 comp rx=0 tx=0).stop 2024-05-20T20:03:19.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 --2- 172.21.15.28:0/26785473 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3818103f90 0x7f381819c3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:19.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 >> 172.21.15.28:0/26785473 conn(0x7f381807a6a0 msgr2=0x7f3818101500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:19.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 shutdown_connections 2024-05-20T20:03:19.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:19.950+0000 7f381f200700 1 -- 172.21.15.28:0/26785473 wait complete. 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 -- 172.21.15.28:0/49173544 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 msgr2=0x7fb5a81141d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 --2- 172.21.15.28:0/49173544 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 0x7fb5a81141d0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fb594009990 tx=0x7fb594009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 -- 172.21.15.28:0/49173544 shutdown_connections 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 --2- 172.21.15.28:0/49173544 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8114710 0x7fb5a8116b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 --2- 172.21.15.28:0/49173544 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5a8075770 0x7fb5a8075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 --2- 172.21.15.28:0/49173544 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 0x7fb5a81141d0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 -- 172.21.15.28:0/49173544 >> 172.21.15.28:0/49173544 conn(0x7fb5a807a6a0 msgr2=0x7fb5a807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 -- 172.21.15.28:0/49173544 shutdown_connections 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.031+0000 7fb5ad600700 1 -- 172.21.15.28:0/49173544 wait complete. 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.032+0000 7fb5ad600700 1 Processor -- start 2024-05-20T20:03:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.032+0000 7fb5ad600700 1 -- start start 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5ad600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8075770 0x7fb5a81a02a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5ad600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 0x7fb5a81a07e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5ad600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5a8114710 0x7fb5a81a4b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5ad600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb5a8119d50 con 0x7fb5a8076090 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5ad600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb5a8119bd0 con 0x7fb5a8114710 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5ad600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb5a8119ed0 con 0x7fb5a8075770 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8075770 0x7fb5a81a02a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8075770 0x7fb5a81a02a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58304/0 (socket says 172.21.15.28:58304) 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a6a00700 1 -- 172.21.15.28:0/1674103161 learned_addr learned my addr 172.21.15.28:0/1674103161 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a6000700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 0x7fb5a81a07e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a7400700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5a8114710 0x7fb5a81a4b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:20.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a6a00700 1 -- 172.21.15.28:0/1674103161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5a8114710 msgr2=0x7fb5a81a4b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a6a00700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5a8114710 0x7fb5a81a4b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.033+0000 7fb5a6a00700 1 -- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 msgr2=0x7fb5a81a07e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb5a6a00700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 0x7fb5a81a07e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb5a6a00700 1 -- 172.21.15.28:0/1674103161 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb5a81a52b0 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb5a6a00700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8075770 0x7fb5a81a02a0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fb59800e9a0 tx=0x7fb59800ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb59f400700 1 -- 172.21.15.28:0/1674103161 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb59800ca10 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb59f400700 1 -- 172.21.15.28:0/1674103161 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb59800cbb0 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb5a81a55a0 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb59f400700 1 -- 172.21.15.28:0/1674103161 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb598018680 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.034+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb5a81ace20 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.035+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb5a806a230 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.037+0000 7fb59f400700 1 -- 172.21.15.28:0/1674103161 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fb598010430 con 0x7fb5a8075770 2024-05-20T20:03:20.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.039+0000 7fb59f400700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb58c0b2910 0x7fb58c0b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:20.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.039+0000 7fb59f400700 1 -- 172.21.15.28:0/1674103161 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fb5980e2530 con 0x7fb5a8075770 2024-05-20T20:03:20.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.039+0000 7fb5a6000700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb58c0b2910 0x7fb58c0b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:20.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.040+0000 7fb5a6000700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb58c0b2910 0x7fb58c0b4dd0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fb5a8101c10 tx=0x7fb59400b260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:20.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.043+0000 7fb59f400700 1 -- 172.21.15.28:0/1674103161 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb5980aa0b0 con 0x7fb5a8075770 2024-05-20T20:03:20.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:19 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:03:20.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:19 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:03:20.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.293+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fb5a81acff0 con 0x7fb5a8075770 2024-05-20T20:03:20.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.293+0000 7fb59f400700 1 -- 172.21.15.28:0/1674103161 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fb5980b2130 con 0x7fb5a8075770 2024-05-20T20:03:20.295 INFO:teuthology.orchestra.run.smithi028.stdout:55834574946 2024-05-20T20:03:20.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.296+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb58c0b2910 msgr2=0x7fb58c0b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:20.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.296+0000 7fb5ad600700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb58c0b2910 0x7fb58c0b4dd0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fb5a8101c10 tx=0x7fb59400b260 comp rx=0 tx=0).stop 2024-05-20T20:03:20.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.296+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8075770 msgr2=0x7fb5a81a02a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:20.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.296+0000 7fb5ad600700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8075770 0x7fb5a81a02a0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fb59800e9a0 tx=0x7fb59800ecb0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 shutdown_connections 2024-05-20T20:03:20.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb58c0b2910 0x7fb58c0b4dd0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb5a8114710 0x7fb5a81a4b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb5a8075770 0x7fb5a81a02a0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 --2- 172.21.15.28:0/1674103161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb5a8076090 0x7fb5a81a07e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:20.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 >> 172.21.15.28:0/1674103161 conn(0x7fb5a807a6a0 msgr2=0x7fb5a81101b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:20.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 shutdown_connections 2024-05-20T20:03:20.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:20.297+0000 7fb5ad600700 1 -- 172.21.15.28:0/1674103161 wait complete. 2024-05-20T20:03:20.962 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:20 smithi028 ceph-mon[31178]: pgmap v378: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 11 op/s 2024-05-20T20:03:20.962 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:20 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/26785473' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T20:03:20.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:20.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:20 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:20.963 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:20 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1674103161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T20:03:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:20 smithi154 ceph-mon[36314]: pgmap v378: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 11 op/s 2024-05-20T20:03:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:20 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/26785473' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T20:03:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:20 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:20 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1674103161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T20:03:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:20 smithi033 ceph-mon[36179]: pgmap v378: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 11 op/s 2024-05-20T20:03:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:20 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/26785473' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T20:03:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:20 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:20 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1674103161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T20:03:22.171 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.171 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.171 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.171 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:22 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.171 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:22 smithi028 ceph-mon[31178]: pgmap v379: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.4 KiB/s rd, 8 op/s 2024-05-20T20:03:22.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:22 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:22 smithi154 ceph-mon[36314]: pgmap v379: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.4 KiB/s rd, 8 op/s 2024-05-20T20:03:22.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:22 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:22.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:22 smithi033 ceph-mon[36179]: pgmap v379: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.4 KiB/s rd, 8 op/s 2024-05-20T20:03:23.251 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:23 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3776334367' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-05-20T20:03:23.254 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626497 got 180388626501 for osd.6 2024-05-20T20:03:23.255 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:23.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:23 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3776334367' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-05-20T20:03:23.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:23 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3776334367' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-05-20T20:03:23.632 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:23.633 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:23.633 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:23.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 -- 172.21.15.28:0/3575796788 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 msgr2=0x7f108810e9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:23.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 --2- 172.21.15.28:0/3575796788 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 0x7f108810e9d0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f1080009f90 tx=0x7f108000b330 comp rx=0 tx=0).stop 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 -- 172.21.15.28:0/3575796788 shutdown_connections 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 --2- 172.21.15.28:0/3575796788 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 0x7f108810e9d0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 --2- 172.21.15.28:0/3575796788 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f108806a540 0x7f1088106340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 --2- 172.21.15.28:0/3575796788 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 0x7f1088106880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 -- 172.21.15.28:0/3575796788 >> 172.21.15.28:0/3575796788 conn(0x7f10880771e0 msgr2=0x7f10880775f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 -- 172.21.15.28:0/3575796788 shutdown_connections 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.457+0000 7f1087200700 1 -- 172.21.15.28:0/3575796788 wait complete. 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.459+0000 7f1087200700 1 Processor -- start 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.459+0000 7f1087200700 1 -- start start 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.459+0000 7f1087200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f108806a540 0x7f10881a1930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.459+0000 7f1087200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 0x7f10881a1e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:23.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.459+0000 7f1087200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 0x7f108819baf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1087200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1088111460 con 0x7f1088102460 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1087200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f10881112e0 con 0x7f108810c5e0 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1087200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f10881115e0 con 0x7f108806a540 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 0x7f10881a1e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 0x7f10881a1e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54898/0 (socket says 172.21.15.28:54898) 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 -- 172.21.15.28:0/1790762912 learned_addr learned my addr 172.21.15.28:0/1790762912 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085e00700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f108806a540 0x7f10881a1930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1086800700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 0x7f108819baf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 -- 172.21.15.28:0/1790762912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f108806a540 msgr2=0x7f10881a1930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f108806a540 0x7f10881a1930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 -- 172.21.15.28:0/1790762912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 msgr2=0x7f108819baf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:23.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 0x7f108819baf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.460+0000 7f1085400700 1 -- 172.21.15.28:0/1790762912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f108819c250 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.461+0000 7f1085400700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 0x7f10881a1e70 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f107400ea90 tx=0x7f107400eda0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.461+0000 7f107ea00700 1 -- 172.21.15.28:0/1790762912 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f107400ccd0 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.461+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f108819c540 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.461+0000 7f107ea00700 1 -- 172.21.15.28:0/1790762912 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1074016890 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.461+0000 7f107ea00700 1 -- 172.21.15.28:0/1790762912 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1074015760 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.461+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f10881a8800 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.462+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f108806a230 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.466+0000 7f107ea00700 1 -- 172.21.15.28:0/1790762912 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f10740163e0 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.467+0000 7f107ea00700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f106c0b2910 0x7f106c0b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.467+0000 7f107ea00700 1 -- 172.21.15.28:0/1790762912 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f10740e2710 con 0x7f1088102460 2024-05-20T20:03:23.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.468+0000 7f1085e00700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f106c0b2910 0x7f106c0b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:23.930 INFO:teuthology.orchestra.run.smithi028.stdout:253403070516 2024-05-20T20:03:23.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.468+0000 7f1085e00700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f106c0b2910 0x7f106c0b4dd0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f1078009c60 tx=0x7f1078009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:23.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.472+0000 7f107ea00700 1 -- 172.21.15.28:0/1790762912 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f10740aa290 con 0x7f1088102460 2024-05-20T20:03:23.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.770+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 9} v 0) -- 0x7f10881a8a90 con 0x7f1088102460 2024-05-20T20:03:23.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.770+0000 7f107ea00700 1 -- 172.21.15.28:0/1790762912 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 9}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f10740a9b90 con 0x7f1088102460 2024-05-20T20:03:23.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f106c0b2910 msgr2=0x7f106c0b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f106c0b2910 0x7f106c0b4dd0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f1078009c60 tx=0x7f1078009260 comp rx=0 tx=0).stop 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 msgr2=0x7f10881a1e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 0x7f10881a1e70 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f107400ea90 tx=0x7f107400eda0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 shutdown_connections 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f106c0b2910 0x7f106c0b4dd0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f108810c5e0 0x7f108819baf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f108806a540 0x7f10881a1930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 --2- 172.21.15.28:0/1790762912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1088102460 0x7f10881a1e70 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 >> 172.21.15.28:0/1790762912 conn(0x7f10880771e0 msgr2=0x7f1088100930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 shutdown_connections 2024-05-20T20:03:23.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:23.775+0000 7f1087200700 1 -- 172.21.15.28:0/1790762912 wait complete. 2024-05-20T20:03:24.166 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:24.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:24 smithi028 ceph-mon[31178]: pgmap v380: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:24.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:24 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1790762912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-05-20T20:03:24.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:24 smithi154 ceph-mon[36314]: pgmap v380: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:24.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:24 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1790762912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-05-20T20:03:24.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:24 smithi033 ceph-mon[36179]: pgmap v380: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:24.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:24 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1790762912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.911+0000 7ff1c1a00700 1 -- 172.21.15.28:0/4200549366 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 msgr2=0x7ff1bc10a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.911+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/4200549366 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc10a1b0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7ff1ac0099c0 tx=0x7ff1ac009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.912+0000 7ff1c1a00700 1 -- 172.21.15.28:0/4200549366 shutdown_connections 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.912+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/4200549366 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1bc10eeb0 0x7ff1bc116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.912+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/4200549366 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc10a1b0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.912+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/4200549366 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1bc10a780 0x7ff1bc10e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.912+0000 7ff1c1a00700 1 -- 172.21.15.28:0/4200549366 >> 172.21.15.28:0/4200549366 conn(0x7ff1bc076d10 msgr2=0x7ff1bc108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.912+0000 7ff1c1a00700 1 -- 172.21.15.28:0/4200549366 shutdown_connections 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.912+0000 7ff1c1a00700 1 -- 172.21.15.28:0/4200549366 wait complete. 2024-05-20T20:03:24.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.913+0000 7ff1c1a00700 1 Processor -- start 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.914+0000 7ff1c1a00700 1 -- start start 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.914+0000 7ff1c1a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc1a0240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.914+0000 7ff1c1a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1bc10a780 0x7ff1bc1a0780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.914+0000 7ff1c1a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1bc10eeb0 0x7ff1bc1a4ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.914+0000 7ff1c1a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff1bc119db0 con 0x7ff1bc10a780 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.914+0000 7ff1c1a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff1bc119c30 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.914+0000 7ff1c1a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff1bc119f30 con 0x7ff1bc109dd0 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1baa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc1a0240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1baa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc1a0240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58310/0 (socket says 172.21.15.28:58310) 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1baa00700 1 -- 172.21.15.28:0/3776334367 learned_addr learned my addr 172.21.15.28:0/3776334367 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1ba000700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1bc10a780 0x7ff1bc1a0780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1bb400700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1bc10eeb0 0x7ff1bc1a4ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1bb400700 1 -- 172.21.15.28:0/3776334367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 msgr2=0x7ff1bc1a0240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:24.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1bb400700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc1a0240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1bb400700 1 -- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1bc10a780 msgr2=0x7ff1bc1a0780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1bb400700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1bc10a780 0x7ff1bc1a0780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1bb400700 1 -- 172.21.15.28:0/3776334367 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff1bc1a5200 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1baa00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc1a0240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.915+0000 7ff1bb400700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1bc10eeb0 0x7ff1bc1a4ac0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7ff1b400efc0 tx=0x7ff1b400c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.916+0000 7ff1b3400700 1 -- 172.21.15.28:0/3776334367 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff1b40092e0 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.916+0000 7ff1b3400700 1 -- 172.21.15.28:0/3776334367 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff1b4004bb0 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.916+0000 7ff1b3400700 1 -- 172.21.15.28:0/3776334367 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff1b4004020 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.916+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff1bc1a5490 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.916+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff1bc1acd70 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.917+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff1bc06a230 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.919+0000 7ff1b3400700 1 -- 172.21.15.28:0/3776334367 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7ff1b401b030 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.921+0000 7ff1b3400700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1a00b2990 0x7ff1a00b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.921+0000 7ff1b3400700 1 -- 172.21.15.28:0/3776334367 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7ff1b40e1df0 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.921+0000 7ff1baa00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1a00b2990 0x7ff1a00b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.922+0000 7ff1baa00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1a00b2990 0x7ff1a00b4e50 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7ff1ac009990 tx=0x7ff1ac005ab0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:21.925+0000 7ff1b3400700 1 -- 172.21.15.28:0/3776334367 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7ff1b4022020 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.172+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 11} v 0) -- 0x7ff1bc02e560 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.172+0000 7ff1b3400700 1 -- 172.21.15.28:0/3776334367 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 11}]=0 v0) ==== 75+0+13 (secure 0 0 0) 0x7ff1b4010040 con 0x7ff1bc10eeb0 2024-05-20T20:03:24.391 INFO:teuthology.orchestra.run.smithi028.stdout:300647710761 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1a00b2990 msgr2=0x7ff1a00b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1a00b2990 0x7ff1a00b4e50 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7ff1ac009990 tx=0x7ff1ac005ab0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1bc10eeb0 msgr2=0x7ff1bc1a4ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1bc10eeb0 0x7ff1bc1a4ac0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7ff1b400efc0 tx=0x7ff1b400c3a0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 shutdown_connections 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7ff1a00b2990 0x7ff1a00b4e50 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff1bc10eeb0 0x7ff1bc1a4ac0 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff1bc109dd0 0x7ff1bc1a0240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 --2- 172.21.15.28:0/3776334367 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff1bc10a780 0x7ff1bc1a0780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 >> 172.21.15.28:0/3776334367 conn(0x7ff1bc076d10 msgr2=0x7ff1bc078380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 shutdown_connections 2024-05-20T20:03:24.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:22.176+0000 7ff1c1a00700 1 -- 172.21.15.28:0/3776334367 wait complete. 2024-05-20T20:03:25.218 INFO:tasks.cephadm.ceph_manager.ceph:need seq 300647710757 got 300647710761 for osd.11 2024-05-20T20:03:25.218 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:26.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:26 smithi028 ceph-mon[31178]: pgmap v381: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:26.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:26 smithi154 ceph-mon[36314]: pgmap v381: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:26.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:26 smithi033 ceph-mon[36179]: pgmap v381: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:27.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 -- 172.21.15.28:0/3676848710 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c101760 msgr2=0x7f323c101be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:27.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 --2- 172.21.15.28:0/3676848710 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c101760 0x7f323c101be0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f3228009990 tx=0x7f3228009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 -- 172.21.15.28:0/3676848710 shutdown_connections 2024-05-20T20:03:27.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 --2- 172.21.15.28:0/3676848710 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f323c102120 0x7f323c10e630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 --2- 172.21.15.28:0/3676848710 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f323c107760 0x7f323c107b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 --2- 172.21.15.28:0/3676848710 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c101760 0x7f323c101be0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 -- 172.21.15.28:0/3676848710 >> 172.21.15.28:0/3676848710 conn(0x7f323c078640 msgr2=0x7f323c0ff9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:27.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 -- 172.21.15.28:0/3676848710 shutdown_connections 2024-05-20T20:03:27.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.578+0000 7f3241400700 1 -- 172.21.15.28:0/3676848710 wait complete. 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 Processor -- start 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 -- start start 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f323c101760 0x7f323c19be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f323c102120 0x7f323c19c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c107760 0x7f323c1a0730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f323c1115b0 con 0x7f323c107760 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f323c111430 con 0x7f323c102120 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f3241400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f323c111730 con 0x7f323c101760 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f323b200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c107760 0x7f323c1a0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.580+0000 7f323a800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f323c101760 0x7f323c19be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:27.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c107760 0x7f323c1a0730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52558/0 (socket says 172.21.15.28:52558) 2024-05-20T20:03:27.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 -- 172.21.15.28:0/4127812841 learned_addr learned my addr 172.21.15.28:0/4127812841 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:27.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f3239e00700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f323c102120 0x7f323c19c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 -- 172.21.15.28:0/4127812841 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f323c101760 msgr2=0x7f323c19be60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f323c101760 0x7f323c19be60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 -- 172.21.15.28:0/4127812841 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f323c102120 msgr2=0x7f323c19c3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f323c102120 0x7f323c19c3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 -- 172.21.15.28:0/4127812841 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f323c1a0e70 con 0x7f323c107760 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323a800700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f323c101760 0x7f323c19be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.581+0000 7f323b200700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c107760 0x7f323c1a0730 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f3234007690 tx=0x7f32340079a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.582+0000 7f3233400700 1 -- 172.21.15.28:0/4127812841 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3234007e20 con 0x7f323c107760 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.582+0000 7f3233400700 1 -- 172.21.15.28:0/4127812841 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3234004020 con 0x7f323c107760 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.582+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f323c1a1160 con 0x7f323c107760 2024-05-20T20:03:27.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.582+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f323c1a89e0 con 0x7f323c107760 2024-05-20T20:03:27.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.582+0000 7f3233400700 1 -- 172.21.15.28:0/4127812841 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f32340137c0 con 0x7f323c107760 2024-05-20T20:03:27.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.583+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f323c102ea0 con 0x7f323c107760 2024-05-20T20:03:27.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.586+0000 7f3233400700 1 -- 172.21.15.28:0/4127812841 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f323401b030 con 0x7f323c107760 2024-05-20T20:03:27.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.587+0000 7f3233400700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f32200b2990 0x7f32200b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:27.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.587+0000 7f3233400700 1 -- 172.21.15.28:0/4127812841 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f32340e1e80 con 0x7f323c107760 2024-05-20T20:03:27.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.587+0000 7f323a800700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f32200b2990 0x7f32200b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:27.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.588+0000 7f323a800700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f32200b2990 0x7f32200b4e50 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f322c009fd0 tx=0x7f322c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:27.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.590+0000 7f3233400700 1 -- 172.21.15.28:0/4127812841 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f3234022060 con 0x7f323c107760 2024-05-20T20:03:27.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.869+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) -- 0x7f323c02e560 con 0x7f323c107760 2024-05-20T20:03:27.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.869+0000 7f3233400700 1 -- 172.21.15.28:0/4127812841 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f3234011070 con 0x7f323c107760 2024-05-20T20:03:27.870 INFO:teuthology.orchestra.run.smithi028.stdout:154618822733 2024-05-20T20:03:27.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f32200b2990 msgr2=0x7f32200b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:27.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f32200b2990 0x7f32200b4e50 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f322c009fd0 tx=0x7f322c009260 comp rx=0 tx=0).stop 2024-05-20T20:03:27.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c107760 msgr2=0x7f323c1a0730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:27.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c107760 0x7f323c1a0730 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f3234007690 tx=0x7f32340079a0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 shutdown_connections 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f32200b2990 0x7f32200b4e50 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f323c107760 0x7f323c1a0730 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f323c101760 0x7f323c19be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.874+0000 7f3241400700 1 --2- 172.21.15.28:0/4127812841 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f323c102120 0x7f323c19c3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.875+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 >> 172.21.15.28:0/4127812841 conn(0x7f323c078640 msgr2=0x7f323c0ffdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.875+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 shutdown_connections 2024-05-20T20:03:27.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:27.875+0000 7f3241400700 1 -- 172.21.15.28:0/4127812841 wait complete. 2024-05-20T20:03:28.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:28 smithi028 ceph-mon[31178]: pgmap v382: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:28.687 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:28 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/4127812841' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T20:03:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:28 smithi154 ceph-mon[36314]: pgmap v382: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:28 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/4127812841' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T20:03:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:28 smithi033 ceph-mon[36179]: pgmap v382: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:28 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/4127812841' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T20:03:29.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 -- 172.21.15.28:0/2451851268 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 msgr2=0x7fc03c10d440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 --2- 172.21.15.28:0/2451851268 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 0x7fc03c10d440 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fc028009990 tx=0x7fc028009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 -- 172.21.15.28:0/2451851268 shutdown_connections 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 --2- 172.21.15.28:0/2451851268 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c10da60 0x7fc03c10ff20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 --2- 172.21.15.28:0/2451851268 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc03c104630 0x7fc03c104a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 --2- 172.21.15.28:0/2451851268 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 0x7fc03c10d440 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 -- 172.21.15.28:0/2451851268 >> 172.21.15.28:0/2451851268 conn(0x7fc03c0fde60 msgr2=0x7fc03c100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 -- 172.21.15.28:0/2451851268 shutdown_connections 2024-05-20T20:03:29.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.178+0000 7fc041800700 1 -- 172.21.15.28:0/2451851268 wait complete. 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.179+0000 7fc041800700 1 Processor -- start 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.179+0000 7fc041800700 1 -- start start 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc041800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c104630 0x7fc03c19bdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc041800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 0x7fc03c19c2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc041800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc03c10da60 0x7fc03c1a0680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc041800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc03c113680 con 0x7fc03c104f50 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc041800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc03c113500 con 0x7fc03c10da60 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc041800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc03c113800 con 0x7fc03c104630 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc03b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 0x7fc03c19c2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc03be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c104630 0x7fc03c19bdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:29.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc03be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c104630 0x7fc03c19bdb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56240/0 (socket says 172.21.15.28:56240) 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc03be00700 1 -- 172.21.15.28:0/1865118511 learned_addr learned my addr 172.21.15.28:0/1865118511 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.180+0000 7fc040e00700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc03c10da60 0x7fc03c1a0680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc03b400700 1 -- 172.21.15.28:0/1865118511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c104630 msgr2=0x7fc03c19bdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc03b400700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c104630 0x7fc03c19bdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc03b400700 1 -- 172.21.15.28:0/1865118511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc03c10da60 msgr2=0x7fc03c1a0680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc03b400700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc03c10da60 0x7fc03c1a0680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc03b400700 1 -- 172.21.15.28:0/1865118511 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc03c1a0dc0 con 0x7fc03c104f50 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc03be00700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c104630 0x7fc03c19bdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:29.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc040e00700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc03c10da60 0x7fc03c1a0680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:29.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc03b400700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 0x7fc03c19c2f0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fc0280098e0 tx=0x7fc0280057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:29.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.181+0000 7fc038c00700 1 -- 172.21.15.28:0/1865118511 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc028027070 con 0x7fc03c104f50 2024-05-20T20:03:29.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.182+0000 7fc038c00700 1 -- 172.21.15.28:0/1865118511 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc028022470 con 0x7fc03c104f50 2024-05-20T20:03:29.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.182+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc03c1a1050 con 0x7fc03c104f50 2024-05-20T20:03:29.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.182+0000 7fc038c00700 1 -- 172.21.15.28:0/1865118511 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc028017650 con 0x7fc03c104f50 2024-05-20T20:03:29.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.182+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc03c1a8930 con 0x7fc03c104f50 2024-05-20T20:03:29.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.183+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc03c06a230 con 0x7fc03c104f50 2024-05-20T20:03:29.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.186+0000 7fc038c00700 1 -- 172.21.15.28:0/1865118511 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7fc02802a070 con 0x7fc03c104f50 2024-05-20T20:03:29.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.188+0000 7fc038c00700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc0200b2990 0x7fc0200b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:29.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.188+0000 7fc038c00700 1 -- 172.21.15.28:0/1865118511 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fc0280e3050 con 0x7fc03c104f50 2024-05-20T20:03:29.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.188+0000 7fc03be00700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc0200b2990 0x7fc0200b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:29.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.189+0000 7fc03be00700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc0200b2990 0x7fc0200b4e50 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fc02c011630 tx=0x7fc02c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:29.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.192+0000 7fc038c00700 1 -- 172.21.15.28:0/1865118511 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fc028013070 con 0x7fc03c104f50 2024-05-20T20:03:29.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.441+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 8} v 0) -- 0x7fc03c02e560 con 0x7fc03c104f50 2024-05-20T20:03:29.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.442+0000 7fc038c00700 1 -- 172.21.15.28:0/1865118511 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 8}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fc0280aac60 con 0x7fc03c104f50 2024-05-20T20:03:29.443 INFO:teuthology.orchestra.run.smithi028.stdout:227633266747 2024-05-20T20:03:29.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc0200b2990 msgr2=0x7fc0200b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:29.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc0200b2990 0x7fc0200b4e50 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fc02c011630 tx=0x7fc02c009260 comp rx=0 tx=0).stop 2024-05-20T20:03:29.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 msgr2=0x7fc03c19c2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:29.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 0x7fc03c19c2f0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fc0280098e0 tx=0x7fc0280057c0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 shutdown_connections 2024-05-20T20:03:29.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc0200b2990 0x7fc0200b4e50 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc03c10da60 0x7fc03c1a0680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc03c104630 0x7fc03c19bdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 --2- 172.21.15.28:0/1865118511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc03c104f50 0x7fc03c19c2f0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:29.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.445+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 >> 172.21.15.28:0/1865118511 conn(0x7fc03c0fde60 msgr2=0x7fc03c0fead0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:29.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.446+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 shutdown_connections 2024-05-20T20:03:29.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:29.446+0000 7fc041800700 1 -- 172.21.15.28:0/1865118511 wait complete. 2024-05-20T20:03:29.533 INFO:tasks.cephadm.ceph_manager.ceph:need seq 253403070513 got 253403070516 for osd.9 2024-05-20T20:03:29.533 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:29.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:29 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1865118511' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-05-20T20:03:29.835 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:29.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:29 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1865118511' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-05-20T20:03:29.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:29 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1865118511' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-05-20T20:03:30.678 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574940 got 55834574946 for osd.1 2024-05-20T20:03:30.678 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.695+0000 7f29d3000700 1 -- 172.21.15.28:0/670884173 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc105760 msgr2=0x7f29cc105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.695+0000 7f29d3000700 1 --2- 172.21.15.28:0/670884173 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc105760 0x7f29cc105be0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f29b80098f0 tx=0x7f29b8009c00 comp rx=0 tx=0).stop 2024-05-20T20:03:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.696+0000 7f29d3000700 1 -- 172.21.15.28:0/670884173 shutdown_connections 2024-05-20T20:03:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.696+0000 7f29d3000700 1 --2- 172.21.15.28:0/670884173 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc106120 0x7f29cc112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.696+0000 7f29d3000700 1 --2- 172.21.15.28:0/670884173 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29cc10b760 0x7f29cc10bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.696+0000 7f29d3000700 1 --2- 172.21.15.28:0/670884173 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc105760 0x7f29cc105be0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.696+0000 7f29d3000700 1 -- 172.21.15.28:0/670884173 >> 172.21.15.28:0/670884173 conn(0x7f29cc101250 msgr2=0x7f29cc103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.696+0000 7f29d3000700 1 -- 172.21.15.28:0/670884173 shutdown_connections 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.696+0000 7f29d3000700 1 -- 172.21.15.28:0/670884173 wait complete. 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 Processor -- start 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 -- start start 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29cc105760 0x7f29cc19bee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc106120 0x7f29cc19c420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc10b760 0x7f29cc1a07b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f29cc1159f0 con 0x7f29cc10b760 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f29cc115870 con 0x7f29cc106120 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d3000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f29cc115b70 con 0x7f29cc105760 2024-05-20T20:03:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.697+0000 7f29d1000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc10b760 0x7f29cc1a07b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29cc105760 0x7f29cc19bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29d1000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc10b760 0x7f29cc1a07b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52588/0 (socket says 172.21.15.28:52588) 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29d1000700 1 -- 172.21.15.28:0/2398178537 learned_addr learned my addr 172.21.15.28:0/2398178537 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cb400700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc106120 0x7f29cc19c420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cbe00700 1 -- 172.21.15.28:0/2398178537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc106120 msgr2=0x7f29cc19c420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cbe00700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc106120 0x7f29cc19c420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cbe00700 1 -- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc10b760 msgr2=0x7f29cc1a07b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cbe00700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc10b760 0x7f29cc1a07b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cbe00700 1 -- 172.21.15.28:0/2398178537 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f29cc1a0ef0 con 0x7f29cc105760 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29d1000700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc10b760 0x7f29cc1a07b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.698+0000 7f29cbe00700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29cc105760 0x7f29cc19bee0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f29bc00eb90 tx=0x7f29bc00eea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.699+0000 7f29c8c00700 1 -- 172.21.15.28:0/2398178537 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29bc00cc90 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.699+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f29cc1a11e0 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.699+0000 7f29c8c00700 1 -- 172.21.15.28:0/2398178537 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f29bc00ce30 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.699+0000 7f29c8c00700 1 -- 172.21.15.28:0/2398178537 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29bc010430 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.699+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f29cc1a8a60 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.700+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f29cc06a230 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.702+0000 7f29c8c00700 1 -- 172.21.15.28:0/2398178537 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 17) ==== 149703+0+0 (secure 0 0 0) 0x7f29bc003ef0 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.704+0000 7f29c8c00700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f29b00b2910 0x7f29b00b4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.704+0000 7f29c8c00700 1 -- 172.21.15.28:0/2398178537 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f29bc0e2700 con 0x7f29cc105760 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.704+0000 7f29cb400700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f29b00b2910 0x7f29b00b4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.704+0000 7f29cb400700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f29b00b2910 0x7f29b00b4dd0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f29b80098c0 tx=0x7f29b8019040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.706+0000 7f29c8c00700 1 -- 172.21.15.28:0/2398178537 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f29bc0aa310 con 0x7f29cc105760 2024-05-20T20:03:30.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:30 smithi028 ceph-mon[31178]: pgmap v383: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:30.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:30 smithi154 ceph-mon[36314]: pgmap v383: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:30 smithi033 ceph-mon[36179]: pgmap v383: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:30.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.952+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 10} v 0) -- 0x7f29cc19d240 con 0x7f29cc105760 2024-05-20T20:03:30.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.953+0000 7f29c8c00700 1 -- 172.21.15.28:0/2398178537 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 10}]=0 v0) ==== 75+0+13 (secure 0 0 0) 0x7f29bc0a9c10 con 0x7f29cc105760 2024-05-20T20:03:30.953 INFO:teuthology.orchestra.run.smithi028.stdout:274877906992 2024-05-20T20:03:30.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f29b00b2910 msgr2=0x7f29b00b4dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:30.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f29b00b2910 0x7f29b00b4dd0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f29b80098c0 tx=0x7f29b8019040 comp rx=0 tx=0).stop 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29cc105760 msgr2=0x7f29cc19bee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29cc105760 0x7f29cc19bee0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f29bc00eb90 tx=0x7f29bc00eea0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 shutdown_connections 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f29b00b2910 0x7f29b00b4dd0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29cc10b760 0x7f29cc1a07b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29cc105760 0x7f29cc19bee0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 --2- 172.21.15.28:0/2398178537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f29cc106120 0x7f29cc19c420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 >> 172.21.15.28:0/2398178537 conn(0x7f29cc101250 msgr2=0x7f29cc101990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 shutdown_connections 2024-05-20T20:03:30.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:30.956+0000 7f29d3000700 1 -- 172.21.15.28:0/2398178537 wait complete. 2024-05-20T20:03:31.566 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:31 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2398178537' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-05-20T20:03:31.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:31 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2398178537' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-05-20T20:03:31.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:31 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2398178537' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-05-20T20:03:32.335 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822726 got 154618822733 for osd.5 2024-05-20T20:03:32.336 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:32.724 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:32 smithi028 ceph-mon[31178]: pgmap v384: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:03:32.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:32 smithi154 ceph-mon[36314]: pgmap v384: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:03:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:32 smithi033 ceph-mon[36179]: pgmap v384: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:03:34.194 INFO:tasks.cephadm.ceph_manager.ceph:need seq 227633266742 got 227633266747 for osd.8 2024-05-20T20:03:34.194 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:34.541 INFO:tasks.cephadm.ceph_manager.ceph:need seq 274877906986 got 274877906992 for osd.10 2024-05-20T20:03:34.541 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:34.795 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:34 smithi028 ceph-mon[31178]: pgmap v385: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:34.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:34 smithi154 ceph-mon[36314]: pgmap v385: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:34.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:34 smithi033 ceph-mon[36179]: pgmap v385: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:35.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.371+0000 7f2761000700 1 -- 172.21.15.28:0/2855313119 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1037f0 msgr2=0x7f275c103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:35.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.371+0000 7f2761000700 1 --2- 172.21.15.28:0/2855313119 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1037f0 0x7f275c103bd0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f274c0099c0 tx=0x7f274c009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.371+0000 7f2761000700 1 -- 172.21.15.28:0/2855313119 shutdown_connections 2024-05-20T20:03:35.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.371+0000 7f2761000700 1 --2- 172.21.15.28:0/2855313119 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f275c1088f0 0x7f275c110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.371+0000 7f2761000700 1 --2- 172.21.15.28:0/2855313119 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1037f0 0x7f275c103bd0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.371+0000 7f2761000700 1 --2- 172.21.15.28:0/2855313119 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275c1041a0 0x7f275c1081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.371+0000 7f2761000700 1 -- 172.21.15.28:0/2855313119 >> 172.21.15.28:0/2855313119 conn(0x7f275c07a6a0 msgr2=0x7f275c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:35.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.372+0000 7f2761000700 1 -- 172.21.15.28:0/2855313119 shutdown_connections 2024-05-20T20:03:35.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.372+0000 7f2761000700 1 -- 172.21.15.28:0/2855313119 wait complete. 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 Processor -- start 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 -- start start 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f275c1037f0 0x7f275c19f850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1041a0 0x7f275c19fd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275c1088f0 0x7f275c199a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f275c113730 con 0x7f275c1041a0 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f275c1135b0 con 0x7f275c1088f0 2024-05-20T20:03:35.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.373+0000 7f2761000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f275c1138b0 con 0x7f275c1037f0 2024-05-20T20:03:35.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1041a0 0x7f275c19fd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:35.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1041a0 0x7f275c19fd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52592/0 (socket says 172.21.15.28:52592) 2024-05-20T20:03:35.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759400700 1 -- 172.21.15.28:0/2635561514 learned_addr learned my addr 172.21.15.28:0/2635561514 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:35.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759e00700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f275c1037f0 0x7f275c19f850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:35.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f275a800700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275c1088f0 0x7f275c199a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:35.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759e00700 1 -- 172.21.15.28:0/2635561514 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275c1088f0 msgr2=0x7f275c199a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:35.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759e00700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275c1088f0 0x7f275c199a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759e00700 1 -- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1041a0 msgr2=0x7f275c19fd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:35.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759e00700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1041a0 0x7f275c19fd90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759e00700 1 -- 172.21.15.28:0/2635561514 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f275c19a170 con 0x7f275c1037f0 2024-05-20T20:03:35.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f275a800700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275c1088f0 0x7f275c199a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.374+0000 7f2759400700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1041a0 0x7f275c19fd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.375+0000 7f2759e00700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f275c1037f0 0x7f275c19f850 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f274c0098e0 tx=0x7f274c005280 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.375+0000 7f2752a00700 1 -- 172.21.15.28:0/2635561514 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f274c026070 con 0x7f275c1037f0 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.375+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f275c19a400 con 0x7f275c1037f0 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.375+0000 7f2752a00700 1 -- 172.21.15.28:0/2635561514 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f274c0056c0 con 0x7f275c1037f0 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.375+0000 7f2752a00700 1 -- 172.21.15.28:0/2635561514 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f274c00f990 con 0x7f275c1037f0 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.375+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f275c1a6720 con 0x7f275c1037f0 2024-05-20T20:03:35.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.376+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f275c104c80 con 0x7f275c1037f0 2024-05-20T20:03:35.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.379+0000 7f2752a00700 1 -- 172.21.15.28:0/2635561514 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f274c029070 con 0x7f275c1037f0 2024-05-20T20:03:35.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.380+0000 7f2752a00700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f27400b2880 0x7f27400b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:35.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.381+0000 7f2752a00700 1 -- 172.21.15.28:0/2635561514 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f274c0e2e80 con 0x7f275c1037f0 2024-05-20T20:03:35.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.381+0000 7f2759400700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f27400b2880 0x7f27400b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:35.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.381+0000 7f2759400700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f27400b2880 0x7f27400b4d40 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f275c103140 tx=0x7f2748009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:35.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.382+0000 7f2752a00700 1 -- 172.21.15.28:0/2635561514 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f274c030020 con 0x7f275c1037f0 2024-05-20T20:03:35.576 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:35 smithi028 ceph-mon[31178]: mgrmap e18: x(active, since 10m), standbys: z, y 2024-05-20T20:03:35.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.645+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 7} v 0) -- 0x7f275c02e560 con 0x7f275c1037f0 2024-05-20T20:03:35.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.646+0000 7f2752a00700 1 -- 172.21.15.28:0/2635561514 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 7}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f274c013070 con 0x7f275c1037f0 2024-05-20T20:03:35.646 INFO:teuthology.orchestra.run.smithi028.stdout:201863462979 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.649+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f27400b2880 msgr2=0x7f27400b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f27400b2880 0x7f27400b4d40 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f275c103140 tx=0x7f2748009260 comp rx=0 tx=0).stop 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f275c1037f0 msgr2=0x7f275c19f850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f275c1037f0 0x7f275c19f850 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f274c0098e0 tx=0x7f274c005280 comp rx=0 tx=0).stop 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 shutdown_connections 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f27400b2880 0x7f27400b4d40 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275c1088f0 0x7f275c199a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f275c1037f0 0x7f275c19f850 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 --2- 172.21.15.28:0/2635561514 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275c1041a0 0x7f275c19fd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 >> 172.21.15.28:0/2635561514 conn(0x7f275c07a6a0 msgr2=0x7f275c1017d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:35.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 shutdown_connections 2024-05-20T20:03:35.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:35.650+0000 7f2761000700 1 -- 172.21.15.28:0/2635561514 wait complete. 2024-05-20T20:03:35.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:35 smithi154 ceph-mon[36314]: mgrmap e18: x(active, since 10m), standbys: z, y 2024-05-20T20:03:35.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:35 smithi033 ceph-mon[36179]: mgrmap e18: x(active, since 10m), standbys: z, y 2024-05-20T20:03:36.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 -- 172.21.15.28:0/1263033574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 msgr2=0x7fd5b8109990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:36.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 --2- 172.21.15.28:0/1263033574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 0x7fd5b8109990 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8009990 tx=0x7fd5a8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 -- 172.21.15.28:0/1263033574 shutdown_connections 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 --2- 172.21.15.28:0/1263033574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806bbe0 0x7fd5b81103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 --2- 172.21.15.28:0/1263033574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 0x7fd5b8109990 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 --2- 172.21.15.28:0/1263033574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd5b806b220 0x7fd5b806b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 -- 172.21.15.28:0/1263033574 >> 172.21.15.28:0/1263033574 conn(0x7fd5b80793f0 msgr2=0x7fd5b8079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 -- 172.21.15.28:0/1263033574 shutdown_connections 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.738+0000 7fd5be200700 1 -- 172.21.15.28:0/1263033574 wait complete. 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.739+0000 7fd5be200700 1 Processor -- start 2024-05-20T20:03:36.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.739+0000 7fd5be200700 1 -- start start 2024-05-20T20:03:36.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5be200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806b220 0x7fd5b819f8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5be200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd5b806bbe0 0x7fd5b819fe00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5be200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 0x7fd5b8199a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5be200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd5b81136c0 con 0x7fd5b806b220 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5be200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd5b8113540 con 0x7fd5b81095b0 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5be200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd5b8113840 con 0x7fd5b806bbe0 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5b7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806b220 0x7fd5b819f8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5b7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806b220 0x7fd5b819f8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52602/0 (socket says 172.21.15.28:52602) 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5b7400700 1 -- 172.21.15.28:0/1723452595 learned_addr learned my addr 172.21.15.28:0/1723452595 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5b6a00700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd5b806bbe0 0x7fd5b819fe00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:36.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.740+0000 7fd5b7e00700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 0x7fd5b8199a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5b7400700 1 -- 172.21.15.28:0/1723452595 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd5b806bbe0 msgr2=0x7fd5b819fe00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5b7400700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd5b806bbe0 0x7fd5b819fe00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5b7400700 1 -- 172.21.15.28:0/1723452595 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 msgr2=0x7fd5b8199a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5b7400700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 0x7fd5b8199a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5b7400700 1 -- 172.21.15.28:0/1723452595 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd5b819a1e0 con 0x7fd5b806b220 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5b7400700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806b220 0x7fd5b819f8c0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8000c00 tx=0x7fd5a800bfd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5afe00700 1 -- 172.21.15.28:0/1723452595 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd5a8027070 con 0x7fd5b806b220 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5afe00700 1 -- 172.21.15.28:0/1723452595 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd5a8022470 con 0x7fd5b806b220 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.741+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd5b819a470 con 0x7fd5b806b220 2024-05-20T20:03:36.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.742+0000 7fd5afe00700 1 -- 172.21.15.28:0/1723452595 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd5a8017650 con 0x7fd5b806b220 2024-05-20T20:03:36.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.742+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd5b81a6790 con 0x7fd5b806b220 2024-05-20T20:03:36.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.743+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd5b810ba90 con 0x7fd5b806b220 2024-05-20T20:03:36.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.746+0000 7fd5afe00700 1 -- 172.21.15.28:0/1723452595 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fd5a802a070 con 0x7fd5b806b220 2024-05-20T20:03:36.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.747+0000 7fd5afe00700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd59c0b2880 0x7fd59c0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:36.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.747+0000 7fd5afe00700 1 -- 172.21.15.28:0/1723452595 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fd5a80e2c30 con 0x7fd5b806b220 2024-05-20T20:03:36.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.747+0000 7fd5b6a00700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd59c0b2880 0x7fd59c0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:36.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.748+0000 7fd5b6a00700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd59c0b2880 0x7fd59c0b4d40 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fd5b8079b10 tx=0x7fd5a4009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:36.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.752+0000 7fd5afe00700 1 -- 172.21.15.28:0/1723452595 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fd5a805edf0 con 0x7fd5b806b220 2024-05-20T20:03:36.815 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:36 smithi028 ceph-mon[31178]: pgmap v386: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 0 B/s wr, 9 op/s 2024-05-20T20:03:36.815 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:36 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2635561514' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T20:03:36.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:36 smithi154 ceph-mon[36314]: pgmap v386: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 0 B/s wr, 9 op/s 2024-05-20T20:03:36.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:36 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2635561514' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T20:03:36.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:36 smithi033 ceph-mon[36179]: pgmap v386: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 0 B/s wr, 9 op/s 2024-05-20T20:03:36.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:36 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2635561514' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T20:03:36.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.989+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fd5b819afb0 con 0x7fd5b806b220 2024-05-20T20:03:36.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.990+0000 7fd5afe00700 1 -- 172.21.15.28:0/1723452595 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd5a8013070 con 0x7fd5b806b220 2024-05-20T20:03:36.990 INFO:teuthology.orchestra.run.smithi028.stdout:77309411423 2024-05-20T20:03:36.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd59c0b2880 msgr2=0x7fd59c0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd59c0b2880 0x7fd59c0b4d40 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fd5b8079b10 tx=0x7fd5a4009260 comp rx=0 tx=0).stop 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806b220 msgr2=0x7fd5b819f8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806b220 0x7fd5b819f8c0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8000c00 tx=0x7fd5a800bfd0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 shutdown_connections 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd59c0b2880 0x7fd59c0b4d40 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd5b81095b0 0x7fd5b8199a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd5b806b220 0x7fd5b819f8c0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 --2- 172.21.15.28:0/1723452595 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd5b806bbe0 0x7fd5b819fe00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.994+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 >> 172.21.15.28:0/1723452595 conn(0x7fd5b80793f0 msgr2=0x7fd5b81011a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.995+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 shutdown_connections 2024-05-20T20:03:36.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:36.995+0000 7fd5be200700 1 -- 172.21.15.28:0/1723452595 wait complete. 2024-05-20T20:03:37.817 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:37 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1723452595' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T20:03:37.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:37 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1723452595' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T20:03:37.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:37 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1723452595' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T20:03:38.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:38 smithi028 ceph-mon[31178]: pgmap v387: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 9 op/s 2024-05-20T20:03:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:38 smithi154 ceph-mon[36314]: pgmap v387: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 9 op/s 2024-05-20T20:03:38.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:38 smithi033 ceph-mon[36179]: pgmap v387: 97 pgs: 97 active+clean; 644 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 9 op/s 2024-05-20T20:03:39.215 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462973 got 201863462979 for osd.7 2024-05-20T20:03:39.215 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:39.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 -- 172.21.15.28:0/3435596209 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 msgr2=0x7f8bb8103910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 --2- 172.21.15.28:0/3435596209 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 0x7f8bb8103910 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f8ba40099c0 tx=0x7f8ba4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 -- 172.21.15.28:0/3435596209 shutdown_connections 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 --2- 172.21.15.28:0/3435596209 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb8103e50 0x7f8bb8110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 --2- 172.21.15.28:0/3435596209 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb8109540 0x7f8bb8109920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 --2- 172.21.15.28:0/3435596209 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 0x7f8bb8103910 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 -- 172.21.15.28:0/3435596209 >> 172.21.15.28:0/3435596209 conn(0x7f8bb80fefc0 msgr2=0x7f8bb81013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 -- 172.21.15.28:0/3435596209 shutdown_connections 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.109+0000 7f8bbe600700 1 -- 172.21.15.28:0/3435596209 wait complete. 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.110+0000 7f8bbe600700 1 Processor -- start 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.110+0000 7f8bbe600700 1 -- start start 2024-05-20T20:03:39.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbe600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 0x7f8bb8077430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbe600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb8103e50 0x7f8bb8075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbe600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb8109540 0x7f8bb8075ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbe600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb8113690 con 0x7f8bb8109540 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbe600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb8113510 con 0x7f8bb8103490 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbe600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb8113810 con 0x7f8bb8103e50 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbd200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 0x7f8bb8077430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbdc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb8109540 0x7f8bb8075ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bb7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb8103e50 0x7f8bb8075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbd200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 0x7f8bb8077430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51340/0 (socket says 172.21.15.28:51340) 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbd200700 1 -- 172.21.15.28:0/2933926197 learned_addr learned my addr 172.21.15.28:0/2933926197 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbdc00700 1 -- 172.21.15.28:0/2933926197 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb8103e50 msgr2=0x7f8bb8075840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:39.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbdc00700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb8103e50 0x7f8bb8075840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbdc00700 1 -- 172.21.15.28:0/2933926197 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 msgr2=0x7f8bb8077430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbdc00700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 0x7f8bb8077430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.111+0000 7f8bbdc00700 1 -- 172.21.15.28:0/2933926197 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8bb81ac4d0 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.112+0000 7f8bbdc00700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb8109540 0x7f8bb8075ea0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f8bb000efb0 tx=0x7f8bb000c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.112+0000 7f8bb5600700 1 -- 172.21.15.28:0/2933926197 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bb0003a00 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.112+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8bb81ac760 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.112+0000 7f8bb5600700 1 -- 172.21.15.28:0/2933926197 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8bb000ce50 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.112+0000 7f8bb5600700 1 -- 172.21.15.28:0/2933926197 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bb0004dd0 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.112+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8bb81acc20 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.113+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8bb810b9d0 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.116+0000 7f8bb5600700 1 -- 172.21.15.28:0/2933926197 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f8bb0012050 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.117+0000 7f8bb5600700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ba00b2880 0x7f8ba00b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.117+0000 7f8bb5600700 1 -- 172.21.15.28:0/2933926197 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f8bb00e2260 con 0x7f8bb8109540 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.118+0000 7f8bbd200700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ba00b2880 0x7f8ba00b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:39.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.118+0000 7f8bbd200700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ba00b2880 0x7f8ba00b4d40 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f8bac009ce0 tx=0x7f8bac009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:39.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.121+0000 7f8bb5600700 1 -- 172.21.15.28:0/2933926197 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f8bb0022020 con 0x7f8bb8109540 2024-05-20T20:03:39.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.371+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) -- 0x7f8bb810b3b0 con 0x7f8bb8109540 2024-05-20T20:03:39.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.372+0000 7f8bb5600700 1 -- 172.21.15.28:0/2933926197 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f8bb0010040 con 0x7f8bb8109540 2024-05-20T20:03:39.373 INFO:teuthology.orchestra.run.smithi028.stdout:133143986260 2024-05-20T20:03:39.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ba00b2880 msgr2=0x7f8ba00b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:39.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ba00b2880 0x7f8ba00b4d40 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f8bac009ce0 tx=0x7f8bac009260 comp rx=0 tx=0).stop 2024-05-20T20:03:39.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb8109540 msgr2=0x7f8bb8075ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:39.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb8109540 0x7f8bb8075ea0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f8bb000efb0 tx=0x7f8bb000c3a0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 shutdown_connections 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8ba00b2880 0x7f8ba00b4d40 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb8109540 0x7f8bb8075ea0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb8103490 0x7f8bb8077430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 --2- 172.21.15.28:0/2933926197 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb8103e50 0x7f8bb8075840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 >> 172.21.15.28:0/2933926197 conn(0x7f8bb80fefc0 msgr2=0x7f8bb8100730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 shutdown_connections 2024-05-20T20:03:39.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:39.375+0000 7f8bbe600700 1 -- 172.21.15.28:0/2933926197 wait complete. 2024-05-20T20:03:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:39 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2933926197' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T20:03:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:39 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2933926197' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T20:03:39.916 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:39 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2933926197' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T20:03:39.989 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411418 got 77309411423 for osd.2 2024-05-20T20:03:39.989 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:40.757 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:40 smithi028 ceph-mon[31178]: pgmap v388: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 426 B/s wr, 12 op/s 2024-05-20T20:03:40.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.753+0000 7fd8bf600700 1 -- 172.21.15.28:0/239577003 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 msgr2=0x7fd8b81081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:40.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.753+0000 7fd8bf600700 1 --2- 172.21.15.28:0/239577003 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b81081c0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fd8a40099c0 tx=0x7fd8a4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.754+0000 7fd8bf600700 1 -- 172.21.15.28:0/239577003 shutdown_connections 2024-05-20T20:03:40.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.754+0000 7fd8bf600700 1 --2- 172.21.15.28:0/239577003 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8b81088f0 0x7fd8b8110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.754+0000 7fd8bf600700 1 --2- 172.21.15.28:0/239577003 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8b81037f0 0x7fd8b8103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.754+0000 7fd8bf600700 1 --2- 172.21.15.28:0/239577003 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b81081c0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.754+0000 7fd8bf600700 1 -- 172.21.15.28:0/239577003 >> 172.21.15.28:0/239577003 conn(0x7fd8b807a6a0 msgr2=0x7fd8b807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.754+0000 7fd8bf600700 1 -- 172.21.15.28:0/239577003 shutdown_connections 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.754+0000 7fd8bf600700 1 -- 172.21.15.28:0/239577003 wait complete. 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.755+0000 7fd8bf600700 1 Processor -- start 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.755+0000 7fd8bf600700 1 -- start start 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bf600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8b81037f0 0x7fd8b819bd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bf600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b819c2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bf600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8b81088f0 0x7fd8b81a0650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bf600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd8b81136d0 con 0x7fd8b81041a0 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bf600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd8b8113550 con 0x7fd8b81088f0 2024-05-20T20:03:40.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bf600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd8b8113850 con 0x7fd8b81037f0 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8b7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b819c2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8b7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b819c2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45886/0 (socket says 172.21.15.28:45886) 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bd600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8b81088f0 0x7fd8b81a0650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8b7e00700 1 -- 172.21.15.28:0/1689490078 learned_addr learned my addr 172.21.15.28:0/1689490078 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bcc00700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8b81037f0 0x7fd8b819bd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.756+0000 7fd8bd600700 1 -- 172.21.15.28:0/1689490078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8b81037f0 msgr2=0x7fd8b819bd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8bd600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8b81037f0 0x7fd8b819bd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8bd600700 1 -- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 msgr2=0x7fd8b819c2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8bd600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b819c2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8bd600700 1 -- 172.21.15.28:0/1689490078 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd8b81a0d90 con 0x7fd8b81088f0 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8b7e00700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b819c2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:40.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8bd600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8b81088f0 0x7fd8b81a0650 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7fd8b000efc0 tx=0x7fd8b000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8b5600700 1 -- 172.21.15.28:0/1689490078 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8b00092e0 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8b5600700 1 -- 172.21.15.28:0/1689490078 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd8b0004bb0 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8b5600700 1 -- 172.21.15.28:0/1689490078 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8b0004020 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd8b81a1020 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.757+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd8b81a8900 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.759+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd8b806a230 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.761+0000 7fd8b5600700 1 -- 172.21.15.28:0/1689490078 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fd8b001b030 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.762+0000 7fd8b5600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8a00b2880 0x7fd8a00b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.763+0000 7fd8b5600700 1 -- 172.21.15.28:0/1689490078 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fd8b00e1d30 con 0x7fd8b81088f0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.763+0000 7fd8bcc00700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8a00b2880 0x7fd8a00b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.764+0000 7fd8bcc00700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8a00b2880 0x7fd8a00b4d40 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fd8ac009c60 tx=0x7fd8ac009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:40.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.767+0000 7fd8b5600700 1 -- 172.21.15.28:0/1689490078 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fd8b0022020 con 0x7fd8b81088f0 2024-05-20T20:03:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:40 smithi154 ceph-mon[36314]: pgmap v388: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 426 B/s wr, 12 op/s 2024-05-20T20:03:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:40 smithi033 ceph-mon[36179]: pgmap v388: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 426 B/s wr, 12 op/s 2024-05-20T20:03:40.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 -- 172.21.15.28:0/1561886641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c1046f0 msgr2=0x7f217c104ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:40.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 --2- 172.21.15.28:0/1561886641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c1046f0 0x7f217c104ad0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f216c009990 tx=0x7f216c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 -- 172.21.15.28:0/1561886641 shutdown_connections 2024-05-20T20:03:40.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 --2- 172.21.15.28:0/1561886641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c10dad0 0x7f217c10ff20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 --2- 172.21.15.28:0/1561886641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c1046f0 0x7f217c104ad0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 --2- 172.21.15.28:0/1561886641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f217c105010 0x7f217c10d500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 -- 172.21.15.28:0/1561886641 >> 172.21.15.28:0/1561886641 conn(0x7f217c0fde60 msgr2=0x7f217c100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 -- 172.21.15.28:0/1561886641 shutdown_connections 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.891+0000 7f2181600700 1 -- 172.21.15.28:0/1561886641 wait complete. 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.892+0000 7f2181600700 1 Processor -- start 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.892+0000 7f2181600700 1 -- start start 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2181600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c1046f0 0x7f217c102350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2181600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f217c105010 0x7f217c102890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2181600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c10dad0 0x7f217c102dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2181600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f217c1136b0 con 0x7f217c1046f0 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2181600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f217c113530 con 0x7f217c10dad0 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2181600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f217c113830 con 0x7f217c105010 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c1046f0 0x7f217c102350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c1046f0 0x7f217c102350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45900/0 (socket says 172.21.15.28:45900) 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 -- 172.21.15.28:0/2182064383 learned_addr learned my addr 172.21.15.28:0/2182064383 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2180c00700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c10dad0 0x7f217c102dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f2173e00700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f217c105010 0x7f217c102890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 -- 172.21.15.28:0/2182064383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f217c105010 msgr2=0x7f217c102890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f217c105010 0x7f217c102890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 -- 172.21.15.28:0/2182064383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c10dad0 msgr2=0x7f217c102dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c10dad0 0x7f217c102dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.893+0000 7f217be00700 1 -- 172.21.15.28:0/2182064383 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f217c1a80f0 con 0x7f217c1046f0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.894+0000 7f217be00700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c1046f0 0x7f217c102350 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f216c000c00 tx=0x7f216c0057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.894+0000 7f2179600700 1 -- 172.21.15.28:0/2182064383 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f216c027070 con 0x7f217c1046f0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.894+0000 7f2179600700 1 -- 172.21.15.28:0/2182064383 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f216c022470 con 0x7f217c1046f0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.894+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f217c1a8380 con 0x7f217c1046f0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.894+0000 7f2179600700 1 -- 172.21.15.28:0/2182064383 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f216c017650 con 0x7f217c1046f0 2024-05-20T20:03:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.894+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f217c1a8860 con 0x7f217c1046f0 2024-05-20T20:03:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.896+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f217c06a230 con 0x7f217c1046f0 2024-05-20T20:03:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.898+0000 7f2179600700 1 -- 172.21.15.28:0/2182064383 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f216c02a070 con 0x7f217c1046f0 2024-05-20T20:03:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.899+0000 7f2179600700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21600b2880 0x7f21600b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.900+0000 7f2179600700 1 -- 172.21.15.28:0/2182064383 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f216c0e2ff0 con 0x7f217c1046f0 2024-05-20T20:03:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.900+0000 7f2173e00700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21600b2880 0x7f21600b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.900+0000 7f2173e00700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21600b2880 0x7f21600b4d40 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f2164009680 tx=0x7f2164009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:40.902+0000 7f2179600700 1 -- 172.21.15.28:0/2182064383 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f216c013070 con 0x7f217c1046f0 2024-05-20T20:03:41.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.019+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fd8b81a8ad0 con 0x7fd8b81088f0 2024-05-20T20:03:41.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.019+0000 7fd8b5600700 1 -- 172.21.15.28:0/1689490078 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd8b0010040 con 0x7fd8b81088f0 2024-05-20T20:03:41.021 INFO:teuthology.orchestra.run.smithi028.stdout:34359738476 2024-05-20T20:03:41.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8a00b2880 msgr2=0x7fd8a00b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:41.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8a00b2880 0x7fd8a00b4d40 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fd8ac009c60 tx=0x7fd8ac009260 comp rx=0 tx=0).stop 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8b81088f0 msgr2=0x7fd8b81a0650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8b81088f0 0x7fd8b81a0650 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7fd8b000efc0 tx=0x7fd8b000c3a0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 shutdown_connections 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd8a00b2880 0x7fd8a00b4d40 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8b81088f0 0x7fd8b81a0650 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8b81037f0 0x7fd8b819bd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 --2- 172.21.15.28:0/1689490078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd8b81041a0 0x7fd8b819c2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 >> 172.21.15.28:0/1689490078 conn(0x7fd8b807a6a0 msgr2=0x7fd8b81016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 shutdown_connections 2024-05-20T20:03:41.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.023+0000 7fd8bf600700 1 -- 172.21.15.28:0/1689490078 wait complete. 2024-05-20T20:03:41.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.174+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f217c02e560 con 0x7f217c1046f0 2024-05-20T20:03:41.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.174+0000 7f2179600700 1 -- 172.21.15.28:0/2182064383 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f216c0aad80 con 0x7f217c1046f0 2024-05-20T20:03:41.176 INFO:teuthology.orchestra.run.smithi028.stdout:107374182490 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.177+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21600b2880 msgr2=0x7f21600b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.177+0000 7f2181600700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21600b2880 0x7f21600b4d40 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f2164009680 tx=0x7f2164009260 comp rx=0 tx=0).stop 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.177+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c1046f0 msgr2=0x7f217c102350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.177+0000 7f2181600700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c1046f0 0x7f217c102350 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f216c000c00 tx=0x7f216c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 shutdown_connections 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f21600b2880 0x7f21600b4d40 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f217c10dad0 0x7f217c102dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f217c1046f0 0x7f217c102350 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 --2- 172.21.15.28:0/2182064383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f217c105010 0x7f217c102890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:41.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 >> 172.21.15.28:0/2182064383 conn(0x7f217c0fde60 msgr2=0x7f217c0feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:41.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 shutdown_connections 2024-05-20T20:03:41.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:41.178+0000 7f2181600700 1 -- 172.21.15.28:0/2182064383 wait complete. 2024-05-20T20:03:41.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:41 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1689490078' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T20:03:41.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:41 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2182064383' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T20:03:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:41 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1689490078' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T20:03:41.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:41 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2182064383' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T20:03:41.990 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:41 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1689490078' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T20:03:41.990 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:41 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2182064383' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T20:03:42.062 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986255 got 133143986260 for osd.4 2024-05-20T20:03:42.062 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:42.656 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182483 got 107374182490 for osd.3 2024-05-20T20:03:42.657 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:42.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:42 smithi154 ceph-mon[36314]: pgmap v389: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:42 smithi033 ceph-mon[36179]: pgmap v389: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:43.030 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:42 smithi028 ceph-mon[31178]: pgmap v389: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:43.091 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738470 got 34359738476 for osd.0 2024-05-20T20:03:43.091 DEBUG:teuthology.parallel:result is None 2024-05-20T20:03:43.091 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-20T20:03:43.091 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T20:03:43.535 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:43.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:43.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:43 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:43.943 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:43 smithi028 ceph-mon[31178]: pgmap v390: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 426 B/s wr, 12 op/s 2024-05-20T20:03:44.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:44.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:43 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:44.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:43 smithi154 ceph-mon[36314]: pgmap v390: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 426 B/s wr, 12 op/s 2024-05-20T20:03:44.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:44.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:43 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:44.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:43 smithi033 ceph-mon[36179]: pgmap v390: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 426 B/s wr, 12 op/s 2024-05-20T20:03:45.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:03:45.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:03:45.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:45.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:03:45.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:03:45.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:03:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:03:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:03:46.018 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:45 smithi028 ceph-mon[31178]: pgmap v391: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:46.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 -- 172.21.15.28:0/2648372143 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3970109a80 msgr2=0x7f3970109f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:46.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 --2- 172.21.15.28:0/2648372143 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3970109a80 0x7f3970109f00 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f395c0098f0 tx=0x7f395c009c00 comp rx=0 tx=0).stop 2024-05-20T20:03:46.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 -- 172.21.15.28:0/2648372143 shutdown_connections 2024-05-20T20:03:46.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 --2- 172.21.15.28:0/2648372143 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f397010a440 0x7f39701168e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 --2- 172.21.15.28:0/2648372143 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f397010e950 0x7f397010ed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 --2- 172.21.15.28:0/2648372143 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3970109a80 0x7f3970109f00 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 -- 172.21.15.28:0/2648372143 >> 172.21.15.28:0/2648372143 conn(0x7f39700792d0 msgr2=0x7f3970108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 -- 172.21.15.28:0/2648372143 shutdown_connections 2024-05-20T20:03:46.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.074+0000 7f3975a00700 1 -- 172.21.15.28:0/2648372143 wait complete. 2024-05-20T20:03:46.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.076+0000 7f3975a00700 1 Processor -- start 2024-05-20T20:03:46.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.076+0000 7f3975a00700 1 -- start start 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f3975a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3970109a80 0x7f39701a01b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f3975a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f397010a440 0x7f39701a06f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f3975a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f397010e950 0x7f39701a4a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f3975a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3970119c80 con 0x7f3970109a80 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f3975a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3970119b00 con 0x7f397010a440 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f3975a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3970119e00 con 0x7f397010e950 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f396fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3970109a80 0x7f39701a01b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f396f400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f397010a440 0x7f39701a06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f396fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3970109a80 0x7f39701a01b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45926/0 (socket says 172.21.15.28:45926) 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f396fe00700 1 -- 172.21.15.28:0/2870330397 learned_addr learned my addr 172.21.15.28:0/2870330397 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:46.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f3975000700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f397010e950 0x7f39701a4a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.077+0000 7f396f400700 1 -- 172.21.15.28:0/2870330397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f397010e950 msgr2=0x7f39701a4a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396f400700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f397010e950 0x7f39701a4a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396f400700 1 -- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3970109a80 msgr2=0x7f39701a01b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396f400700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3970109a80 0x7f39701a01b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396f400700 1 -- 172.21.15.28:0/2870330397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f39701a51c0 con 0x7f397010a440 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396f400700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f397010a440 0x7f39701a06f0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f395c00c010 tx=0x7f395c00fa00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396cc00700 1 -- 172.21.15.28:0/2870330397 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f395c0050c0 con 0x7f397010a440 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396cc00700 1 -- 172.21.15.28:0/2870330397 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f395c00b560 con 0x7f397010a440 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f39701a5450 con 0x7f397010a440 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f396cc00700 1 -- 172.21.15.28:0/2870330397 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f395c003a00 con 0x7f397010a440 2024-05-20T20:03:46.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.078+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f39701acd30 con 0x7f397010a440 2024-05-20T20:03:46.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.079+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f397006a230 con 0x7f397010a440 2024-05-20T20:03:46.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.082+0000 7f396cc00700 1 -- 172.21.15.28:0/2870330397 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f395c00b710 con 0x7f397010a440 2024-05-20T20:03:46.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.084+0000 7f396cc00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f39540b2900 0x7f39540b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:46.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.084+0000 7f396cc00700 1 -- 172.21.15.28:0/2870330397 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f395c0e2f60 con 0x7f397010a440 2024-05-20T20:03:46.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.084+0000 7f396fe00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f39540b2900 0x7f39540b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:46.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.085+0000 7f396fe00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f39540b2900 0x7f39540b4dc0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f3960009ce0 tx=0x7f3960009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:46.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.089+0000 7f396cc00700 1 -- 172.21.15.28:0/2870330397 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f395c013030 con 0x7f397010a440 2024-05-20T20:03:46.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.321+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f39701a1510 con 0x7f39540b2900 2024-05-20T20:03:46.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.331+0000 7f396cc00700 1 -- 172.21.15.28:0/2870330397 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+259110 (secure 0 0 0) 0x7f39701a1510 con 0x7f39540b2900 2024-05-20T20:03:46.339 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:03:46.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.338+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f39540b2900 msgr2=0x7f39540b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:46.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.338+0000 7f3975a00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f39540b2900 0x7f39540b4dc0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f3960009ce0 tx=0x7f3960009260 comp rx=0 tx=0).stop 2024-05-20T20:03:46.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.338+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f397010a440 msgr2=0x7f39701a06f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:46.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.338+0000 7f3975a00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f397010a440 0x7f39701a06f0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f395c00c010 tx=0x7f395c00fa00 comp rx=0 tx=0).stop 2024-05-20T20:03:46.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 shutdown_connections 2024-05-20T20:03:46.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f39540b2900 0x7f39540b4dc0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f397010e950 0x7f39701a4a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3970109a80 0x7f39701a01b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 --2- 172.21.15.28:0/2870330397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f397010a440 0x7f39701a06f0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:46.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 >> 172.21.15.28:0/2870330397 conn(0x7f39700792d0 msgr2=0x7f39700799f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:46.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 shutdown_connections 2024-05-20T20:03:46.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:46.339+0000 7f3975a00700 1 -- 172.21.15.28:0/2870330397 wait complete. 2024-05-20T20:03:46.342 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:03:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:45 smithi154 ceph-mon[36314]: pgmap v391: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:45 smithi033 ceph-mon[36179]: pgmap v391: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:46 smithi028 ceph-mon[31178]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:03:47.312 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":391,"stamp":"2024-05-20T20:03:45.428890+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":660140,"num_objects":102,"num_object_clones":0,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":0,"num_read":1287,"num_read_kb":749,"num_write":376,"num_write_kb":2293,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":809,"ondisk_log_size":809,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347484,"kb_used_data":10812,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124594340,"statfs":{"total":1151940427776,"available":1151584604160,"internally_reserved":0,"allocated":11071488,"data_stored":7076040,"data_compressed":177837,"data_compressed_allocated":233472,"data_compressed_original":1966080,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":390,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":112,"num_read_kb":57,"num_write":7,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004642"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406944+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:03:33.406944+0000","last_peered":"2024-05-20T20:03:33.406944+0000","last_clean":"2024-05-20T20:03:33.406944+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:03:33.406944+0000","last_undegraded":"2024-05-20T20:03:33.406944+0000","last_fullsized":"2024-05-20T20:03:33.406944+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.395385+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:03:33.395385+0000","last_peered":"2024-05-20T20:03:33.395385+0000","last_clean":"2024-05-20T20:03:33.395385+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:03:33.395385+0000","last_undegraded":"2024-05-20T20:03:33.395385+0000","last_fullsized":"2024-05-20T20:03:33.395385+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.401648+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:03:33.401648+0000","last_peered":"2024-05-20T20:03:33.401648+0000","last_clean":"2024-05-20T20:03:33.401648+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:03:33.401648+0000","last_undegraded":"2024-05-20T20:03:33.401648+0000","last_fullsized":"2024-05-20T20:03:33.401648+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.370508+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:03:33.370508+0000","last_peered":"2024-05-20T20:03:33.370508+0000","last_clean":"2024-05-20T20:03:33.370508+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:03:33.370508+0000","last_undegraded":"2024-05-20T20:03:33.370508+0000","last_fullsized":"2024-05-20T20:03:33.370508+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":24,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.390381+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:03:33.390381+0000","last_peered":"2024-05-20T20:03:33.390381+0000","last_clean":"2024-05-20T20:03:33.390381+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:03:33.390381+0000","last_undegraded":"2024-05-20T20:03:33.390381+0000","last_fullsized":"2024-05-20T20:03:33.390381+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.366770+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:03:33.366770+0000","last_peered":"2024-05-20T20:03:33.366770+0000","last_clean":"2024-05-20T20:03:33.366770+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:03:33.366770+0000","last_undegraded":"2024-05-20T20:03:33.366770+0000","last_fullsized":"2024-05-20T20:03:33.366770+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.379472+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:03:33.379472+0000","last_peered":"2024-05-20T20:03:33.379472+0000","last_clean":"2024-05-20T20:03:33.379472+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:03:33.379472+0000","last_undegraded":"2024-05-20T20:03:33.379472+0000","last_fullsized":"2024-05-20T20:03:33.379472+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.397632+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:03:33.397632+0000","last_peered":"2024-05-20T20:03:33.397632+0000","last_clean":"2024-05-20T20:03:33.397632+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:03:33.397632+0000","last_undegraded":"2024-05-20T20:03:33.397632+0000","last_fullsized":"2024-05-20T20:03:33.397632+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.378063+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:03:33.378063+0000","last_peered":"2024-05-20T20:03:33.378063+0000","last_clean":"2024-05-20T20:03:33.378063+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:03:33.378063+0000","last_undegraded":"2024-05-20T20:03:33.378063+0000","last_fullsized":"2024-05-20T20:03:33.378063+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":42,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.371132+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:03:33.371132+0000","last_peered":"2024-05-20T20:03:33.371132+0000","last_clean":"2024-05-20T20:03:33.371132+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:03:33.371132+0000","last_undegraded":"2024-05-20T20:03:33.371132+0000","last_fullsized":"2024-05-20T20:03:33.371132+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.386382+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:03:33.386382+0000","last_peered":"2024-05-20T20:03:33.386382+0000","last_clean":"2024-05-20T20:03:33.386382+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:03:33.386382+0000","last_undegraded":"2024-05-20T20:03:33.386382+0000","last_fullsized":"2024-05-20T20:03:33.386382+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.362232+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:03:33.362232+0000","last_peered":"2024-05-20T20:03:33.362232+0000","last_clean":"2024-05-20T20:03:33.362232+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:03:33.362232+0000","last_undegraded":"2024-05-20T20:03:33.362232+0000","last_fullsized":"2024-05-20T20:03:33.362232+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.400964+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:03:33.400964+0000","last_peered":"2024-05-20T20:03:33.400964+0000","last_clean":"2024-05-20T20:03:33.400964+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:03:33.400964+0000","last_undegraded":"2024-05-20T20:03:33.400964+0000","last_fullsized":"2024-05-20T20:03:33.400964+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.397551+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:03:33.397551+0000","last_peered":"2024-05-20T20:03:33.397551+0000","last_clean":"2024-05-20T20:03:33.397551+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:03:33.397551+0000","last_undegraded":"2024-05-20T20:03:33.397551+0000","last_fullsized":"2024-05-20T20:03:33.397551+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.396857+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:03:33.396857+0000","last_peered":"2024-05-20T20:03:33.396857+0000","last_clean":"2024-05-20T20:03:33.396857+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:03:33.396857+0000","last_undegraded":"2024-05-20T20:03:33.396857+0000","last_fullsized":"2024-05-20T20:03:33.396857+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.405677+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:03:33.405677+0000","last_peered":"2024-05-20T20:03:33.405677+0000","last_clean":"2024-05-20T20:03:33.405677+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:03:33.405677+0000","last_undegraded":"2024-05-20T20:03:33.405677+0000","last_fullsized":"2024-05-20T20:03:33.405677+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406980+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:03:33.406980+0000","last_peered":"2024-05-20T20:03:33.406980+0000","last_clean":"2024-05-20T20:03:33.406980+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:03:33.406980+0000","last_undegraded":"2024-05-20T20:03:33.406980+0000","last_fullsized":"2024-05-20T20:03:33.406980+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.379709+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:03:33.379709+0000","last_peered":"2024-05-20T20:03:33.379709+0000","last_clean":"2024-05-20T20:03:33.379709+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:03:33.379709+0000","last_undegraded":"2024-05-20T20:03:33.379709+0000","last_fullsized":"2024-05-20T20:03:33.379709+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.390834+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:03:33.390834+0000","last_peered":"2024-05-20T20:03:33.390834+0000","last_clean":"2024-05-20T20:03:33.390834+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:03:33.390834+0000","last_undegraded":"2024-05-20T20:03:33.390834+0000","last_fullsized":"2024-05-20T20:03:33.390834+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"78'13","reported_seq":159,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:42.943725+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:03:42.943725+0000","last_peered":"2024-05-20T20:03:42.943725+0000","last_clean":"2024-05-20T20:03:42.943725+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:03:42.943725+0000","last_undegraded":"2024-05-20T20:03:42.943725+0000","last_fullsized":"2024-05-20T20:03:42.943725+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":119,"num_read_kb":119,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.395728+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:03:33.395728+0000","last_peered":"2024-05-20T20:03:33.395728+0000","last_clean":"2024-05-20T20:03:33.395728+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:03:33.395728+0000","last_undegraded":"2024-05-20T20:03:33.395728+0000","last_fullsized":"2024-05-20T20:03:33.395728+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753264+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:02:04.753264+0000","last_peered":"2024-05-20T20:02:04.753264+0000","last_clean":"2024-05-20T20:02:04.753264+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:02:04.753264+0000","last_undegraded":"2024-05-20T20:02:04.753264+0000","last_fullsized":"2024-05-20T20:02:04.753264+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.407890+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:03:33.407890+0000","last_peered":"2024-05-20T20:03:33.407890+0000","last_clean":"2024-05-20T20:03:33.407890+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:03:33.407890+0000","last_undegraded":"2024-05-20T20:03:33.407890+0000","last_fullsized":"2024-05-20T20:03:33.407890+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"78'12","reported_seq":39,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752192+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:02:04.752192+0000","last_peered":"2024-05-20T20:02:04.752192+0000","last_clean":"2024-05-20T20:02:04.752192+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:02:04.752192+0000","last_undegraded":"2024-05-20T20:02:04.752192+0000","last_fullsized":"2024-05-20T20:02:04.752192+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.399806+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:03:33.399806+0000","last_peered":"2024-05-20T20:03:33.399806+0000","last_clean":"2024-05-20T20:03:33.399806+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:03:33.399806+0000","last_undegraded":"2024-05-20T20:03:33.399806+0000","last_fullsized":"2024-05-20T20:03:33.399806+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751794+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:02:04.751794+0000","last_peered":"2024-05-20T20:02:04.751794+0000","last_clean":"2024-05-20T20:02:04.751794+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:02:04.751794+0000","last_undegraded":"2024-05-20T20:02:04.751794+0000","last_fullsized":"2024-05-20T20:02:04.751794+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.383831+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:03:33.383831+0000","last_peered":"2024-05-20T20:03:33.383831+0000","last_clean":"2024-05-20T20:03:33.383831+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:03:33.383831+0000","last_undegraded":"2024-05-20T20:03:33.383831+0000","last_fullsized":"2024-05-20T20:03:33.383831+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"80'7","reported_seq":153,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:40.844710+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:03:40.844710+0000","last_peered":"2024-05-20T20:03:40.844710+0000","last_clean":"2024-05-20T20:03:40.844710+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:03:40.844710+0000","last_undegraded":"2024-05-20T20:03:40.844710+0000","last_fullsized":"2024-05-20T20:03:40.844710+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1154,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":120,"num_read_kb":4,"num_write":4,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.385883+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:03:33.385883+0000","last_peered":"2024-05-20T20:03:33.385883+0000","last_clean":"2024-05-20T20:03:33.385883+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:03:33.385883+0000","last_undegraded":"2024-05-20T20:03:33.385883+0000","last_fullsized":"2024-05-20T20:03:33.385883+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748817+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:02:04.748817+0000","last_peered":"2024-05-20T20:02:04.748817+0000","last_clean":"2024-05-20T20:02:04.748817+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:02:04.748817+0000","last_undegraded":"2024-05-20T20:02:04.748817+0000","last_fullsized":"2024-05-20T20:02:04.748817+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.381104+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:03:33.381104+0000","last_peered":"2024-05-20T20:03:33.381104+0000","last_clean":"2024-05-20T20:03:33.381104+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:03:33.381104+0000","last_undegraded":"2024-05-20T20:03:33.381104+0000","last_fullsized":"2024-05-20T20:03:33.381104+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153602+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:02:05.153602+0000","last_peered":"2024-05-20T20:02:05.153602+0000","last_clean":"2024-05-20T20:02:05.153602+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:02:05.153602+0000","last_undegraded":"2024-05-20T20:02:05.153602+0000","last_fullsized":"2024-05-20T20:02:05.153602+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.405402+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:03:33.405402+0000","last_peered":"2024-05-20T20:03:33.405402+0000","last_clean":"2024-05-20T20:03:33.405402+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:03:33.405402+0000","last_undegraded":"2024-05-20T20:03:33.405402+0000","last_fullsized":"2024-05-20T20:03:33.405402+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748365+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:02:04.748365+0000","last_peered":"2024-05-20T20:02:04.748365+0000","last_clean":"2024-05-20T20:02:04.748365+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:02:04.748365+0000","last_undegraded":"2024-05-20T20:02:04.748365+0000","last_fullsized":"2024-05-20T20:02:04.748365+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.399298+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:03:33.399298+0000","last_peered":"2024-05-20T20:03:33.399298+0000","last_clean":"2024-05-20T20:03:33.399298+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:03:33.399298+0000","last_undegraded":"2024-05-20T20:03:33.399298+0000","last_fullsized":"2024-05-20T20:03:33.399298+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"80'5","reported_seq":263,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:41.845331+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:03:41.845331+0000","last_peered":"2024-05-20T20:03:41.845331+0000","last_clean":"2024-05-20T20:03:41.845331+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:03:41.845331+0000","last_undegraded":"2024-05-20T20:03:41.845331+0000","last_fullsized":"2024-05-20T20:03:41.845331+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":232,"num_read_kb":232,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.381428+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:03:33.381428+0000","last_peered":"2024-05-20T20:03:33.381428+0000","last_clean":"2024-05-20T20:03:33.381428+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:03:33.381428+0000","last_undegraded":"2024-05-20T20:03:33.381428+0000","last_fullsized":"2024-05-20T20:03:33.381428+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930581+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:02:04.930581+0000","last_peered":"2024-05-20T20:02:04.930581+0000","last_clean":"2024-05-20T20:02:04.930581+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:02:04.930581+0000","last_undegraded":"2024-05-20T20:02:04.930581+0000","last_fullsized":"2024-05-20T20:02:04.930581+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406607+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:03:33.406607+0000","last_peered":"2024-05-20T20:03:33.406607+0000","last_clean":"2024-05-20T20:03:33.406607+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:03:33.406607+0000","last_undegraded":"2024-05-20T20:03:33.406607+0000","last_fullsized":"2024-05-20T20:03:33.406607+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.749018+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:02:04.749018+0000","last_peered":"2024-05-20T20:02:04.749018+0000","last_clean":"2024-05-20T20:02:04.749018+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:02:04.749018+0000","last_undegraded":"2024-05-20T20:02:04.749018+0000","last_fullsized":"2024-05-20T20:02:04.749018+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.384161+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:03:33.384161+0000","last_peered":"2024-05-20T20:03:33.384161+0000","last_clean":"2024-05-20T20:03:33.384161+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:03:33.384161+0000","last_undegraded":"2024-05-20T20:03:33.384161+0000","last_fullsized":"2024-05-20T20:03:33.384161+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"80'34","reported_seq":303,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:41.846010+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:03:41.846010+0000","last_peered":"2024-05-20T20:03:41.846010+0000","last_clean":"2024-05-20T20:03:41.846010+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:03:41.846010+0000","last_undegraded":"2024-05-20T20:03:41.846010+0000","last_fullsized":"2024-05-20T20:03:41.846010+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62584,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":242,"num_read_kb":182,"num_write":36,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.385739+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:03:33.385739+0000","last_peered":"2024-05-20T20:03:33.385739+0000","last_clean":"2024-05-20T20:03:33.385739+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:03:33.385739+0000","last_undegraded":"2024-05-20T20:03:33.385739+0000","last_fullsized":"2024-05-20T20:03:33.385739+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.773205+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:02:04.773205+0000","last_peered":"2024-05-20T20:02:04.773205+0000","last_clean":"2024-05-20T20:02:04.773205+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:02:04.773205+0000","last_undegraded":"2024-05-20T20:02:04.773205+0000","last_fullsized":"2024-05-20T20:02:04.773205+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404483+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:03:33.404483+0000","last_peered":"2024-05-20T20:03:33.404483+0000","last_clean":"2024-05-20T20:03:33.404483+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:03:33.404483+0000","last_undegraded":"2024-05-20T20:03:33.404483+0000","last_fullsized":"2024-05-20T20:03:33.404483+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748855+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:02:04.748855+0000","last_peered":"2024-05-20T20:02:04.748855+0000","last_clean":"2024-05-20T20:02:04.748855+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:02:04.748855+0000","last_undegraded":"2024-05-20T20:02:04.748855+0000","last_fullsized":"2024-05-20T20:02:04.748855+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":36,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.400099+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:03:33.400099+0000","last_peered":"2024-05-20T20:03:33.400099+0000","last_clean":"2024-05-20T20:03:33.400099+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:03:33.400099+0000","last_undegraded":"2024-05-20T20:03:33.400099+0000","last_fullsized":"2024-05-20T20:03:33.400099+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"78'5","reported_seq":150,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:41.943419+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:03:41.943419+0000","last_peered":"2024-05-20T20:03:41.943419+0000","last_clean":"2024-05-20T20:03:41.943419+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:03:41.943419+0000","last_undegraded":"2024-05-20T20:03:41.943419+0000","last_fullsized":"2024-05-20T20:03:41.943419+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":119,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.374449+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:03:33.374449+0000","last_peered":"2024-05-20T20:03:33.374449+0000","last_clean":"2024-05-20T20:03:33.374449+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:03:33.374449+0000","last_undegraded":"2024-05-20T20:03:33.374449+0000","last_fullsized":"2024-05-20T20:03:33.374449+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":69,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.169710+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:02:05.169710+0000","last_peered":"2024-05-20T20:02:05.169710+0000","last_clean":"2024-05-20T20:02:05.169710+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:02:05.169710+0000","last_undegraded":"2024-05-20T20:02:05.169710+0000","last_fullsized":"2024-05-20T20:02:05.169710+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.378302+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:03:33.378302+0000","last_peered":"2024-05-20T20:03:33.378302+0000","last_clean":"2024-05-20T20:03:33.378302+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:03:33.378302+0000","last_undegraded":"2024-05-20T20:03:33.378302+0000","last_fullsized":"2024-05-20T20:03:33.378302+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"78'2","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752194+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:02:04.752194+0000","last_peered":"2024-05-20T20:02:04.752194+0000","last_clean":"2024-05-20T20:02:04.752194+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:02:04.752194+0000","last_undegraded":"2024-05-20T20:02:04.752194+0000","last_fullsized":"2024-05-20T20:02:04.752194+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.375428+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:03:33.375428+0000","last_peered":"2024-05-20T20:03:33.375428+0000","last_clean":"2024-05-20T20:03:33.375428+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:03:33.375428+0000","last_undegraded":"2024-05-20T20:03:33.375428+0000","last_fullsized":"2024-05-20T20:03:33.375428+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.757207+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:02:04.757207+0000","last_peered":"2024-05-20T20:02:04.757207+0000","last_clean":"2024-05-20T20:02:04.757207+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:02:04.757207+0000","last_undegraded":"2024-05-20T20:02:04.757207+0000","last_fullsized":"2024-05-20T20:02:04.757207+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398629+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:03:33.398629+0000","last_peered":"2024-05-20T20:03:33.398629+0000","last_clean":"2024-05-20T20:03:33.398629+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:03:33.398629+0000","last_undegraded":"2024-05-20T20:03:33.398629+0000","last_fullsized":"2024-05-20T20:03:33.398629+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"78'5","reported_seq":151,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:42.944660+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:03:42.944660+0000","last_peered":"2024-05-20T20:03:42.944660+0000","last_clean":"2024-05-20T20:03:42.944660+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:03:42.944660+0000","last_undegraded":"2024-05-20T20:03:42.944660+0000","last_fullsized":"2024-05-20T20:03:42.944660+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":119,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.395302+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:03:33.395302+0000","last_peered":"2024-05-20T20:03:33.395302+0000","last_clean":"2024-05-20T20:03:33.395302+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:03:33.395302+0000","last_undegraded":"2024-05-20T20:03:33.395302+0000","last_fullsized":"2024-05-20T20:03:33.395302+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.372182+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:03:33.372182+0000","last_peered":"2024-05-20T20:03:33.372182+0000","last_clean":"2024-05-20T20:03:33.372182+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:03:33.372182+0000","last_undegraded":"2024-05-20T20:03:33.372182+0000","last_fullsized":"2024-05-20T20:03:33.372182+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"78'5","reported_seq":149,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:40.940673+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:03:40.940673+0000","last_peered":"2024-05-20T20:03:40.940673+0000","last_clean":"2024-05-20T20:03:40.940673+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:03:40.940673+0000","last_undegraded":"2024-05-20T20:03:40.940673+0000","last_fullsized":"2024-05-20T20:03:40.940673+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":117,"num_read_kb":117,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.402541+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:03:33.402541+0000","last_peered":"2024-05-20T20:03:33.402541+0000","last_clean":"2024-05-20T20:03:33.402541+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:03:33.402541+0000","last_undegraded":"2024-05-20T20:03:33.402541+0000","last_fullsized":"2024-05-20T20:03:33.402541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":36,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154652+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:02:05.154652+0000","last_peered":"2024-05-20T20:02:05.154652+0000","last_clean":"2024-05-20T20:02:05.154652+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:02:05.154652+0000","last_undegraded":"2024-05-20T20:02:05.154652+0000","last_fullsized":"2024-05-20T20:02:05.154652+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"20'92","reported_seq":100,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930195+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:02:04.930195+0000","last_peered":"2024-05-20T20:02:04.930195+0000","last_clean":"2024-05-20T20:02:04.930195+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:02:04.930195+0000","last_undegraded":"2024-05-20T20:02:04.930195+0000","last_fullsized":"2024-05-20T20:02:04.930195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398178+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:03:33.398178+0000","last_peered":"2024-05-20T20:03:33.398178+0000","last_clean":"2024-05-20T20:03:33.398178+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:03:33.398178+0000","last_undegraded":"2024-05-20T20:03:33.398178+0000","last_fullsized":"2024-05-20T20:03:33.398178+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930350+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:02:04.930350+0000","last_peered":"2024-05-20T20:02:04.930350+0000","last_clean":"2024-05-20T20:02:04.930350+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:02:04.930350+0000","last_undegraded":"2024-05-20T20:02:04.930350+0000","last_fullsized":"2024-05-20T20:02:04.930350+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752065+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:02:04.752065+0000","last_peered":"2024-05-20T20:02:04.752065+0000","last_clean":"2024-05-20T20:02:04.752065+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:02:04.752065+0000","last_undegraded":"2024-05-20T20:02:04.752065+0000","last_fullsized":"2024-05-20T20:02:04.752065+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398150+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:03:33.398150+0000","last_peered":"2024-05-20T20:03:33.398150+0000","last_clean":"2024-05-20T20:03:33.398150+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:03:33.398150+0000","last_undegraded":"2024-05-20T20:03:33.398150+0000","last_fullsized":"2024-05-20T20:03:33.398150+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"80'16","reported_seq":59,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:29.755099+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:03:29.755099+0000","last_peered":"2024-05-20T20:03:29.755099+0000","last_clean":"2024-05-20T20:03:29.755099+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:03:29.755099+0000","last_undegraded":"2024-05-20T20:03:29.755099+0000","last_fullsized":"2024-05-20T20:03:29.755099+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":29,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.390053+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:03:33.390053+0000","last_peered":"2024-05-20T20:03:33.390053+0000","last_clean":"2024-05-20T20:03:33.390053+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:03:33.390053+0000","last_undegraded":"2024-05-20T20:03:33.390053+0000","last_fullsized":"2024-05-20T20:03:33.390053+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":35,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.155373+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:02:05.155373+0000","last_peered":"2024-05-20T20:02:05.155373+0000","last_clean":"2024-05-20T20:02:05.155373+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:02:05.155373+0000","last_undegraded":"2024-05-20T20:02:05.155373+0000","last_fullsized":"2024-05-20T20:02:05.155373+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":34,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404085+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:03:33.404085+0000","last_peered":"2024-05-20T20:03:33.404085+0000","last_clean":"2024-05-20T20:03:33.404085+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:03:33.404085+0000","last_undegraded":"2024-05-20T20:03:33.404085+0000","last_fullsized":"2024-05-20T20:03:33.404085+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153898+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:02:05.153898+0000","last_peered":"2024-05-20T20:02:05.153898+0000","last_clean":"2024-05-20T20:02:05.153898+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:02:05.153898+0000","last_undegraded":"2024-05-20T20:02:05.153898+0000","last_fullsized":"2024-05-20T20:02:05.153898+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":23,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.362950+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:03:33.362950+0000","last_peered":"2024-05-20T20:03:33.362950+0000","last_clean":"2024-05-20T20:03:33.362950+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:03:33.362950+0000","last_undegraded":"2024-05-20T20:03:33.362950+0000","last_fullsized":"2024-05-20T20:03:33.362950+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":34,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751099+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:02:04.751099+0000","last_peered":"2024-05-20T20:02:04.751099+0000","last_clean":"2024-05-20T20:02:04.751099+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:02:04.751099+0000","last_undegraded":"2024-05-20T20:02:04.751099+0000","last_fullsized":"2024-05-20T20:02:04.751099+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404966+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:03:33.404966+0000","last_peered":"2024-05-20T20:03:33.404966+0000","last_clean":"2024-05-20T20:03:33.404966+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:03:33.404966+0000","last_undegraded":"2024-05-20T20:03:33.404966+0000","last_fullsized":"2024-05-20T20:03:33.404966+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753891+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:02:04.753891+0000","last_peered":"2024-05-20T20:02:04.753891+0000","last_clean":"2024-05-20T20:02:04.753891+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:02:04.753891+0000","last_undegraded":"2024-05-20T20:02:04.753891+0000","last_fullsized":"2024-05-20T20:02:04.753891+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406608+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:03:33.406608+0000","last_peered":"2024-05-20T20:03:33.406608+0000","last_clean":"2024-05-20T20:03:33.406608+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:03:33.406608+0000","last_undegraded":"2024-05-20T20:03:33.406608+0000","last_fullsized":"2024-05-20T20:03:33.406608+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751709+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:02:04.751709+0000","last_peered":"2024-05-20T20:02:04.751709+0000","last_clean":"2024-05-20T20:02:04.751709+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:02:04.751709+0000","last_undegraded":"2024-05-20T20:02:04.751709+0000","last_fullsized":"2024-05-20T20:02:04.751709+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.401240+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:03:33.401240+0000","last_peered":"2024-05-20T20:03:33.401240+0000","last_clean":"2024-05-20T20:03:33.401240+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:03:33.401240+0000","last_undegraded":"2024-05-20T20:03:33.401240+0000","last_fullsized":"2024-05-20T20:03:33.401240+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"78'5","reported_seq":150,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:40.844469+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:03:40.844469+0000","last_peered":"2024-05-20T20:03:40.844469+0000","last_clean":"2024-05-20T20:03:40.844469+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:03:40.844469+0000","last_undegraded":"2024-05-20T20:03:40.844469+0000","last_fullsized":"2024-05-20T20:03:40.844469+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1170,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":118,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.386953+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:03:33.386953+0000","last_peered":"2024-05-20T20:03:33.386953+0000","last_clean":"2024-05-20T20:03:33.386953+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:03:33.386953+0000","last_undegraded":"2024-05-20T20:03:33.386953+0000","last_fullsized":"2024-05-20T20:03:33.386953+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"78'3","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748859+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:02:04.748859+0000","last_peered":"2024-05-20T20:02:04.748859+0000","last_clean":"2024-05-20T20:02:04.748859+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:02:04.748859+0000","last_undegraded":"2024-05-20T20:02:04.748859+0000","last_fullsized":"2024-05-20T20:02:04.748859+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398328+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:03:33.398328+0000","last_peered":"2024-05-20T20:03:33.398328+0000","last_clean":"2024-05-20T20:03:33.398328+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:03:33.398328+0000","last_undegraded":"2024-05-20T20:03:33.398328+0000","last_fullsized":"2024-05-20T20:03:33.398328+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154133+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:02:05.154133+0000","last_peered":"2024-05-20T20:02:05.154133+0000","last_clean":"2024-05-20T20:02:05.154133+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:02:05.154133+0000","last_undegraded":"2024-05-20T20:02:05.154133+0000","last_fullsized":"2024-05-20T20:02:05.154133+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.386794+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:03:33.386794+0000","last_peered":"2024-05-20T20:03:33.386794+0000","last_clean":"2024-05-20T20:03:33.386794+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:03:33.386794+0000","last_undegraded":"2024-05-20T20:03:33.386794+0000","last_fullsized":"2024-05-20T20:03:33.386794+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.369190+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:03:33.369190+0000","last_peered":"2024-05-20T20:03:33.369190+0000","last_clean":"2024-05-20T20:03:33.369190+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:03:33.369190+0000","last_undegraded":"2024-05-20T20:03:33.369190+0000","last_fullsized":"2024-05-20T20:03:33.369190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.364428+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:03:33.364428+0000","last_peered":"2024-05-20T20:03:33.364428+0000","last_clean":"2024-05-20T20:03:33.364428+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:03:33.364428+0000","last_undegraded":"2024-05-20T20:03:33.364428+0000","last_fullsized":"2024-05-20T20:03:33.364428+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":35,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.382068+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:03:33.382068+0000","last_peered":"2024-05-20T20:03:33.382068+0000","last_clean":"2024-05-20T20:03:33.382068+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:03:33.382068+0000","last_undegraded":"2024-05-20T20:03:33.382068+0000","last_fullsized":"2024-05-20T20:03:33.382068+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.403923+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:03:33.403923+0000","last_peered":"2024-05-20T20:03:33.403923+0000","last_clean":"2024-05-20T20:03:33.403923+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:03:33.403923+0000","last_undegraded":"2024-05-20T20:03:33.403923+0000","last_fullsized":"2024-05-20T20:03:33.403923+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404968+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:03:33.404968+0000","last_peered":"2024-05-20T20:03:33.404968+0000","last_clean":"2024-05-20T20:03:33.404968+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:03:33.404968+0000","last_undegraded":"2024-05-20T20:03:33.404968+0000","last_fullsized":"2024-05-20T20:03:33.404968+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.368057+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:03:33.368057+0000","last_peered":"2024-05-20T20:03:33.368057+0000","last_clean":"2024-05-20T20:03:33.368057+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:03:33.368057+0000","last_undegraded":"2024-05-20T20:03:33.368057+0000","last_fullsized":"2024-05-20T20:03:33.368057+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":36,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.376129+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:03:33.376129+0000","last_peered":"2024-05-20T20:03:33.376129+0000","last_clean":"2024-05-20T20:03:33.376129+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:03:33.376129+0000","last_undegraded":"2024-05-20T20:03:33.376129+0000","last_fullsized":"2024-05-20T20:03:33.376129+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":23,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.373253+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:03:33.373253+0000","last_peered":"2024-05-20T20:03:33.373253+0000","last_clean":"2024-05-20T20:03:33.373253+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:03:33.373253+0000","last_undegraded":"2024-05-20T20:03:33.373253+0000","last_fullsized":"2024-05-20T20:03:33.373253+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.389099+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:03:33.389099+0000","last_peered":"2024-05-20T20:03:33.389099+0000","last_clean":"2024-05-20T20:03:33.389099+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:03:33.389099+0000","last_undegraded":"2024-05-20T20:03:33.389099+0000","last_fullsized":"2024-05-20T20:03:33.389099+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.402884+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:03:33.402884+0000","last_peered":"2024-05-20T20:03:33.402884+0000","last_clean":"2024-05-20T20:03:33.402884+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:03:33.402884+0000","last_undegraded":"2024-05-20T20:03:33.402884+0000","last_fullsized":"2024-05-20T20:03:33.402884+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404187+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:03:33.404187+0000","last_peered":"2024-05-20T20:03:33.404187+0000","last_clean":"2024-05-20T20:03:33.404187+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:03:33.404187+0000","last_undegraded":"2024-05-20T20:03:33.404187+0000","last_fullsized":"2024-05-20T20:03:33.404187+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406894+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:03:33.406894+0000","last_peered":"2024-05-20T20:03:33.406894+0000","last_clean":"2024-05-20T20:03:33.406894+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:03:33.406894+0000","last_undegraded":"2024-05-20T20:03:33.406894+0000","last_fullsized":"2024-05-20T20:03:33.406894+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.374538+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:03:33.374538+0000","last_peered":"2024-05-20T20:03:33.374538+0000","last_clean":"2024-05-20T20:03:33.374538+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:03:33.374538+0000","last_undegraded":"2024-05-20T20:03:33.374538+0000","last_fullsized":"2024-05-20T20:03:33.374538+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":4132576,"data_compressed":248011,"data_compressed_allocated":315392,"data_compressed_original":4079616,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":69538,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":1206,"num_read_kb":680,"num_write":251,"num_write_kb":141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":577536,"data_stored":208614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":215,"ondisk_log_size":215,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710766,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":425895,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877906995,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28024,"kb_used_data":1008,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717128,"statfs":{"total":95995035648,"available":95966339072,"internally_reserved":0,"allocated":1032192,"data_stored":1079440,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070521,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32052,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713100,"statfs":{"total":95995035648,"available":95962214400,"internally_reserved":0,"allocated":897024,"data_stored":425757,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266751,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27888,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717264,"statfs":{"total":95995035648,"available":95966478336,"internally_reserved":0,"allocated":892928,"data_stored":426135,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863462981,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32100,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713052,"statfs":{"total":95995035648,"available":95962165248,"internally_reserved":0,"allocated":946176,"data_stored":1018794,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626506,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":425474,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574951,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":488971,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738477,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":426813,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411425,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":928,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":950272,"data_stored":489543,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182492,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":897024,"data_stored":425299,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986262,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27884,"kb_used_data":868,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717268,"statfs":{"total":95995035648,"available":95966482432,"internally_reserved":0,"allocated":888832,"data_stored":425447,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822736,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32112,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713040,"statfs":{"total":95995035648,"available":95962152960,"internally_reserved":0,"allocated":958464,"data_stored":1018472,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":86016,"data_stored":64336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":64908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":3352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":3674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":64320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:03:47.318 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T20:03:47.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:46 smithi154 ceph-mon[36314]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:03:47.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:46 smithi033 ceph-mon[36179]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:03:47.828 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:48.166 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:47 smithi028 ceph-mon[31178]: pgmap v392: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:48.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:47 smithi154 ceph-mon[36314]: pgmap v392: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:48.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:47 smithi033 ceph-mon[36179]: pgmap v392: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 426 B/s wr, 9 op/s 2024-05-20T20:03:50.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 -- 172.21.15.28:0/3783247937 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858075770 msgr2=0x7fb858075b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:50.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 --2- 172.21.15.28:0/3783247937 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858075770 0x7fb858075b50 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fb84c009990 tx=0x7fb84c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 -- 172.21.15.28:0/3783247937 shutdown_connections 2024-05-20T20:03:50.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 --2- 172.21.15.28:0/3783247937 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 0x7fb858112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 --2- 172.21.15.28:0/3783247937 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858075770 0x7fb858075b50 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 --2- 172.21.15.28:0/3783247937 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb858076090 0x7fb85810fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 -- 172.21.15.28:0/3783247937 >> 172.21.15.28:0/3783247937 conn(0x7fb85807a6a0 msgr2=0x7fb85807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 -- 172.21.15.28:0/3783247937 shutdown_connections 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.087+0000 7fb85fe00700 1 -- 172.21.15.28:0/3783247937 wait complete. 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.088+0000 7fb85fe00700 1 Processor -- start 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85fe00700 1 -- start start 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb858075770 0x7fb85819bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858076090 0x7fb85819c320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 0x7fb8581a0660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85fe00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb858115910 con 0x7fb85810ff40 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85fe00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb858115790 con 0x7fb858075770 2024-05-20T20:03:50.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85fe00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb858115a90 con 0x7fb858076090 2024-05-20T20:03:50.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 0x7fb8581a0660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:50.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 0x7fb8581a0660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51754/0 (socket says 172.21.15.28:51754) 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85de00700 1 -- 172.21.15.28:0/1956570373 learned_addr learned my addr 172.21.15.28:0/1956570373 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb858075770 0x7fb85819bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.089+0000 7fb85ca00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858076090 0x7fb85819c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb85d400700 1 -- 172.21.15.28:0/1956570373 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858076090 msgr2=0x7fb85819c320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb85d400700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858076090 0x7fb85819c320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb85d400700 1 -- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 msgr2=0x7fb8581a0660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb85d400700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 0x7fb8581a0660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb85d400700 1 -- 172.21.15.28:0/1956570373 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb8581a0da0 con 0x7fb858075770 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb85de00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 0x7fb8581a0660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb85d400700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb858075770 0x7fb85819bde0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fb84c006010 tx=0x7fb84c0057c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb856000700 1 -- 172.21.15.28:0/1956570373 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb84c027070 con 0x7fb858075770 2024-05-20T20:03:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.090+0000 7fb856000700 1 -- 172.21.15.28:0/1956570373 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb84c022470 con 0x7fb858075770 2024-05-20T20:03:50.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.091+0000 7fb856000700 1 -- 172.21.15.28:0/1956570373 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb84c017650 con 0x7fb858075770 2024-05-20T20:03:50.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.091+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb8581a1030 con 0x7fb858075770 2024-05-20T20:03:50.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.091+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb8581a8910 con 0x7fb858075770 2024-05-20T20:03:50.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.092+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb85806a230 con 0x7fb858075770 2024-05-20T20:03:50.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.094+0000 7fb856000700 1 -- 172.21.15.28:0/1956570373 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fb84c009d10 con 0x7fb858075770 2024-05-20T20:03:50.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.096+0000 7fb856000700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb8400b2880 0x7fb8400b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:50.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.096+0000 7fb856000700 1 -- 172.21.15.28:0/1956570373 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fb84c0e2b10 con 0x7fb858075770 2024-05-20T20:03:50.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.096+0000 7fb85ca00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb8400b2880 0x7fb8400b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:50.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.097+0000 7fb85ca00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb8400b2880 0x7fb8400b4d40 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fb858103130 tx=0x7fb844009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:50.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.101+0000 7fb856000700 1 -- 172.21.15.28:0/1956570373 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb84c013070 con 0x7fb858075770 2024-05-20T20:03:50.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.335+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fb85819d0f0 con 0x7fb8400b2880 2024-05-20T20:03:50.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.356+0000 7fb856000700 1 -- 172.21.15.28:0/1956570373 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+259110 (secure 0 0 0) 0x7fb85819d0f0 con 0x7fb8400b2880 2024-05-20T20:03:50.363 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb8400b2880 msgr2=0x7fb8400b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb8400b2880 0x7fb8400b4d40 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7fb858103130 tx=0x7fb844009260 comp rx=0 tx=0).stop 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb858075770 msgr2=0x7fb85819bde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb858075770 0x7fb85819bde0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fb84c006010 tx=0x7fb84c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 shutdown_connections 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb8400b2880 0x7fb8400b4d40 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85810ff40 0x7fb8581a0660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb858075770 0x7fb85819bde0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 --2- 172.21.15.28:0/1956570373 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb858076090 0x7fb85819c320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 >> 172.21.15.28:0/1956570373 conn(0x7fb85807a6a0 msgr2=0x7fb8581007f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 shutdown_connections 2024-05-20T20:03:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:50.363+0000 7fb85fe00700 1 -- 172.21.15.28:0/1956570373 wait complete. 2024-05-20T20:03:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:03:50.676 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:50 smithi028 ceph-mon[31178]: pgmap v393: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 341 B/s wr, 12 op/s 2024-05-20T20:03:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:50 smithi154 ceph-mon[36314]: pgmap v393: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 341 B/s wr, 12 op/s 2024-05-20T20:03:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:50 smithi033 ceph-mon[36179]: pgmap v393: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 341 B/s wr, 12 op/s 2024-05-20T20:03:51.133 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":393,"stamp":"2024-05-20T20:03:49.430677+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":660140,"num_objects":102,"num_object_clones":0,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":0,"num_read":1332,"num_read_kb":769,"num_write":376,"num_write_kb":2293,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":809,"ondisk_log_size":809,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347484,"kb_used_data":10812,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124594340,"statfs":{"total":1151940427776,"available":1151584604160,"internally_reserved":0,"allocated":11071488,"data_stored":7076040,"data_compressed":177837,"data_compressed_allocated":233472,"data_compressed_original":1966080,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":249,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":141,"num_read_kb":71,"num_write":4,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005111"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406944+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:03:33.406944+0000","last_peered":"2024-05-20T20:03:33.406944+0000","last_clean":"2024-05-20T20:03:33.406944+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:03:33.406944+0000","last_undegraded":"2024-05-20T20:03:33.406944+0000","last_fullsized":"2024-05-20T20:03:33.406944+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.395385+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:03:33.395385+0000","last_peered":"2024-05-20T20:03:33.395385+0000","last_clean":"2024-05-20T20:03:33.395385+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:03:33.395385+0000","last_undegraded":"2024-05-20T20:03:33.395385+0000","last_fullsized":"2024-05-20T20:03:33.395385+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.401648+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:03:33.401648+0000","last_peered":"2024-05-20T20:03:33.401648+0000","last_clean":"2024-05-20T20:03:33.401648+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:03:33.401648+0000","last_undegraded":"2024-05-20T20:03:33.401648+0000","last_fullsized":"2024-05-20T20:03:33.401648+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.370508+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:03:33.370508+0000","last_peered":"2024-05-20T20:03:33.370508+0000","last_clean":"2024-05-20T20:03:33.370508+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:03:33.370508+0000","last_undegraded":"2024-05-20T20:03:33.370508+0000","last_fullsized":"2024-05-20T20:03:33.370508+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":24,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.390381+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:03:33.390381+0000","last_peered":"2024-05-20T20:03:33.390381+0000","last_clean":"2024-05-20T20:03:33.390381+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:03:33.390381+0000","last_undegraded":"2024-05-20T20:03:33.390381+0000","last_fullsized":"2024-05-20T20:03:33.390381+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.366770+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:03:33.366770+0000","last_peered":"2024-05-20T20:03:33.366770+0000","last_clean":"2024-05-20T20:03:33.366770+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:03:33.366770+0000","last_undegraded":"2024-05-20T20:03:33.366770+0000","last_fullsized":"2024-05-20T20:03:33.366770+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.379472+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:03:33.379472+0000","last_peered":"2024-05-20T20:03:33.379472+0000","last_clean":"2024-05-20T20:03:33.379472+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:03:33.379472+0000","last_undegraded":"2024-05-20T20:03:33.379472+0000","last_fullsized":"2024-05-20T20:03:33.379472+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.397632+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:03:33.397632+0000","last_peered":"2024-05-20T20:03:33.397632+0000","last_clean":"2024-05-20T20:03:33.397632+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:03:33.397632+0000","last_undegraded":"2024-05-20T20:03:33.397632+0000","last_fullsized":"2024-05-20T20:03:33.397632+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.378256+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:03:33.378256+0000","last_peered":"2024-05-20T20:03:33.378256+0000","last_clean":"2024-05-20T20:03:33.378256+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:03:33.378256+0000","last_undegraded":"2024-05-20T20:03:33.378256+0000","last_fullsized":"2024-05-20T20:03:33.378256+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":42,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.371132+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:03:33.371132+0000","last_peered":"2024-05-20T20:03:33.371132+0000","last_clean":"2024-05-20T20:03:33.371132+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:03:33.371132+0000","last_undegraded":"2024-05-20T20:03:33.371132+0000","last_fullsized":"2024-05-20T20:03:33.371132+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.386779+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:03:33.386779+0000","last_peered":"2024-05-20T20:03:33.386779+0000","last_clean":"2024-05-20T20:03:33.386779+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:03:33.386779+0000","last_undegraded":"2024-05-20T20:03:33.386779+0000","last_fullsized":"2024-05-20T20:03:33.386779+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.362410+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:03:33.362410+0000","last_peered":"2024-05-20T20:03:33.362410+0000","last_clean":"2024-05-20T20:03:33.362410+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:03:33.362410+0000","last_undegraded":"2024-05-20T20:03:33.362410+0000","last_fullsized":"2024-05-20T20:03:33.362410+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.403368+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:03:33.403368+0000","last_peered":"2024-05-20T20:03:33.403368+0000","last_clean":"2024-05-20T20:03:33.403368+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:03:33.403368+0000","last_undegraded":"2024-05-20T20:03:33.403368+0000","last_fullsized":"2024-05-20T20:03:33.403368+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.397551+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:03:33.397551+0000","last_peered":"2024-05-20T20:03:33.397551+0000","last_clean":"2024-05-20T20:03:33.397551+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:03:33.397551+0000","last_undegraded":"2024-05-20T20:03:33.397551+0000","last_fullsized":"2024-05-20T20:03:33.397551+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.396857+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:03:33.396857+0000","last_peered":"2024-05-20T20:03:33.396857+0000","last_clean":"2024-05-20T20:03:33.396857+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:03:33.396857+0000","last_undegraded":"2024-05-20T20:03:33.396857+0000","last_fullsized":"2024-05-20T20:03:33.396857+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.405677+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:03:33.405677+0000","last_peered":"2024-05-20T20:03:33.405677+0000","last_clean":"2024-05-20T20:03:33.405677+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:03:33.405677+0000","last_undegraded":"2024-05-20T20:03:33.405677+0000","last_fullsized":"2024-05-20T20:03:33.405677+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.407057+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:03:33.407057+0000","last_peered":"2024-05-20T20:03:33.407057+0000","last_clean":"2024-05-20T20:03:33.407057+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:03:33.407057+0000","last_undegraded":"2024-05-20T20:03:33.407057+0000","last_fullsized":"2024-05-20T20:03:33.407057+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.379709+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:03:33.379709+0000","last_peered":"2024-05-20T20:03:33.379709+0000","last_clean":"2024-05-20T20:03:33.379709+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:03:33.379709+0000","last_undegraded":"2024-05-20T20:03:33.379709+0000","last_fullsized":"2024-05-20T20:03:33.379709+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.390834+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:03:33.390834+0000","last_peered":"2024-05-20T20:03:33.390834+0000","last_clean":"2024-05-20T20:03:33.390834+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:03:33.390834+0000","last_undegraded":"2024-05-20T20:03:33.390834+0000","last_fullsized":"2024-05-20T20:03:33.390834+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"78'13","reported_seq":159,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:42.943725+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:03:42.943725+0000","last_peered":"2024-05-20T20:03:42.943725+0000","last_clean":"2024-05-20T20:03:42.943725+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:03:42.943725+0000","last_undegraded":"2024-05-20T20:03:42.943725+0000","last_fullsized":"2024-05-20T20:03:42.943725+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":119,"num_read_kb":119,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.395728+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:03:33.395728+0000","last_peered":"2024-05-20T20:03:33.395728+0000","last_clean":"2024-05-20T20:03:33.395728+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:03:33.395728+0000","last_undegraded":"2024-05-20T20:03:33.395728+0000","last_fullsized":"2024-05-20T20:03:33.395728+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753264+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:02:04.753264+0000","last_peered":"2024-05-20T20:02:04.753264+0000","last_clean":"2024-05-20T20:02:04.753264+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:02:04.753264+0000","last_undegraded":"2024-05-20T20:02:04.753264+0000","last_fullsized":"2024-05-20T20:02:04.753264+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.407890+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:03:33.407890+0000","last_peered":"2024-05-20T20:03:33.407890+0000","last_clean":"2024-05-20T20:03:33.407890+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:03:33.407890+0000","last_undegraded":"2024-05-20T20:03:33.407890+0000","last_fullsized":"2024-05-20T20:03:33.407890+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"78'12","reported_seq":40,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752241+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:02:04.752241+0000","last_peered":"2024-05-20T20:02:04.752241+0000","last_clean":"2024-05-20T20:02:04.752241+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:02:04.752241+0000","last_undegraded":"2024-05-20T20:02:04.752241+0000","last_fullsized":"2024-05-20T20:02:04.752241+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.399806+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:03:33.399806+0000","last_peered":"2024-05-20T20:03:33.399806+0000","last_clean":"2024-05-20T20:03:33.399806+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:03:33.399806+0000","last_undegraded":"2024-05-20T20:03:33.399806+0000","last_fullsized":"2024-05-20T20:03:33.399806+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751853+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:02:04.751853+0000","last_peered":"2024-05-20T20:02:04.751853+0000","last_clean":"2024-05-20T20:02:04.751853+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:02:04.751853+0000","last_undegraded":"2024-05-20T20:02:04.751853+0000","last_fullsized":"2024-05-20T20:02:04.751853+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.383831+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:03:33.383831+0000","last_peered":"2024-05-20T20:03:33.383831+0000","last_clean":"2024-05-20T20:03:33.383831+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:03:33.383831+0000","last_undegraded":"2024-05-20T20:03:33.383831+0000","last_fullsized":"2024-05-20T20:03:33.383831+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"80'7","reported_seq":158,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:45.852329+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:03:45.852329+0000","last_peered":"2024-05-20T20:03:45.852329+0000","last_clean":"2024-05-20T20:03:45.852329+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:03:45.852329+0000","last_undegraded":"2024-05-20T20:03:45.852329+0000","last_fullsized":"2024-05-20T20:03:45.852329+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1154,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":125,"num_read_kb":4,"num_write":4,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.385883+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:03:33.385883+0000","last_peered":"2024-05-20T20:03:33.385883+0000","last_clean":"2024-05-20T20:03:33.385883+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:03:33.385883+0000","last_undegraded":"2024-05-20T20:03:33.385883+0000","last_fullsized":"2024-05-20T20:03:33.385883+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748817+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:02:04.748817+0000","last_peered":"2024-05-20T20:02:04.748817+0000","last_clean":"2024-05-20T20:02:04.748817+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:02:04.748817+0000","last_undegraded":"2024-05-20T20:02:04.748817+0000","last_fullsized":"2024-05-20T20:02:04.748817+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.381104+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:03:33.381104+0000","last_peered":"2024-05-20T20:03:33.381104+0000","last_clean":"2024-05-20T20:03:33.381104+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:03:33.381104+0000","last_undegraded":"2024-05-20T20:03:33.381104+0000","last_fullsized":"2024-05-20T20:03:33.381104+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153679+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:02:05.153679+0000","last_peered":"2024-05-20T20:02:05.153679+0000","last_clean":"2024-05-20T20:02:05.153679+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:02:05.153679+0000","last_undegraded":"2024-05-20T20:02:05.153679+0000","last_fullsized":"2024-05-20T20:02:05.153679+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.405402+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:03:33.405402+0000","last_peered":"2024-05-20T20:03:33.405402+0000","last_clean":"2024-05-20T20:03:33.405402+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:03:33.405402+0000","last_undegraded":"2024-05-20T20:03:33.405402+0000","last_fullsized":"2024-05-20T20:03:33.405402+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748414+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:02:04.748414+0000","last_peered":"2024-05-20T20:02:04.748414+0000","last_clean":"2024-05-20T20:02:04.748414+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:02:04.748414+0000","last_undegraded":"2024-05-20T20:02:04.748414+0000","last_fullsized":"2024-05-20T20:02:04.748414+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.399298+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:03:33.399298+0000","last_peered":"2024-05-20T20:03:33.399298+0000","last_clean":"2024-05-20T20:03:33.399298+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:03:33.399298+0000","last_undegraded":"2024-05-20T20:03:33.399298+0000","last_fullsized":"2024-05-20T20:03:33.399298+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"80'5","reported_seq":274,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:46.853111+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:03:46.853111+0000","last_peered":"2024-05-20T20:03:46.853111+0000","last_clean":"2024-05-20T20:03:46.853111+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:03:46.853111+0000","last_undegraded":"2024-05-20T20:03:46.853111+0000","last_fullsized":"2024-05-20T20:03:46.853111+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":242,"num_read_kb":242,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.383327+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:03:33.383327+0000","last_peered":"2024-05-20T20:03:33.383327+0000","last_clean":"2024-05-20T20:03:33.383327+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:03:33.383327+0000","last_undegraded":"2024-05-20T20:03:33.383327+0000","last_fullsized":"2024-05-20T20:03:33.383327+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930581+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:02:04.930581+0000","last_peered":"2024-05-20T20:02:04.930581+0000","last_clean":"2024-05-20T20:02:04.930581+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:02:04.930581+0000","last_undegraded":"2024-05-20T20:02:04.930581+0000","last_fullsized":"2024-05-20T20:02:04.930581+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.408113+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:03:33.408113+0000","last_peered":"2024-05-20T20:03:33.408113+0000","last_clean":"2024-05-20T20:03:33.408113+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:03:33.408113+0000","last_undegraded":"2024-05-20T20:03:33.408113+0000","last_fullsized":"2024-05-20T20:03:33.408113+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.749085+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:02:04.749085+0000","last_peered":"2024-05-20T20:02:04.749085+0000","last_clean":"2024-05-20T20:02:04.749085+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:02:04.749085+0000","last_undegraded":"2024-05-20T20:02:04.749085+0000","last_fullsized":"2024-05-20T20:02:04.749085+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.384161+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:03:33.384161+0000","last_peered":"2024-05-20T20:03:33.384161+0000","last_clean":"2024-05-20T20:03:33.384161+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:03:33.384161+0000","last_undegraded":"2024-05-20T20:03:33.384161+0000","last_fullsized":"2024-05-20T20:03:33.384161+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"80'34","reported_seq":313,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:46.852914+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:03:46.852914+0000","last_peered":"2024-05-20T20:03:46.852914+0000","last_clean":"2024-05-20T20:03:46.852914+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:03:46.852914+0000","last_undegraded":"2024-05-20T20:03:46.852914+0000","last_fullsized":"2024-05-20T20:03:46.852914+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62584,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":252,"num_read_kb":187,"num_write":36,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.385845+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:03:33.385845+0000","last_peered":"2024-05-20T20:03:33.385845+0000","last_clean":"2024-05-20T20:03:33.385845+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:03:33.385845+0000","last_undegraded":"2024-05-20T20:03:33.385845+0000","last_fullsized":"2024-05-20T20:03:33.385845+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.773205+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:02:04.773205+0000","last_peered":"2024-05-20T20:02:04.773205+0000","last_clean":"2024-05-20T20:02:04.773205+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:02:04.773205+0000","last_undegraded":"2024-05-20T20:02:04.773205+0000","last_fullsized":"2024-05-20T20:02:04.773205+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404483+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:03:33.404483+0000","last_peered":"2024-05-20T20:03:33.404483+0000","last_clean":"2024-05-20T20:03:33.404483+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:03:33.404483+0000","last_undegraded":"2024-05-20T20:03:33.404483+0000","last_fullsized":"2024-05-20T20:03:33.404483+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748855+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:02:04.748855+0000","last_peered":"2024-05-20T20:02:04.748855+0000","last_clean":"2024-05-20T20:02:04.748855+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:02:04.748855+0000","last_undegraded":"2024-05-20T20:02:04.748855+0000","last_fullsized":"2024-05-20T20:02:04.748855+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":36,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.400099+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:03:33.400099+0000","last_peered":"2024-05-20T20:03:33.400099+0000","last_clean":"2024-05-20T20:03:33.400099+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:03:33.400099+0000","last_undegraded":"2024-05-20T20:03:33.400099+0000","last_fullsized":"2024-05-20T20:03:33.400099+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"78'5","reported_seq":155,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:45.952286+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:03:45.952286+0000","last_peered":"2024-05-20T20:03:45.952286+0000","last_clean":"2024-05-20T20:03:45.952286+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:03:45.952286+0000","last_undegraded":"2024-05-20T20:03:45.952286+0000","last_fullsized":"2024-05-20T20:03:45.952286+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":124,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.374449+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:03:33.374449+0000","last_peered":"2024-05-20T20:03:33.374449+0000","last_clean":"2024-05-20T20:03:33.374449+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:03:33.374449+0000","last_undegraded":"2024-05-20T20:03:33.374449+0000","last_fullsized":"2024-05-20T20:03:33.374449+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":69,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.169710+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:02:05.169710+0000","last_peered":"2024-05-20T20:02:05.169710+0000","last_clean":"2024-05-20T20:02:05.169710+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:02:05.169710+0000","last_undegraded":"2024-05-20T20:02:05.169710+0000","last_fullsized":"2024-05-20T20:02:05.169710+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.378507+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:03:33.378507+0000","last_peered":"2024-05-20T20:03:33.378507+0000","last_clean":"2024-05-20T20:03:33.378507+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:03:33.378507+0000","last_undegraded":"2024-05-20T20:03:33.378507+0000","last_fullsized":"2024-05-20T20:03:33.378507+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"78'2","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752194+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:02:04.752194+0000","last_peered":"2024-05-20T20:02:04.752194+0000","last_clean":"2024-05-20T20:02:04.752194+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:02:04.752194+0000","last_undegraded":"2024-05-20T20:02:04.752194+0000","last_fullsized":"2024-05-20T20:02:04.752194+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.375428+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:03:33.375428+0000","last_peered":"2024-05-20T20:03:33.375428+0000","last_clean":"2024-05-20T20:03:33.375428+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:03:33.375428+0000","last_undegraded":"2024-05-20T20:03:33.375428+0000","last_fullsized":"2024-05-20T20:03:33.375428+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.757207+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:02:04.757207+0000","last_peered":"2024-05-20T20:02:04.757207+0000","last_clean":"2024-05-20T20:02:04.757207+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:02:04.757207+0000","last_undegraded":"2024-05-20T20:02:04.757207+0000","last_fullsized":"2024-05-20T20:02:04.757207+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398629+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:03:33.398629+0000","last_peered":"2024-05-20T20:03:33.398629+0000","last_clean":"2024-05-20T20:03:33.398629+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:03:33.398629+0000","last_undegraded":"2024-05-20T20:03:33.398629+0000","last_fullsized":"2024-05-20T20:03:33.398629+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"78'5","reported_seq":156,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:47.958084+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:03:47.958084+0000","last_peered":"2024-05-20T20:03:47.958084+0000","last_clean":"2024-05-20T20:03:47.958084+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:03:47.958084+0000","last_undegraded":"2024-05-20T20:03:47.958084+0000","last_fullsized":"2024-05-20T20:03:47.958084+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":588,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":124,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.395302+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:03:33.395302+0000","last_peered":"2024-05-20T20:03:33.395302+0000","last_clean":"2024-05-20T20:03:33.395302+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:03:33.395302+0000","last_undegraded":"2024-05-20T20:03:33.395302+0000","last_fullsized":"2024-05-20T20:03:33.395302+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.372182+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:03:33.372182+0000","last_peered":"2024-05-20T20:03:33.372182+0000","last_clean":"2024-05-20T20:03:33.372182+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:03:33.372182+0000","last_undegraded":"2024-05-20T20:03:33.372182+0000","last_fullsized":"2024-05-20T20:03:33.372182+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"78'5","reported_seq":154,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:45.951061+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:03:45.951061+0000","last_peered":"2024-05-20T20:03:45.951061+0000","last_clean":"2024-05-20T20:03:45.951061+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:03:45.951061+0000","last_undegraded":"2024-05-20T20:03:45.951061+0000","last_fullsized":"2024-05-20T20:03:45.951061+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":122,"num_read_kb":122,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.402541+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:03:33.402541+0000","last_peered":"2024-05-20T20:03:33.402541+0000","last_clean":"2024-05-20T20:03:33.402541+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:03:33.402541+0000","last_undegraded":"2024-05-20T20:03:33.402541+0000","last_fullsized":"2024-05-20T20:03:33.402541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":36,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154652+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:02:05.154652+0000","last_peered":"2024-05-20T20:02:05.154652+0000","last_clean":"2024-05-20T20:02:05.154652+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:02:05.154652+0000","last_undegraded":"2024-05-20T20:02:05.154652+0000","last_fullsized":"2024-05-20T20:02:05.154652+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"20'92","reported_seq":100,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930195+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:02:04.930195+0000","last_peered":"2024-05-20T20:02:04.930195+0000","last_clean":"2024-05-20T20:02:04.930195+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:02:04.930195+0000","last_undegraded":"2024-05-20T20:02:04.930195+0000","last_fullsized":"2024-05-20T20:02:04.930195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398178+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:03:33.398178+0000","last_peered":"2024-05-20T20:03:33.398178+0000","last_clean":"2024-05-20T20:03:33.398178+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:03:33.398178+0000","last_undegraded":"2024-05-20T20:03:33.398178+0000","last_fullsized":"2024-05-20T20:03:33.398178+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.930350+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:02:04.930350+0000","last_peered":"2024-05-20T20:02:04.930350+0000","last_clean":"2024-05-20T20:02:04.930350+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:02:04.930350+0000","last_undegraded":"2024-05-20T20:02:04.930350+0000","last_fullsized":"2024-05-20T20:02:04.930350+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.752065+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:02:04.752065+0000","last_peered":"2024-05-20T20:02:04.752065+0000","last_clean":"2024-05-20T20:02:04.752065+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:02:04.752065+0000","last_undegraded":"2024-05-20T20:02:04.752065+0000","last_fullsized":"2024-05-20T20:02:04.752065+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398150+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:03:33.398150+0000","last_peered":"2024-05-20T20:03:33.398150+0000","last_clean":"2024-05-20T20:03:33.398150+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:03:33.398150+0000","last_undegraded":"2024-05-20T20:03:33.398150+0000","last_fullsized":"2024-05-20T20:03:33.398150+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"80'16","reported_seq":59,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:29.755099+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:03:29.755099+0000","last_peered":"2024-05-20T20:03:29.755099+0000","last_clean":"2024-05-20T20:03:29.755099+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:03:29.755099+0000","last_undegraded":"2024-05-20T20:03:29.755099+0000","last_fullsized":"2024-05-20T20:03:29.755099+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":29,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.390053+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:03:33.390053+0000","last_peered":"2024-05-20T20:03:33.390053+0000","last_clean":"2024-05-20T20:03:33.390053+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:03:33.390053+0000","last_undegraded":"2024-05-20T20:03:33.390053+0000","last_fullsized":"2024-05-20T20:03:33.390053+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":35,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.155373+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:02:05.155373+0000","last_peered":"2024-05-20T20:02:05.155373+0000","last_clean":"2024-05-20T20:02:05.155373+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:02:05.155373+0000","last_undegraded":"2024-05-20T20:02:05.155373+0000","last_fullsized":"2024-05-20T20:02:05.155373+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":34,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404085+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:03:33.404085+0000","last_peered":"2024-05-20T20:03:33.404085+0000","last_clean":"2024-05-20T20:03:33.404085+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:03:33.404085+0000","last_undegraded":"2024-05-20T20:03:33.404085+0000","last_fullsized":"2024-05-20T20:03:33.404085+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.153898+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:02:05.153898+0000","last_peered":"2024-05-20T20:02:05.153898+0000","last_clean":"2024-05-20T20:02:05.153898+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:02:05.153898+0000","last_undegraded":"2024-05-20T20:02:05.153898+0000","last_fullsized":"2024-05-20T20:02:05.153898+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":23,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.362950+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:03:33.362950+0000","last_peered":"2024-05-20T20:03:33.362950+0000","last_clean":"2024-05-20T20:03:33.362950+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:03:33.362950+0000","last_undegraded":"2024-05-20T20:03:33.362950+0000","last_fullsized":"2024-05-20T20:03:33.362950+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":34,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751099+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:02:04.751099+0000","last_peered":"2024-05-20T20:02:04.751099+0000","last_clean":"2024-05-20T20:02:04.751099+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:02:04.751099+0000","last_undegraded":"2024-05-20T20:02:04.751099+0000","last_fullsized":"2024-05-20T20:02:04.751099+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404966+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:03:33.404966+0000","last_peered":"2024-05-20T20:03:33.404966+0000","last_clean":"2024-05-20T20:03:33.404966+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:03:33.404966+0000","last_undegraded":"2024-05-20T20:03:33.404966+0000","last_fullsized":"2024-05-20T20:03:33.404966+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.753891+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:02:04.753891+0000","last_peered":"2024-05-20T20:02:04.753891+0000","last_clean":"2024-05-20T20:02:04.753891+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:02:04.753891+0000","last_undegraded":"2024-05-20T20:02:04.753891+0000","last_fullsized":"2024-05-20T20:02:04.753891+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406608+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:03:33.406608+0000","last_peered":"2024-05-20T20:03:33.406608+0000","last_clean":"2024-05-20T20:03:33.406608+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:03:33.406608+0000","last_undegraded":"2024-05-20T20:03:33.406608+0000","last_fullsized":"2024-05-20T20:03:33.406608+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.751709+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:02:04.751709+0000","last_peered":"2024-05-20T20:02:04.751709+0000","last_clean":"2024-05-20T20:02:04.751709+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:02:04.751709+0000","last_undegraded":"2024-05-20T20:02:04.751709+0000","last_fullsized":"2024-05-20T20:02:04.751709+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":26,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.401240+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:03:33.401240+0000","last_peered":"2024-05-20T20:03:33.401240+0000","last_clean":"2024-05-20T20:03:33.401240+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:03:33.401240+0000","last_undegraded":"2024-05-20T20:03:33.401240+0000","last_fullsized":"2024-05-20T20:03:33.401240+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"78'5","reported_seq":155,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:45.852203+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:03:45.852203+0000","last_peered":"2024-05-20T20:03:45.852203+0000","last_clean":"2024-05-20T20:03:45.852203+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:03:45.852203+0000","last_undegraded":"2024-05-20T20:03:45.852203+0000","last_fullsized":"2024-05-20T20:03:45.852203+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1170,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":123,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.386953+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:03:33.386953+0000","last_peered":"2024-05-20T20:03:33.386953+0000","last_clean":"2024-05-20T20:03:33.386953+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:03:33.386953+0000","last_undegraded":"2024-05-20T20:03:33.386953+0000","last_fullsized":"2024-05-20T20:03:33.386953+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"78'3","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:04.748859+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:02:04.748859+0000","last_peered":"2024-05-20T20:02:04.748859+0000","last_clean":"2024-05-20T20:02:04.748859+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:02:04.748859+0000","last_undegraded":"2024-05-20T20:02:04.748859+0000","last_fullsized":"2024-05-20T20:02:04.748859+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.398328+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:03:33.398328+0000","last_peered":"2024-05-20T20:03:33.398328+0000","last_clean":"2024-05-20T20:03:33.398328+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:03:33.398328+0000","last_undegraded":"2024-05-20T20:03:33.398328+0000","last_fullsized":"2024-05-20T20:03:33.398328+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:02:05.154133+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:02:05.154133+0000","last_peered":"2024-05-20T20:02:05.154133+0000","last_clean":"2024-05-20T20:02:05.154133+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:02:05.154133+0000","last_undegraded":"2024-05-20T20:02:05.154133+0000","last_fullsized":"2024-05-20T20:02:05.154133+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.386794+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:03:33.386794+0000","last_peered":"2024-05-20T20:03:33.386794+0000","last_clean":"2024-05-20T20:03:33.386794+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:03:33.386794+0000","last_undegraded":"2024-05-20T20:03:33.386794+0000","last_fullsized":"2024-05-20T20:03:33.386794+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":30,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.369190+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:03:33.369190+0000","last_peered":"2024-05-20T20:03:33.369190+0000","last_clean":"2024-05-20T20:03:33.369190+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:03:33.369190+0000","last_undegraded":"2024-05-20T20:03:33.369190+0000","last_fullsized":"2024-05-20T20:03:33.369190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":33,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.364428+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:03:33.364428+0000","last_peered":"2024-05-20T20:03:33.364428+0000","last_clean":"2024-05-20T20:03:33.364428+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:03:33.364428+0000","last_undegraded":"2024-05-20T20:03:33.364428+0000","last_fullsized":"2024-05-20T20:03:33.364428+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":35,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.382068+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:03:33.382068+0000","last_peered":"2024-05-20T20:03:33.382068+0000","last_clean":"2024-05-20T20:03:33.382068+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:03:33.382068+0000","last_undegraded":"2024-05-20T20:03:33.382068+0000","last_fullsized":"2024-05-20T20:03:33.382068+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":28,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.403923+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:03:33.403923+0000","last_peered":"2024-05-20T20:03:33.403923+0000","last_clean":"2024-05-20T20:03:33.403923+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:03:33.403923+0000","last_undegraded":"2024-05-20T20:03:33.403923+0000","last_fullsized":"2024-05-20T20:03:33.403923+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":25,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404968+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:03:33.404968+0000","last_peered":"2024-05-20T20:03:33.404968+0000","last_clean":"2024-05-20T20:03:33.404968+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:03:33.404968+0000","last_undegraded":"2024-05-20T20:03:33.404968+0000","last_fullsized":"2024-05-20T20:03:33.404968+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.368057+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:03:33.368057+0000","last_peered":"2024-05-20T20:03:33.368057+0000","last_clean":"2024-05-20T20:03:33.368057+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:03:33.368057+0000","last_undegraded":"2024-05-20T20:03:33.368057+0000","last_fullsized":"2024-05-20T20:03:33.368057+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":36,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.376129+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:03:33.376129+0000","last_peered":"2024-05-20T20:03:33.376129+0000","last_clean":"2024-05-20T20:03:33.376129+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:03:33.376129+0000","last_undegraded":"2024-05-20T20:03:33.376129+0000","last_fullsized":"2024-05-20T20:03:33.376129+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":23,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.373253+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:03:33.373253+0000","last_peered":"2024-05-20T20:03:33.373253+0000","last_clean":"2024-05-20T20:03:33.373253+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:03:33.373253+0000","last_undegraded":"2024-05-20T20:03:33.373253+0000","last_fullsized":"2024-05-20T20:03:33.373253+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":27,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.389099+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:03:33.389099+0000","last_peered":"2024-05-20T20:03:33.389099+0000","last_clean":"2024-05-20T20:03:33.389099+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:03:33.389099+0000","last_undegraded":"2024-05-20T20:03:33.389099+0000","last_fullsized":"2024-05-20T20:03:33.389099+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.402884+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:03:33.402884+0000","last_peered":"2024-05-20T20:03:33.402884+0000","last_clean":"2024-05-20T20:03:33.402884+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:03:33.402884+0000","last_undegraded":"2024-05-20T20:03:33.402884+0000","last_fullsized":"2024-05-20T20:03:33.402884+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":31,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.404187+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:03:33.404187+0000","last_peered":"2024-05-20T20:03:33.404187+0000","last_clean":"2024-05-20T20:03:33.404187+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:03:33.404187+0000","last_undegraded":"2024-05-20T20:03:33.404187+0000","last_fullsized":"2024-05-20T20:03:33.404187+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":32,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.406894+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:03:33.406894+0000","last_peered":"2024-05-20T20:03:33.406894+0000","last_clean":"2024-05-20T20:03:33.406894+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:03:33.406894+0000","last_undegraded":"2024-05-20T20:03:33.406894+0000","last_fullsized":"2024-05-20T20:03:33.406894+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":29,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-05-20T20:03:33.374538+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:03:33.374538+0000","last_peered":"2024-05-20T20:03:33.374538+0000","last_clean":"2024-05-20T20:03:33.374538+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:03:33.374538+0000","last_undegraded":"2024-05-20T20:03:33.374538+0000","last_fullsized":"2024-05-20T20:03:33.374538+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":4132576,"data_compressed":248011,"data_compressed_allocated":315392,"data_compressed_original":4079616,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":69538,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":1251,"num_read_kb":700,"num_write":251,"num_write_kb":141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":577536,"data_stored":208614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":215,"ondisk_log_size":215,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710767,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":425895,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877906996,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28024,"kb_used_data":1008,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717128,"statfs":{"total":95995035648,"available":95966339072,"internally_reserved":0,"allocated":1032192,"data_stored":1079440,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070521,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32052,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713100,"statfs":{"total":95995035648,"available":95962214400,"internally_reserved":0,"allocated":897024,"data_stored":425757,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266752,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27888,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717264,"statfs":{"total":95995035648,"available":95966478336,"internally_reserved":0,"allocated":892928,"data_stored":426135,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863462982,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32100,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713052,"statfs":{"total":95995035648,"available":95962165248,"internally_reserved":0,"allocated":946176,"data_stored":1018794,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626507,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":425474,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574952,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":488971,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738478,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":426813,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411426,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":928,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":950272,"data_stored":489543,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182492,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":876,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":897024,"data_stored":425299,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986263,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27884,"kb_used_data":868,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717268,"statfs":{"total":95995035648,"available":95966482432,"internally_reserved":0,"allocated":888832,"data_stored":425447,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822737,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32112,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713040,"statfs":{"total":95995035648,"available":95962152960,"internally_reserved":0,"allocated":958464,"data_stored":1018472,"data_compressed":41915,"data_compressed_allocated":53248,"data_compressed_original":606208,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34501,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":86016,"data_stored":64336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":64908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":3352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":3674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":64320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:03:51.137 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-20T20:03:51.137 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-20T20:03:51.138 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-20T20:03:51.138 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph health --format=json 2024-05-20T20:03:51.665 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:51.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:51 smithi028 ceph-mon[31178]: from='client.24845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:03:51.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:51 smithi154 ceph-mon[36314]: from='client.24845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:03:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:51 smithi033 ceph-mon[36179]: from='client.24845 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:03:52.762 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:52 smithi028 ceph-mon[31178]: pgmap v394: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:03:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:52 smithi154 ceph-mon[36314]: pgmap v394: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:03:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:52 smithi033 ceph-mon[36179]: pgmap v394: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:03:54.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 -- 172.21.15.28:0/2590185397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 msgr2=0x7f8f0810e7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:54.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/2590185397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 0x7f8f0810e7d0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f8ef40099c0 tx=0x7f8ef4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 -- 172.21.15.28:0/2590185397 shutdown_connections 2024-05-20T20:03:54.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/2590185397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f0810eeb0 0x7f8f08116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/2590185397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f08109dd0 0x7f8f0810a1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/2590185397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 0x7f8f0810e7d0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 -- 172.21.15.28:0/2590185397 >> 172.21.15.28:0/2590185397 conn(0x7f8f08076d10 msgr2=0x7f8f08108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:54.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 -- 172.21.15.28:0/2590185397 shutdown_connections 2024-05-20T20:03:54.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.095+0000 7f8f0ca00700 1 -- 172.21.15.28:0/2590185397 wait complete. 2024-05-20T20:03:54.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.097+0000 7f8f0ca00700 1 Processor -- start 2024-05-20T20:03:54.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.097+0000 7f8f0ca00700 1 -- start start 2024-05-20T20:03:54.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.097+0000 7f8f0ca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f08109dd0 0x7f8f081a02c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:54.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.097+0000 7f8f0ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 0x7f8f081a0800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:54.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f0ca00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f0810eeb0 0x7f8f081a4b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:54.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f0ca00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8f08119d30 con 0x7f8f0810a780 2024-05-20T20:03:54.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f0ca00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8f08119bb0 con 0x7f8f08109dd0 2024-05-20T20:03:54.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f0ca00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8f08119eb0 con 0x7f8f0810eeb0 2024-05-20T20:03:54.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f05400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 0x7f8f081a0800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:54.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f0810eeb0 0x7f8f081a4b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:54.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f0810eeb0 0x7f8f081a4b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:45636/0 (socket says 172.21.15.28:45636) 2024-05-20T20:03:54.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f05e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f08109dd0 0x7f8f081a02c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:54.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 -- 172.21.15.28:0/763151041 learned_addr learned my addr 172.21.15.28:0/763151041 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:54.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 -- 172.21.15.28:0/763151041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f08109dd0 msgr2=0x7f8f081a02c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:54.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f08109dd0 0x7f8f081a02c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 -- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 msgr2=0x7f8f081a0800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 0x7f8f081a0800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f06800700 1 -- 172.21.15.28:0/763151041 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8f081a5280 con 0x7f8f0810eeb0 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.098+0000 7f8f05e00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f08109dd0 0x7f8f081a02c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.099+0000 7f8f06800700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f0810eeb0 0x7f8f081a4b40 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f8f0000ef30 tx=0x7f8f0000c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.099+0000 7f8efea00700 1 -- 172.21.15.28:0/763151041 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f000092e0 con 0x7f8f0810eeb0 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.099+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8f081a5510 con 0x7f8f0810eeb0 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.099+0000 7f8efea00700 1 -- 172.21.15.28:0/763151041 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8f00004bb0 con 0x7f8f0810eeb0 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.099+0000 7f8efea00700 1 -- 172.21.15.28:0/763151041 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f00004020 con 0x7f8f0810eeb0 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.099+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8f081acdf0 con 0x7f8f0810eeb0 2024-05-20T20:03:54.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.100+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8f0806a230 con 0x7f8f0810eeb0 2024-05-20T20:03:54.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.103+0000 7f8efea00700 1 -- 172.21.15.28:0/763151041 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f8f0001b030 con 0x7f8f0810eeb0 2024-05-20T20:03:54.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.104+0000 7f8efea00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8eec0b2880 0x7f8eec0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:54.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.104+0000 7f8efea00700 1 -- 172.21.15.28:0/763151041 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f8f000e2640 con 0x7f8f0810eeb0 2024-05-20T20:03:54.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.105+0000 7f8f05e00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8eec0b2880 0x7f8eec0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:54.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.105+0000 7f8f05e00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8eec0b2880 0x7f8eec0b4d40 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f8ef8011490 tx=0x7f8ef8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:54.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.106+0000 7f8efea00700 1 -- 172.21.15.28:0/763151041 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f8f00022020 con 0x7f8f0810eeb0 2024-05-20T20:03:54.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.396+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f8f0802e560 con 0x7f8f0810eeb0 2024-05-20T20:03:54.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.397+0000 7f8efea00700 1 -- 172.21.15.28:0/763151041 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f8f00010040 con 0x7f8f0810eeb0 2024-05-20T20:03:54.398 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:03:54.399 INFO:teuthology.orchestra.run.smithi028.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-20T20:03:54.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.400+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8eec0b2880 msgr2=0x7f8eec0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:54.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.400+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8eec0b2880 0x7f8eec0b4d40 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f8ef8011490 tx=0x7f8ef8009260 comp rx=0 tx=0).stop 2024-05-20T20:03:54.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.400+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f0810eeb0 msgr2=0x7f8f081a4b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:54.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.400+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f0810eeb0 0x7f8f081a4b40 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f8f0000ef30 tx=0x7f8f0000c3a0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 shutdown_connections 2024-05-20T20:03:54.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f8eec0b2880 0x7f8eec0b4d40 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f0810eeb0 0x7f8f081a4b40 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f08109dd0 0x7f8f081a02c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 --2- 172.21.15.28:0/763151041 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f0810a780 0x7f8f081a0800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:54.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 >> 172.21.15.28:0/763151041 conn(0x7f8f08076d10 msgr2=0x7f8f08077370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:54.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 shutdown_connections 2024-05-20T20:03:54.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:54.401+0000 7f8f0ca00700 1 -- 172.21.15.28:0/763151041 wait complete. 2024-05-20T20:03:54.794 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:54 smithi028 ceph-mon[31178]: pgmap v395: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:54.794 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:54 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/763151041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T20:03:54.840 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:54 smithi154 ceph-mon[36314]: pgmap v395: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:54.840 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:54 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/763151041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T20:03:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:54 smithi033 ceph-mon[36179]: pgmap v395: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:03:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:54 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/763151041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T20:03:55.240 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-20T20:03:55.241 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:03:55.697 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:03:56.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:56 smithi028 ceph-mon[31178]: pgmap v396: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:56 smithi154 ceph-mon[36314]: pgmap v396: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:56 smithi033 ceph-mon[36179]: pgmap v396: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 -- 172.21.15.28:0/1325912132 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 msgr2=0x7f7d2c075d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 --2- 172.21.15.28:0/1325912132 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c075d50 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c009990 tx=0x7f7d1c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 -- 172.21.15.28:0/1325912132 shutdown_connections 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 --2- 172.21.15.28:0/1325912132 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d2c10fcb0 0x7f7d2c112120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 --2- 172.21.15.28:0/1325912132 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c075d50 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 --2- 172.21.15.28:0/1325912132 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d2c076290 0x7f7d2c10f770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 -- 172.21.15.28:0/1325912132 >> 172.21.15.28:0/1325912132 conn(0x7f7d2c0fefc0 msgr2=0x7f7d2c1013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 -- 172.21.15.28:0/1325912132 shutdown_connections 2024-05-20T20:03:57.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.910+0000 7f7d2be00700 1 -- 172.21.15.28:0/1325912132 wait complete. 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.911+0000 7f7d2be00700 1 Processor -- start 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.911+0000 7f7d2be00700 1 -- start start 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c19bd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d2c076290 0x7f7d2c19c290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d2c10fcb0 0x7f7d2c1a0620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7d2c1158a0 con 0x7f7d2c075970 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2be00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7d2c115720 con 0x7f7d2c076290 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2be00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7d2c115a20 con 0x7f7d2c10fcb0 2024-05-20T20:03:57.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c19bd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c19bd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53128/0 (socket says 172.21.15.28:53128) 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2aa00700 1 -- 172.21.15.28:0/2557551630 learned_addr learned my addr 172.21.15.28:0/2557551630 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d22000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d2c076290 0x7f7d2c19c290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.912+0000 7f7d2b400700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d2c10fcb0 0x7f7d2c1a0620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d22000700 1 -- 172.21.15.28:0/2557551630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d2c10fcb0 msgr2=0x7f7d2c1a0620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d22000700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d2c10fcb0 0x7f7d2c1a0620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d22000700 1 -- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 msgr2=0x7f7d2c19bd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d22000700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c19bd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d22000700 1 -- 172.21.15.28:0/2557551630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7d2c1a0d60 con 0x7f7d2c076290 2024-05-20T20:03:57.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d2aa00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c19bd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:03:57.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d22000700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d2c076290 0x7f7d2c19c290 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f7d1400e9f0 tx=0x7f7d1400ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:57.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.913+0000 7f7d23e00700 1 -- 172.21.15.28:0/2557551630 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d1400cb70 con 0x7f7d2c076290 2024-05-20T20:03:57.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.914+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7d2c1a1050 con 0x7f7d2c076290 2024-05-20T20:03:57.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.914+0000 7f7d23e00700 1 -- 172.21.15.28:0/2557551630 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7d1400ee90 con 0x7f7d2c076290 2024-05-20T20:03:57.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.914+0000 7f7d23e00700 1 -- 172.21.15.28:0/2557551630 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7d14003d40 con 0x7f7d2c076290 2024-05-20T20:03:57.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.914+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7d2c1a88d0 con 0x7f7d2c076290 2024-05-20T20:03:57.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.915+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7d2c06a230 con 0x7f7d2c076290 2024-05-20T20:03:57.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.918+0000 7f7d23e00700 1 -- 172.21.15.28:0/2557551630 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f7d14003ee0 con 0x7f7d2c076290 2024-05-20T20:03:57.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.920+0000 7f7d23e00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d100b2880 0x7f7d100b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:03:57.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.920+0000 7f7d23e00700 1 -- 172.21.15.28:0/2557551630 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f7d140e25b0 con 0x7f7d2c076290 2024-05-20T20:03:57.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.920+0000 7f7d2aa00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d100b2880 0x7f7d100b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:03:57.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.920+0000 7f7d2aa00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d100b2880 0x7f7d100b4d40 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c006010 tx=0x7f7d1c01a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:03:57.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:57.922+0000 7f7d23e00700 1 -- 172.21.15.28:0/2557551630 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7d14054df0 con 0x7f7d2c076290 2024-05-20T20:03:58.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.216+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f7d2c19d0b0 con 0x7f7d2c076290 2024-05-20T20:03:58.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.218+0000 7f7d23e00700 1 -- 172.21.15.28:0/2557551630 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 28 v28) ==== 94+0+11925 (secure 0 0 0) 0x7f7d140aa290 con 0x7f7d2c076290 2024-05-20T20:03:58.218 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:03:58.219 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":28,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":28,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:01:50.441739+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24475,24482,24493,24500,24542]},"id":1}]} 2024-05-20T20:03:58.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d100b2880 msgr2=0x7f7d100b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:58.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d100b2880 0x7f7d100b4d40 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f7d1c006010 tx=0x7f7d1c01a040 comp rx=0 tx=0).stop 2024-05-20T20:03:58.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d2c076290 msgr2=0x7f7d2c19c290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:03:58.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d2c076290 0x7f7d2c19c290 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f7d1400e9f0 tx=0x7f7d1400ed00 comp rx=0 tx=0).stop 2024-05-20T20:03:58.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 shutdown_connections 2024-05-20T20:03:58.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7d100b2880 0x7f7d100b4d40 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:58.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d2c10fcb0 0x7f7d2c1a0620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:58.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d2c075970 0x7f7d2c19bd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:58.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 --2- 172.21.15.28:0/2557551630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d2c076290 0x7f7d2c19c290 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:03:58.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 >> 172.21.15.28:0/2557551630 conn(0x7f7d2c0fefc0 msgr2=0x7f7d2c0ffcf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:03:58.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 shutdown_connections 2024-05-20T20:03:58.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:03:58.221+0000 7f7d2be00700 1 -- 172.21.15.28:0/2557551630 wait complete. 2024-05-20T20:03:58.223 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 28 2024-05-20T20:03:58.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:58 smithi028 ceph-mon[31178]: pgmap v397: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:58.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:03:58 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2557551630' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:03:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:58 smithi154 ceph-mon[36314]: pgmap v397: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:03:58 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2557551630' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:03:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:58 smithi033 ceph-mon[36179]: pgmap v397: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:03:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:03:58 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2557551630' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:03:59.144 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:03:59.606 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:00.794 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:00 smithi028 ceph-mon[31178]: pgmap v398: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:00.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:00 smithi154 ceph-mon[36314]: pgmap v398: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:00.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:00 smithi033 ceph-mon[36179]: pgmap v398: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:01.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.946+0000 7fb929600700 1 -- 172.21.15.28:0/708023362 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb92410b670 msgr2=0x7fb92410ba50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:01.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.946+0000 7fb929600700 1 --2- 172.21.15.28:0/708023362 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb92410b670 0x7fb92410ba50 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fb914009990 tx=0x7fb914009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:01.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.946+0000 7fb929600700 1 -- 172.21.15.28:0/708023362 shutdown_connections 2024-05-20T20:04:01.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.946+0000 7fb929600700 1 --2- 172.21.15.28:0/708023362 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb924106030 0x7fb924112540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.946+0000 7fb929600700 1 --2- 172.21.15.28:0/708023362 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb92410b670 0x7fb92410ba50 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.946+0000 7fb929600700 1 --2- 172.21.15.28:0/708023362 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb924105670 0x7fb924105af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.946+0000 7fb929600700 1 -- 172.21.15.28:0/708023362 >> 172.21.15.28:0/708023362 conn(0x7fb924101160 msgr2=0x7fb924103580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.947+0000 7fb929600700 1 -- 172.21.15.28:0/708023362 shutdown_connections 2024-05-20T20:04:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.947+0000 7fb929600700 1 -- 172.21.15.28:0/708023362 wait complete. 2024-05-20T20:04:01.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.948+0000 7fb929600700 1 Processor -- start 2024-05-20T20:04:01.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.948+0000 7fb929600700 1 -- start start 2024-05-20T20:04:01.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb929600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb924105670 0x7fb9241a1940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:01.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb929600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb924106030 0x7fb9241a1e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb929600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb92410b670 0x7fb92419bb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb929600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb924115850 con 0x7fb924106030 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb929600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb9241156d0 con 0x7fb92410b670 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb929600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb9241159d0 con 0x7fb924105670 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb923e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb924105670 0x7fb9241a1940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb923400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb924106030 0x7fb9241a1e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb928c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb92410b670 0x7fb92419bb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb923e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb924105670 0x7fb9241a1940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33052/0 (socket says 172.21.15.28:33052) 2024-05-20T20:04:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.949+0000 7fb923e00700 1 -- 172.21.15.28:0/4267363831 learned_addr learned my addr 172.21.15.28:0/4267363831 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:01.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb923e00700 1 -- 172.21.15.28:0/4267363831 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb92410b670 msgr2=0x7fb92419bb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb923e00700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb92410b670 0x7fb92419bb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb923e00700 1 -- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb924106030 msgr2=0x7fb9241a1e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb923e00700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb924106030 0x7fb9241a1e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb923e00700 1 -- 172.21.15.28:0/4267363831 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb92419c2b0 con 0x7fb924105670 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb928c00700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb92410b670 0x7fb92419bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb923400700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb924106030 0x7fb9241a1e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb923e00700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb924105670 0x7fb9241a1940 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fb9140057f0 tx=0x7fb91400bfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb920c00700 1 -- 172.21.15.28:0/4267363831 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb914026070 con 0x7fb924105670 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb920c00700 1 -- 172.21.15.28:0/4267363831 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb914021410 con 0x7fb924105670 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb920c00700 1 -- 172.21.15.28:0/4267363831 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb914017650 con 0x7fb924105670 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb92419c540 con 0x7fb924105670 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.950+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb9241a8810 con 0x7fb924105670 2024-05-20T20:04:01.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.952+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb92410dc00 con 0x7fb924105670 2024-05-20T20:04:01.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.954+0000 7fb920c00700 1 -- 172.21.15.28:0/4267363831 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fb914029070 con 0x7fb924105670 2024-05-20T20:04:01.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.955+0000 7fb920c00700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb9080b2900 0x7fb9080b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:01.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.955+0000 7fb920c00700 1 -- 172.21.15.28:0/4267363831 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fb9140e3090 con 0x7fb924105670 2024-05-20T20:04:01.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.956+0000 7fb923400700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb9080b2900 0x7fb9080b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:01.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.956+0000 7fb923400700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb9080b2900 0x7fb9080b4dc0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fb924103820 tx=0x7fb910009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:01.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:01.958+0000 7fb920c00700 1 -- 172.21.15.28:0/4267363831 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb914030080 con 0x7fb924105670 2024-05-20T20:04:02.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.250+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fb92419cff0 con 0x7fb924105670 2024-05-20T20:04:02.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.251+0000 7fb920c00700 1 -- 172.21.15.28:0/4267363831 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 28 v28) ==== 94+0+11925 (secure 0 0 0) 0x7fb914013070 con 0x7fb924105670 2024-05-20T20:04:02.252 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:02.253 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":28,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":28,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:01:50.441739+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24475,24482,24493,24500,24542]},"id":1}]} 2024-05-20T20:04:02.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.254+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb9080b2900 msgr2=0x7fb9080b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:02.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.254+0000 7fb929600700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb9080b2900 0x7fb9080b4dc0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fb924103820 tx=0x7fb910009260 comp rx=0 tx=0).stop 2024-05-20T20:04:02.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb924105670 msgr2=0x7fb9241a1940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:02.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb924105670 0x7fb9241a1940 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fb9140057f0 tx=0x7fb91400bfd0 comp rx=0 tx=0).stop 2024-05-20T20:04:02.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 shutdown_connections 2024-05-20T20:04:02.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb9080b2900 0x7fb9080b4dc0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:02.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb92410b670 0x7fb92419bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:02.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb924105670 0x7fb9241a1940 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:02.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 --2- 172.21.15.28:0/4267363831 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb924106030 0x7fb9241a1e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:02.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 >> 172.21.15.28:0/4267363831 conn(0x7fb924101160 msgr2=0x7fb924101990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:02.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 shutdown_connections 2024-05-20T20:04:02.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:02.255+0000 7fb929600700 1 -- 172.21.15.28:0/4267363831 wait complete. 2024-05-20T20:04:02.257 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 28 2024-05-20T20:04:02.502 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:02 smithi028 ceph-mon[31178]: pgmap v399: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:02.502 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:02 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/4267363831' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:02 smithi154 ceph-mon[36314]: pgmap v399: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:02 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/4267363831' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:02.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:02 smithi033 ceph-mon[36179]: pgmap v399: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:02.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:02 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/4267363831' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:03.225 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:04:03.701 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:04.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:04 smithi028 ceph-mon[31178]: pgmap v400: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:04.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:04 smithi154 ceph-mon[36314]: pgmap v400: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:04 smithi033 ceph-mon[36179]: pgmap v400: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:05.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.990+0000 7f7f2b600700 1 -- 172.21.15.28:0/4032898621 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 msgr2=0x7f7f24112330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.990+0000 7f7f2b600700 1 --2- 172.21.15.28:0/4032898621 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 0x7f7f24112330 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f7f1c00b0a0 tx=0x7f7f1c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.991+0000 7f7f2b600700 1 -- 172.21.15.28:0/4032898621 shutdown_connections 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.991+0000 7f7f2b600700 1 --2- 172.21.15.28:0/4032898621 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 0x7f7f24112330 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.991+0000 7f7f2b600700 1 --2- 172.21.15.28:0/4032898621 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7f24075770 0x7f7f24075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.991+0000 7f7f2b600700 1 --2- 172.21.15.28:0/4032898621 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 0x7f7f2410fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.991+0000 7f7f2b600700 1 -- 172.21.15.28:0/4032898621 >> 172.21.15.28:0/4032898621 conn(0x7f7f2407a6a0 msgr2=0x7f7f2407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.991+0000 7f7f2b600700 1 -- 172.21.15.28:0/4032898621 shutdown_connections 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.991+0000 7f7f2b600700 1 -- 172.21.15.28:0/4032898621 wait complete. 2024-05-20T20:04:05.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 Processor -- start 2024-05-20T20:04:05.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 -- start start 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7f24075770 0x7f7f2419bd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 0x7f7f2419c2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 0x7f7f241a0630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7f24115930 con 0x7f7f24076090 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7f241157b0 con 0x7f7f2410ff40 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.993+0000 7f7f2b600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7f24115ab0 con 0x7f7f24075770 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 0x7f7f2419c2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:05.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 0x7f7f2419c2a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53158/0 (socket says 172.21.15.28:53158) 2024-05-20T20:04:05.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 -- 172.21.15.28:0/1885896329 learned_addr learned my addr 172.21.15.28:0/1885896329 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:05.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f28c00700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7f24075770 0x7f7f2419bd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:05.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f29600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 0x7f7f241a0630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:05.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 -- 172.21.15.28:0/1885896329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7f24075770 msgr2=0x7f7f2419bd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:05.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7f24075770 0x7f7f2419bd60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 -- 172.21.15.28:0/1885896329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 msgr2=0x7f7f241a0630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 0x7f7f241a0630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f23e00700 1 -- 172.21.15.28:0/1885896329 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7f241a0d70 con 0x7f7f24076090 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.994+0000 7f7f29600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 0x7f7f241a0630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.995+0000 7f7f23e00700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 0x7f7f2419c2a0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f7f1000e970 tx=0x7f7f1000ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.995+0000 7f7f21600700 1 -- 172.21.15.28:0/1885896329 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7f10004220 con 0x7f7f24076090 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.995+0000 7f7f21600700 1 -- 172.21.15.28:0/1885896329 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7f100043c0 con 0x7f7f24076090 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.995+0000 7f7f21600700 1 -- 172.21.15.28:0/1885896329 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7f10003ce0 con 0x7f7f24076090 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.995+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7f241a1060 con 0x7f7f24076090 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.995+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7f241a88e0 con 0x7f7f24076090 2024-05-20T20:04:05.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.996+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7f2406a230 con 0x7f7f24076090 2024-05-20T20:04:05.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:05.999+0000 7f7f21600700 1 -- 172.21.15.28:0/1885896329 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f7f1000c7e0 con 0x7f7f24076090 2024-05-20T20:04:06.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.000+0000 7f7f21600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7f0c0b2900 0x7f7f0c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:06.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.000+0000 7f7f21600700 1 -- 172.21.15.28:0/1885896329 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f7f100e2450 con 0x7f7f24076090 2024-05-20T20:04:06.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.001+0000 7f7f28c00700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7f0c0b2900 0x7f7f0c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:06.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.001+0000 7f7f28c00700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7f0c0b2900 0x7f7f0c0b4dc0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f7f18007c60 tx=0x7f7f180073d0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:06.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.003+0000 7f7f21600700 1 -- 172.21.15.28:0/1885896329 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7f100aa1c0 con 0x7f7f24076090 2024-05-20T20:04:06.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.239+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f7f2402e560 con 0x7f7f24076090 2024-05-20T20:04:06.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.245+0000 7f7f21600700 1 -- 172.21.15.28:0/1885896329 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) ==== 74+0+22571 (secure 0 0 0) 0x7f7f100a9ac0 con 0x7f7f24076090 2024-05-20T20:04:06.245 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:06.246 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":80,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:02:04.742448+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:04:06.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7f0c0b2900 msgr2=0x7f7f0c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:06.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7f0c0b2900 0x7f7f0c0b4dc0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f7f18007c60 tx=0x7f7f180073d0 comp rx=0 tx=0).stop 2024-05-20T20:04:06.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 msgr2=0x7f7f2419c2a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:06.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 0x7f7f2419c2a0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f7f1000e970 tx=0x7f7f1000ec80 comp rx=0 tx=0).stop 2024-05-20T20:04:06.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 shutdown_connections 2024-05-20T20:04:06.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7f0c0b2900 0x7f7f0c0b4dc0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:06.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7f2410ff40 0x7f7f241a0630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:06.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7f24075770 0x7f7f2419bd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:06.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 --2- 172.21.15.28:0/1885896329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7f24076090 0x7f7f2419c2a0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:06.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.248+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 >> 172.21.15.28:0/1885896329 conn(0x7f7f2407a6a0 msgr2=0x7f7f241006b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:06.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.249+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 shutdown_connections 2024-05-20T20:04:06.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:06.249+0000 7f7f2b600700 1 -- 172.21.15.28:0/1885896329 wait complete. 2024-05-20T20:04:06.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:06 smithi028 ceph-mon[31178]: pgmap v401: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:06.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:06 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1885896329' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:04:06.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:06 smithi154 ceph-mon[36314]: pgmap v401: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:06.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:06 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1885896329' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:04:06.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:06 smithi033 ceph-mon[36179]: pgmap v401: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:06.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:06 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1885896329' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:04:07.144 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:04:07.564 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:08.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:08 smithi028 ceph-mon[31178]: pgmap v402: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:04:08.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:08 smithi154 ceph-mon[36314]: pgmap v402: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:04:08.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:08 smithi033 ceph-mon[36179]: pgmap v402: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.822+0000 7fb89d600700 1 -- 172.21.15.28:0/1282255342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898105780 msgr2=0x7fb898105c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.822+0000 7fb89d600700 1 --2- 172.21.15.28:0/1282255342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898105780 0x7fb898105c00 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fb884009990 tx=0x7fb884009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.823+0000 7fb89d600700 1 -- 172.21.15.28:0/1282255342 shutdown_connections 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.823+0000 7fb89d600700 1 --2- 172.21.15.28:0/1282255342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898106140 0x7fb898112650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.823+0000 7fb89d600700 1 --2- 172.21.15.28:0/1282255342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb89810b780 0x7fb89810bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.823+0000 7fb89d600700 1 --2- 172.21.15.28:0/1282255342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898105780 0x7fb898105c00 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.823+0000 7fb89d600700 1 -- 172.21.15.28:0/1282255342 >> 172.21.15.28:0/1282255342 conn(0x7fb898101270 msgr2=0x7fb898103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.823+0000 7fb89d600700 1 -- 172.21.15.28:0/1282255342 shutdown_connections 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.823+0000 7fb89d600700 1 -- 172.21.15.28:0/1282255342 wait complete. 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 Processor -- start 2024-05-20T20:04:09.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 -- start start 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898105780 0x7fb89819be80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898106140 0x7fb89819c3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb89810b780 0x7fb8981a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb898115a10 con 0x7fb898105780 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb898115890 con 0x7fb89810b780 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.824+0000 7fb89d600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb898115b90 con 0x7fb898106140 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898105780 0x7fb89819be80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898105780 0x7fb89819be80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53026/0 (socket says 172.21.15.28:53026) 2024-05-20T20:04:09.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 -- 172.21.15.28:0/536308307 learned_addr learned my addr 172.21.15.28:0/536308307 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb897400700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb89810b780 0x7fb8981a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896000700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898106140 0x7fb89819c3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 -- 172.21.15.28:0/536308307 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898106140 msgr2=0x7fb89819c3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898106140 0x7fb89819c3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 -- 172.21.15.28:0/536308307 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb89810b780 msgr2=0x7fb8981a0750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb89810b780 0x7fb8981a0750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 -- 172.21.15.28:0/536308307 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb8981a0e90 con 0x7fb898105780 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.825+0000 7fb896a00700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898105780 0x7fb89819be80 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb88800e9c0 tx=0x7fb88800ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.826+0000 7fb88f400700 1 -- 172.21.15.28:0/536308307 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb88800cbd0 con 0x7fb898105780 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.826+0000 7fb88f400700 1 -- 172.21.15.28:0/536308307 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb88800ee60 con 0x7fb898105780 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.826+0000 7fb88f400700 1 -- 172.21.15.28:0/536308307 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8880055c0 con 0x7fb898105780 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.826+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb8981a1180 con 0x7fb898105780 2024-05-20T20:04:09.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.826+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb8981a8a00 con 0x7fb898105780 2024-05-20T20:04:09.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.827+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb89806a230 con 0x7fb898105780 2024-05-20T20:04:09.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.829+0000 7fb88f400700 1 -- 172.21.15.28:0/536308307 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fb888004750 con 0x7fb898105780 2024-05-20T20:04:09.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.832+0000 7fb88f400700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb87c0b2900 0x7fb87c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:09.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.832+0000 7fb88f400700 1 -- 172.21.15.28:0/536308307 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fb8880e2610 con 0x7fb898105780 2024-05-20T20:04:09.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.832+0000 7fb896000700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb87c0b2900 0x7fb87c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:09.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.833+0000 7fb896000700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb87c0b2900 0x7fb87c0b4dc0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fb898103860 tx=0x7fb884005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:09.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:09.834+0000 7fb88f400700 1 -- 172.21.15.28:0/536308307 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb888054df0 con 0x7fb898105780 2024-05-20T20:04:10.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.127+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fb89802e560 con 0x7fb898105780 2024-05-20T20:04:10.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.129+0000 7fb88f400700 1 -- 172.21.15.28:0/536308307 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 28 v28) ==== 94+0+11925 (secure 0 0 0) 0x7fb8880aa2f0 con 0x7fb898105780 2024-05-20T20:04:10.129 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:10.130 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":28,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":28,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:01:50.441739+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24475,24482,24493,24500,24542]},"id":1}]} 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb87c0b2900 msgr2=0x7fb87c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb87c0b2900 0x7fb87c0b4dc0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fb898103860 tx=0x7fb884005fd0 comp rx=0 tx=0).stop 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898105780 msgr2=0x7fb89819be80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898105780 0x7fb89819be80 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb88800e9c0 tx=0x7fb88800ecd0 comp rx=0 tx=0).stop 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 shutdown_connections 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fb87c0b2900 0x7fb87c0b4dc0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb89810b780 0x7fb8981a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb898105780 0x7fb89819be80 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 --2- 172.21.15.28:0/536308307 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb898106140 0x7fb89819c3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.132+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 >> 172.21.15.28:0/536308307 conn(0x7fb898101270 msgr2=0x7fb898101970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.133+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 shutdown_connections 2024-05-20T20:04:10.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:10.133+0000 7fb89d600700 1 -- 172.21.15.28:0/536308307 wait complete. 2024-05-20T20:04:10.134 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 28 2024-05-20T20:04:10.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:10 smithi028 ceph-mon[31178]: pgmap v403: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:10.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:10 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/536308307' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:10 smithi154 ceph-mon[36314]: pgmap v403: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:10 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/536308307' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:10.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:10 smithi033 ceph-mon[36179]: pgmap v403: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:10.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:10 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/536308307' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:11.073 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 28, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-05-20T20:01:26.231967+0000', 'modified': '2024-05-20T20:01:50.441739+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 5, 'in': [0, 1, 2, 3, 4], 'up': {'mds_0': 24542, 'mds_1': 24500, 'mds_2': 24493, 'mds_3': 24482, 'mds_4': 24475}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_14400': {'gid': 14400, 'name': 'a', 'rank': 4, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.28:6841/1736910968', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.28:6840', 'nonce': 1736910968}, {'type': 'v1', 'addr': '172.21.15.28:6841', 'nonce': 1736910968}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24427': {'gid': 24427, 'name': 'l', 'rank': 3, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.154:6833/2654302434', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6832', 'nonce': 2654302434}, {'type': 'v1', 'addr': '172.21.15.154:6833', 'nonce': 2654302434}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24442': {'gid': 24442, 'name': 'j', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.28:6835/34026174', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.28:6834', 'nonce': 34026174}, {'type': 'v1', 'addr': '172.21.15.28:6835', 'nonce': 34026174}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24455': {'gid': 24455, 'name': 'k', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.33:6833/260147664', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6832', 'nonce': 260147664}, {'type': 'v1', 'addr': '172.21.15.33:6833', 'nonce': 260147664}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24463': {'gid': 24463, 'name': 'f', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.154:6835/3965936826', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6834', 'nonce': 3965936826}, {'type': 'v1', 'addr': '172.21.15.154:6835', 'nonce': 3965936826}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24475': {'gid': 24475, 'name': 'c', 'rank': 4, 'incarnation': 25, 'state': 'up:active', 'state_seq': 17, 'addr': '172.21.15.154:6837/2808233682', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6836', 'nonce': 2808233682}, {'type': 'v1', 'addr': '172.21.15.154:6837', 'nonce': 2808233682}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24482': {'gid': 24482, 'name': 'e', 'rank': 3, 'incarnation': 23, 'state': 'up:active', 'state_seq': 18, 'addr': '172.21.15.33:6835/4076526106', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6834', 'nonce': 4076526106}, {'type': 'v1', 'addr': '172.21.15.33:6835', 'nonce': 4076526106}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24493': {'gid': 24493, 'name': 'i', 'rank': 2, 'incarnation': 21, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.154:6839/1291909351', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6838', 'nonce': 1291909351}, {'type': 'v1', 'addr': '172.21.15.154:6839', 'nonce': 1291909351}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24500': {'gid': 24500, 'name': 'h', 'rank': 1, 'incarnation': 19, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.33:6837/4183908883', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6836', 'nonce': 4183908883}, {'type': 'v1', 'addr': '172.21.15.33:6837', 'nonce': 4183908883}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24542': {'gid': 24542, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 7, 'addr': '172.21.15.33:6839/2646117588', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6838', 'nonce': 2646117588}, {'type': 'v1', 'addr': '172.21.15.33:6839', 'nonce': 2646117588}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 24542, 'qdb_cluster': [24475, 24482, 24493, 24500, 24542]} 2024-05-20T20:04:11.073 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 5/5 2024-05-20T20:04:11.073 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.c status 2024-05-20T20:04:11.547 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:12.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:12 smithi028 ceph-mon[31178]: pgmap v404: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:12.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:12 smithi154 ceph-mon[36314]: pgmap v404: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:12.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:12 smithi033 ceph-mon[36179]: pgmap v404: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:13.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 -- 172.21.15.28:0/2554830950 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd954108500 msgr2=0x7fd9541088e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 --2- 172.21.15.28:0/2554830950 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd954108500 0x7fd9541088e0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fd948009990 tx=0x7fd948009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 -- 172.21.15.28:0/2554830950 shutdown_connections 2024-05-20T20:04:13.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 --2- 172.21.15.28:0/2554830950 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd954103f90 0x7fd954110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 --2- 172.21.15.28:0/2554830950 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd954108500 0x7fd9541088e0 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 --2- 172.21.15.28:0/2554830950 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd9541035f0 0x7fd954103a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 -- 172.21.15.28:0/2554830950 >> 172.21.15.28:0/2554830950 conn(0x7fd95407a6a0 msgr2=0x7fd95407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:13.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 -- 172.21.15.28:0/2554830950 shutdown_connections 2024-05-20T20:04:13.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.860+0000 7fd95bc00700 1 -- 172.21.15.28:0/2554830950 wait complete. 2024-05-20T20:04:13.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.861+0000 7fd95bc00700 1 Processor -- start 2024-05-20T20:04:13.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd95bc00700 1 -- start start 2024-05-20T20:04:13.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd95bc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd9541035f0 0x7fd95419bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd95bc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd954103f90 0x7fd95419c460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd95bc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd954108500 0x7fd9541a07f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd95bc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd954113770 con 0x7fd9541035f0 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd95bc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd9541135f0 con 0x7fd954103f90 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd95bc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd9541138f0 con 0x7fd954108500 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd959200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd9541035f0 0x7fd95419bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd959200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd9541035f0 0x7fd95419bf20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53048/0 (socket says 172.21.15.28:53048) 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.862+0000 7fd959200700 1 -- 172.21.15.28:0/3901836671 learned_addr learned my addr 172.21.15.28:0/3901836671 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd953e00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd954103f90 0x7fd95419c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd959c00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd954108500 0x7fd9541a07f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd959200700 1 -- 172.21.15.28:0/3901836671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd954108500 msgr2=0x7fd9541a07f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd959200700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd954108500 0x7fd9541a07f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd959200700 1 -- 172.21.15.28:0/3901836671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd954103f90 msgr2=0x7fd95419c460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd959200700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd954103f90 0x7fd95419c460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd959200700 1 -- 172.21.15.28:0/3901836671 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd9541a0f30 con 0x7fd9541035f0 2024-05-20T20:04:13.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.863+0000 7fd959200700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd9541035f0 0x7fd95419bf20 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fd948006010 tx=0x7fd9480057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:13.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.864+0000 7fd951600700 1 -- 172.21.15.28:0/3901836671 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd94800fb00 con 0x7fd9541035f0 2024-05-20T20:04:13.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.864+0000 7fd951600700 1 -- 172.21.15.28:0/3901836671 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd94800fca0 con 0x7fd9541035f0 2024-05-20T20:04:13.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.864+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd9541a11c0 con 0x7fd9541035f0 2024-05-20T20:04:13.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.864+0000 7fd951600700 1 -- 172.21.15.28:0/3901836671 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd948017650 con 0x7fd9541035f0 2024-05-20T20:04:13.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.864+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd9541a8aa0 con 0x7fd9541035f0 2024-05-20T20:04:13.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.869+0000 7fd951600700 1 -- 172.21.15.28:0/3901836671 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fd948009d10 con 0x7fd9541035f0 2024-05-20T20:04:13.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.870+0000 7fd966532b80 1 Processor -- start 2024-05-20T20:04:13.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.870+0000 7fd966532b80 1 -- start start 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.870+0000 7fd966532b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d89240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd951600700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd93c0b2880 0x7fd93c0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd966532b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d83210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd966532b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d83c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd966532b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556c98ce1df0 con 0x556c98d89780 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd966532b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556c98cd1990 con 0x556c98d83830 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd966532b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556c98cd59d0 con 0x556c98d88e60 2024-05-20T20:04:13.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd951600700 1 -- 172.21.15.28:0/3901836671 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fd9480e3180 con 0x7fd9541035f0 2024-05-20T20:04:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd953e00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd93c0b2880 0x7fd93c0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd959c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d89240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd959c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d89240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38806/0 (socket says 172.21.15.28:38806) 2024-05-20T20:04:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd959c00700 1 -- 172.21.15.28:0/2692932069 learned_addr learned my addr 172.21.15.28:0/2692932069 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:13.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd959c00700 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d83c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.871+0000 7fd959200700 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d83210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd959c00700 1 -- 172.21.15.28:0/2692932069 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 msgr2=0x556c98d83c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd959c00700 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d83c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd959c00700 1 -- 172.21.15.28:0/2692932069 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 msgr2=0x556c98d83210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd959c00700 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d83210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd959c00700 1 -- 172.21.15.28:0/2692932069 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556c98c9a3d0 con 0x556c98d88e60 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd959200700 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d83210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd959c00700 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d89240 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fd94c0079b0 tx=0x7fd94c00b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=14bb1515b9deb147 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd953e00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd93c0b2880 0x7fd93c0b4d40 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fd9540777d0 tx=0x7fd940009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd946000700 1 -- 172.21.15.28:0/2692932069 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd94c004bb0 con 0x556c98d88e60 2024-05-20T20:04:13.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd946000700 1 -- 172.21.15.28:0/2692932069 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd94c004d50 con 0x556c98d88e60 2024-05-20T20:04:13.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.872+0000 7fd946000700 1 -- 172.21.15.28:0/2692932069 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd94c01a640 con 0x556c98d88e60 2024-05-20T20:04:13.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.873+0000 7fd966532b80 1 -- 172.21.15.28:0/2692932069 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 msgr2=0x556c98d89240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.873+0000 7fd966532b80 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d89240 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fd94c0079b0 tx=0x7fd94c00b040 comp rx=0 tx=0).stop 2024-05-20T20:04:13.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.873+0000 7fd966532b80 1 -- 172.21.15.28:0/2692932069 shutdown_connections 2024-05-20T20:04:13.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.873+0000 7fd966532b80 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d83c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.873+0000 7fd966532b80 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d89240 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.874+0000 7fd966532b80 1 --2- 172.21.15.28:0/2692932069 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d83210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.874+0000 7fd966532b80 1 -- 172.21.15.28:0/2692932069 >> 172.21.15.28:0/2692932069 conn(0x556c98cb1b90 msgr2=0x556c98d7dfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.874+0000 7fd966532b80 1 -- 172.21.15.28:0/2692932069 shutdown_connections 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.874+0000 7fd966532b80 1 -- 172.21.15.28:0/2692932069 wait complete. 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.874+0000 7fd966532b80 1 -- start start 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.874+0000 7fd966532b80 1 Processor -- start 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd966532b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d8f4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd966532b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d8c840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd966532b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d8cd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd966532b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556c98bd7f50 con 0x556c98d89780 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd966532b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556c98c72710 con 0x556c98d83830 2024-05-20T20:04:13.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd966532b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556c98d901f0 con 0x556c98d88e60 2024-05-20T20:04:13.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd966532b80 10 client.0 _ll_register_callbacks cb 0x556c98c7bc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:13.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd959200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d8f4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd959c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d8cd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd959200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d8f4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49406/0 (socket says 172.21.15.28:49406) 2024-05-20T20:04:13.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd959200700 1 -- 172.21.15.28:0/2865653527 learned_addr learned my addr 172.21.15.28:0/2865653527 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:13.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.875+0000 7fd953e00700 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d8c840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd959200700 1 -- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 msgr2=0x556c98d8c840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd959200700 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d8c840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd959200700 1 -- 172.21.15.28:0/2865653527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 msgr2=0x556c98d8cd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd959200700 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d8cd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd959200700 1 -- 172.21.15.28:0/2865653527 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556c98c21880 con 0x556c98d83830 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd959200700 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d8f4a0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fd948009550 tx=0x7fd9480ca3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dd7d1c188b8e5de9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd93aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd93aa00700 1 -- 172.21.15.28:0/2865653527 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd9480c5050 con 0x556c98d83830 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd93aa00700 1 -- 172.21.15.28:0/2865653527 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fd9480b3130 con 0x556c98d83830 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.876+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd94c004d50 con 0x556c98d83830 2024-05-20T20:04:13.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd93aa00700 1 -- 172.21.15.28:0/2865653527 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fd9480cd0b0 con 0x556c98d83830 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 10 client.24887 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x556c98ca2010 con 0x556c98d83830 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd93aa00700 1 -- 172.21.15.28:0/2865653527 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd94c004d50 con 0x556c98d83830 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd93aa00700 1 -- 172.21.15.28:0/2865653527 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fd948092110 con 0x556c98d83830 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 10 client.24887 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 10 client.24887 resolve_mds: resolved name 'c' to daemon mds.c 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 20 client.24887 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98d9cd10 0x556c98d9e180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 4 client.24887 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.877+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556c98ce3d40 con 0x556c98d9cd10 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.878+0000 7fd953e00700 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98d9cd10 0x556c98d9e180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.878+0000 7fd953e00700 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98d9cd10 0x556c98d9e180 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=a4bdd53142fa6956 server_cookie=aeb826a03d01caa8 in_seq=0 out_seq=0 2024-05-20T20:04:13.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.878+0000 7fd93aa00700 10 client.24887 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:04:13.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.879+0000 7fd93aa00700 1 -- 172.21.15.28:0/2865653527 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x556c98ce3d40 con 0x556c98d9cd10 2024-05-20T20:04:13.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.879+0000 7fd93aa00700 10 client.24887 handle_command_reply: tid=0 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 client.24887 shutdown 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 msgr2=0x556c98d8f4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d8f4a0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fd948009550 tx=0x7fd9480ca3b0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 shutdown_connections 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d89780 0x556c98d8cd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98d8f4a0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98d9cd10 0x556c98d9e180 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 --2- 172.21.15.28:0/2865653527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d88e60 0x556c98d8c840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd93aa00700 0 client.24887 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 >> 172.21.15.28:0/2865653527 conn(0x556c98cb1b90 msgr2=0x556c98d82200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 shutdown_connections 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 1 -- 172.21.15.28:0/2865653527 wait complete. 2024-05-20T20:04:13.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.880+0000 7fd966532b80 20 client.24887 trim_cache size 0 max 16384 2024-05-20T20:04:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 Processor -- start 2024-05-20T20:04:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 -- start start 2024-05-20T20:04:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d83830 0x556c98d83210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d88e60 0x556c98d86970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98d86eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556c98d9c4c0 con 0x556c98d83830 2024-05-20T20:04:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556c98d9cb90 con 0x556c98d88e60 2024-05-20T20:04:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.904+0000 7fd966532b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556c98d9d260 con 0x556c98d89780 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d83830 0x556c98d83210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd953e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d88e60 0x556c98d86970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d83830 0x556c98d83210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53080/0 (socket says 172.21.15.28:53080) 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959200700 1 -- 172.21.15.28:0/2698334269 learned_addr learned my addr 172.21.15.28:0/2698334269 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959c00700 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98d86eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959c00700 1 -- 172.21.15.28:0/2698334269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d88e60 msgr2=0x556c98d86970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959c00700 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d88e60 0x556c98d86970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959c00700 1 -- 172.21.15.28:0/2698334269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d83830 msgr2=0x556c98d83210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959c00700 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d83830 0x556c98d83210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959c00700 1 -- 172.21.15.28:0/2698334269 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556c98d8adb0 con 0x556c98d89780 2024-05-20T20:04:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd953e00700 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d88e60 0x556c98d86970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.905+0000 7fd959c00700 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98d86eb0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fd94c007980 tx=0x7fd94c0094d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a7463b5b4aaf7213 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.906+0000 7fd945600700 1 -- 172.21.15.28:0/2698334269 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd94c005920 con 0x556c98d89780 2024-05-20T20:04:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.906+0000 7fd945600700 1 -- 172.21.15.28:0/2698334269 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd94c0050d0 con 0x556c98d89780 2024-05-20T20:04:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.906+0000 7fd945600700 1 -- 172.21.15.28:0/2698334269 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd94c01b440 con 0x556c98d89780 2024-05-20T20:04:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.906+0000 7fd966532b80 1 -- 172.21.15.28:0/2698334269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 msgr2=0x556c98d86eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.906+0000 7fd966532b80 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98d86eb0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fd94c007980 tx=0x7fd94c0094d0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 -- 172.21.15.28:0/2698334269 shutdown_connections 2024-05-20T20:04:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98d86eb0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d83830 0x556c98d83210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 --2- 172.21.15.28:0/2698334269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d88e60 0x556c98d86970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 -- 172.21.15.28:0/2698334269 >> 172.21.15.28:0/2698334269 conn(0x556c98cb1b90 msgr2=0x556c98d8c800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 -- 172.21.15.28:0/2698334269 shutdown_connections 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 -- 172.21.15.28:0/2698334269 wait complete. 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 -- start start 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.907+0000 7fd966532b80 1 Processor -- start 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd966532b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98da77e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd966532b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d88e60 0x556c98da7d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd966532b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98da25b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd966532b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556c98d9e000 con 0x556c98d88e60 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd966532b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556c98d9d930 con 0x556c98d83830 2024-05-20T20:04:13.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd966532b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556c98d9e6d0 con 0x556c98d89780 2024-05-20T20:04:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd966532b80 10 client.0 _ll_register_callbacks cb 0x556c98c7bc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd953e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d88e60 0x556c98da7d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd959c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98da25b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd959c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98da25b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38826/0 (socket says 172.21.15.28:38826) 2024-05-20T20:04:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd959200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98da77e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd959c00700 1 -- 172.21.15.28:0/1359474849 learned_addr learned my addr 172.21.15.28:0/1359474849 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.908+0000 7fd953e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d88e60 0x556c98da7d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53096/0 (socket says 172.21.15.28:53096) 2024-05-20T20:04:13.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd959c00700 1 -- 172.21.15.28:0/1359474849 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 msgr2=0x556c98da77e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd959c00700 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98da77e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd959c00700 1 -- 172.21.15.28:0/1359474849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d88e60 msgr2=0x556c98da7d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd959c00700 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d88e60 0x556c98da7d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd959c00700 1 -- 172.21.15.28:0/1359474849 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556c98da2df0 con 0x556c98d89780 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd953e00700 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d88e60 0x556c98da7d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd959c00700 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98da25b0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fd94c009cb0 tx=0x7fd94c009950 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6a479a655cb6471a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd93aa00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.909+0000 7fd93aa00700 1 -- 172.21.15.28:0/1359474849 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd94c005490 con 0x556c98d89780 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.910+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd94c0050d0 con 0x556c98d89780 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.910+0000 7fd93aa00700 1 -- 172.21.15.28:0/1359474849 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fd94c003a00 con 0x556c98d89780 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.910+0000 7fd93aa00700 1 -- 172.21.15.28:0/1359474849 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fd94c005630 con 0x556c98d89780 2024-05-20T20:04:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.910+0000 7fd966532b80 10 client.24877 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.910+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x556c98e11760 con 0x556c98d89780 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.910+0000 7fd93aa00700 1 -- 172.21.15.28:0/1359474849 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd94c0050d0 con 0x556c98d89780 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd93aa00700 1 -- 172.21.15.28:0/1359474849 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fd94c015070 con 0x556c98d89780 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd966532b80 10 client.24877 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd966532b80 10 client.24877 resolve_mds: resolved name 'c' to daemon mds.c 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd966532b80 20 client.24877 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd966532b80 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98e15bf0 0x556c98e14900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd966532b80 4 client.24877 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "status"}) -- 0x556c98cdd520 con 0x556c98e15bf0 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd959200700 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98e15bf0 0x556c98e14900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd959200700 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98e15bf0 0x556c98e14900 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=487f801e336bb3a2 server_cookie=774fc5334efa1e04 in_seq=0 out_seq=0 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.911+0000 7fd93aa00700 10 client.24877 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:04:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd93aa00700 1 -- 172.21.15.28:0/1359474849 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x556c98cdd520 con 0x556c98e15bf0 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd93aa00700 10 client.24877 handle_command_reply: tid=0 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 client.24877 shutdown 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 msgr2=0x556c98da25b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98da25b0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fd94c009cb0 tx=0x7fd94c009950 comp rx=0 tx=0).stop 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 shutdown_connections 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x556c98e15bf0 0x556c98e14900 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556c98d89780 0x556c98da25b0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556c98d83830 0x556c98da77e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 --2- 172.21.15.28:0/1359474849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556c98d88e60 0x556c98da7d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 >> 172.21.15.28:0/1359474849 conn(0x556c98cb1b90 msgr2=0x556c98d86970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd93aa00700 0 client.24877 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 shutdown_connections 2024-05-20T20:04:13.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 1 -- 172.21.15.28:0/1359474849 wait complete. 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.912+0000 7fd966532b80 20 client.24877 trim_cache size 0 max 16384 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 4, 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24475, 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 148.26590671100001, 2024-05-20T20:04:13.916 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:04:13.917 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:04:13.917 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:04:13.917 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 209.37636325299999 2024-05-20T20:04:13.917 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:04:13.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd93c0b2880 msgr2=0x7fd93c0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd93c0b2880 0x7fd93c0b4d40 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fd9540777d0 tx=0x7fd940009260 comp rx=0 tx=0).stop 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd9541035f0 msgr2=0x7fd95419bf20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd9541035f0 0x7fd95419bf20 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fd948006010 tx=0x7fd9480057c0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 shutdown_connections 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fd93c0b2880 0x7fd93c0b4d40 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd954108500 0x7fd9541a07f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd9541035f0 0x7fd95419bf20 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 --2- 172.21.15.28:0/3901836671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd954103f90 0x7fd95419c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 >> 172.21.15.28:0/3901836671 conn(0x7fd95407a6a0 msgr2=0x7fd954101460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 shutdown_connections 2024-05-20T20:04:13.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:13.913+0000 7fd95bc00700 1 -- 172.21.15.28:0/3901836671 wait complete. 2024-05-20T20:04:14.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:14 smithi028 ceph-mon[31178]: pgmap v405: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:14.845 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.e status 2024-05-20T20:04:14.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:14 smithi154 ceph-mon[36314]: pgmap v405: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:14.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:14 smithi033 ceph-mon[36179]: pgmap v405: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:15.308 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:16.589 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:16 smithi028 ceph-mon[31178]: pgmap v406: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:16 smithi154 ceph-mon[36314]: pgmap v406: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:16.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:16 smithi033 ceph-mon[36179]: pgmap v406: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:17.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 -- 172.21.15.28:0/4245871940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806b220 msgr2=0x7efec806b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 --2- 172.21.15.28:0/4245871940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806b220 0x7efec806b6a0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7efeb4009990 tx=0x7efeb4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 -- 172.21.15.28:0/4245871940 shutdown_connections 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 --2- 172.21.15.28:0/4245871940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efec806bbe0 0x7efec81103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 --2- 172.21.15.28:0/4245871940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec81095b0 0x7efec8109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 --2- 172.21.15.28:0/4245871940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806b220 0x7efec806b6a0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 -- 172.21.15.28:0/4245871940 >> 172.21.15.28:0/4245871940 conn(0x7efec80793f0 msgr2=0x7efec8079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 -- 172.21.15.28:0/4245871940 shutdown_connections 2024-05-20T20:04:17.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.438+0000 7efecf400700 1 -- 172.21.15.28:0/4245871940 wait complete. 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.439+0000 7efecf400700 1 Processor -- start 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.439+0000 7efecf400700 1 -- start start 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecf400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec806b220 0x7efec819be90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecf400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806bbe0 0x7efec819c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecf400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efec81095b0 0x7efec81a0760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecf400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7efec8113720 con 0x7efec806bbe0 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecf400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7efec81135a0 con 0x7efec806b220 2024-05-20T20:04:17.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecf400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7efec81138a0 con 0x7efec81095b0 2024-05-20T20:04:17.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec806b220 0x7efec819be90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efec7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806bbe0 0x7efec819c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec806b220 0x7efec819be90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43180/0 (socket says 172.21.15.28:43180) 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecca00700 1 -- 172.21.15.28:0/4055460505 learned_addr learned my addr 172.21.15.28:0/4055460505 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.440+0000 7efecd400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efec81095b0 0x7efec81a0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efecca00700 1 -- 172.21.15.28:0/4055460505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efec81095b0 msgr2=0x7efec81a0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efecca00700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efec81095b0 0x7efec81a0760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efecca00700 1 -- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806bbe0 msgr2=0x7efec819c3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efecca00700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806bbe0 0x7efec819c3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efecca00700 1 -- 172.21.15.28:0/4055460505 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efec81a0ea0 con 0x7efec806b220 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efecca00700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec806b220 0x7efec819be90 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7efebc00d980 tx=0x7efebc00dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efec5600700 1 -- 172.21.15.28:0/4055460505 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efebc016070 con 0x7efec806b220 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efec5600700 1 -- 172.21.15.28:0/4055460505 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efebc004830 con 0x7efec806b220 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efec5600700 1 -- 172.21.15.28:0/4055460505 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efebc005540 con 0x7efec806b220 2024-05-20T20:04:17.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.441+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efec81a1190 con 0x7efec806b220 2024-05-20T20:04:17.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.442+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efec81a8a10 con 0x7efec806b220 2024-05-20T20:04:17.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.445+0000 7efec5600700 1 -- 172.21.15.28:0/4055460505 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7efebc004220 con 0x7efec806b220 2024-05-20T20:04:17.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.446+0000 7efed9db4b80 1 Processor -- start 2024-05-20T20:04:17.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efed9db4b80 1 -- start start 2024-05-20T20:04:17.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efec5600700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efeb00b2880 0x7efeb00b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d469e60 0x55ec2d46a240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d464210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d464c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efec5600700 1 -- 172.21.15.28:0/4055460505 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7efebc0e1f80 con 0x7efec806b220 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efed9db4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d3c2df0 con 0x55ec2d469e60 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efed9db4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d3b2990 con 0x55ec2d464830 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efed9db4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d3b69d0 con 0x55ec2d46a780 2024-05-20T20:04:17.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.447+0000 7efecd400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efeb00b2880 0x7efeb00b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d464210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d464210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60026/0 (socket says 172.21.15.28:60026) 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 -- 172.21.15.28:0/3815531510 learned_addr learned my addr 172.21.15.28:0/3815531510 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efecd400700 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d464c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efecca00700 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d469e60 0x55ec2d46a240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 -- 172.21.15.28:0/3815531510 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 msgr2=0x55ec2d464c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d464c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 -- 172.21.15.28:0/3815531510 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d469e60 msgr2=0x55ec2d46a240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d469e60 0x55ec2d46a240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 -- 172.21.15.28:0/3815531510 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ec2d37b3d0 con 0x55ec2d46a780 2024-05-20T20:04:17.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efecd400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efeb00b2880 0x7efeb00b4d40 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7efec0009c90 tx=0x7efec0004320 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.448+0000 7efec7e00700 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d464210 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7efeb4015000 tx=0x7efeb4009f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=745a96457d16c02f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.449+0000 7efeba000700 1 -- 172.21.15.28:0/3815531510 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efeb4003a60 con 0x55ec2d46a780 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.449+0000 7efeba000700 1 -- 172.21.15.28:0/3815531510 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efeb4026070 con 0x55ec2d46a780 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.449+0000 7efeba000700 1 -- 172.21.15.28:0/3815531510 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efeb400b7d0 con 0x55ec2d46a780 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 -- 172.21.15.28:0/3815531510 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 msgr2=0x55ec2d464210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d464210 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7efeb4015000 tx=0x7efeb4009f90 comp rx=0 tx=0).stop 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 -- 172.21.15.28:0/3815531510 shutdown_connections 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d464c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d469e60 0x55ec2d46a240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3815531510 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d464210 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 -- 172.21.15.28:0/3815531510 >> 172.21.15.28:0/3815531510 conn(0x55ec2d392b90 msgr2=0x55ec2d45efc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:17.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 -- 172.21.15.28:0/3815531510 shutdown_connections 2024-05-20T20:04:17.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 -- 172.21.15.28:0/3815531510 wait complete. 2024-05-20T20:04:17.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.450+0000 7efed9db4b80 1 -- start start 2024-05-20T20:04:17.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 1 Processor -- start 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d470410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d469e60 0x55ec2d46d7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d46a780 0x55ec2d46dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d2b8f50 con 0x55ec2d46a780 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d353710 con 0x55ec2d464830 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d471190 con 0x55ec2d469e60 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efed9db4b80 10 client.0 _ll_register_callbacks cb 0x55ec2d35cc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:17.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efecca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d470410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efec7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d469e60 0x55ec2d46d7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.451+0000 7efecca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d470410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43196/0 (socket says 172.21.15.28:43196) 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecca00700 1 -- 172.21.15.28:0/913152386 learned_addr learned my addr 172.21.15.28:0/913152386 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efec7e00700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d469e60 0x55ec2d46d7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60030/0 (socket says 172.21.15.28:60030) 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecd400700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d46a780 0x55ec2d46dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecca00700 1 -- 172.21.15.28:0/913152386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d469e60 msgr2=0x55ec2d46d7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecca00700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d469e60 0x55ec2d46d7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecca00700 1 -- 172.21.15.28:0/913152386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d46a780 msgr2=0x55ec2d46dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecca00700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d46a780 0x55ec2d46dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecca00700 1 -- 172.21.15.28:0/913152386 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ec2d302880 con 0x55ec2d464830 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efec7e00700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d469e60 0x55ec2d46d7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efecca00700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d470410 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7efebc0a94a0 tx=0x7efebc0a91d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a66a2bd0ef1f554e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:17.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.452+0000 7efeaea00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efeaea00700 1 -- 172.21.15.28:0/913152386 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efebc0c3060 con 0x55ec2d464830 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efeaea00700 1 -- 172.21.15.28:0/913152386 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7efebc004ce0 con 0x55ec2d464830 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efeb4026070 con 0x55ec2d464830 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efeaea00700 1 -- 172.21.15.28:0/913152386 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7efebc0ad910 con 0x55ec2d464830 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efed9db4b80 10 client.24905 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55ec2d383010 con 0x55ec2d464830 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efeaea00700 1 -- 172.21.15.28:0/913152386 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efeb4026070 con 0x55ec2d464830 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efeaea00700 1 -- 172.21.15.28:0/913152386 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7efebc0a4ed0 con 0x55ec2d464830 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efed9db4b80 10 client.24905 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efed9db4b80 10 client.24905 resolve_mds: resolved name 'e' to daemon mds.e 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.453+0000 7efed9db4b80 20 client.24905 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.454+0000 7efed9db4b80 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d47dbe0 0x55ec2d47f050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.454+0000 7efed9db4b80 4 client.24905 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:04:17.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.454+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ec2d3c4d40 con 0x55ec2d47dbe0 2024-05-20T20:04:17.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.454+0000 7efec7e00700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d47dbe0 0x55ec2d47f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.454+0000 7efec7e00700 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d47dbe0 0x55ec2d47f050 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=4e8eb38cbb22234a server_cookie=a8d6bc62307b54b4 in_seq=0 out_seq=0 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.454+0000 7efeaea00700 10 client.24905 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.455+0000 7efeaea00700 1 -- 172.21.15.28:0/913152386 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55ec2d3c4d40 con 0x55ec2d47dbe0 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.455+0000 7efeaea00700 10 client.24905 handle_command_reply: tid=0 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.455+0000 7efed9db4b80 1 client.24905 shutdown 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 msgr2=0x55ec2d470410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d470410 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7efebc0a94a0 tx=0x7efebc0a91d0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 shutdown_connections 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d47dbe0 0x55ec2d47f050 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d46a780 0x55ec2d46dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d464830 0x55ec2d470410 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 --2- 172.21.15.28:0/913152386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d469e60 0x55ec2d46d7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efeaea00700 0 client.24905 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 >> 172.21.15.28:0/913152386 conn(0x55ec2d392b90 msgr2=0x55ec2d463200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 shutdown_connections 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 1 -- 172.21.15.28:0/913152386 wait complete. 2024-05-20T20:04:17.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.456+0000 7efed9db4b80 20 client.24905 trim_cache size 0 max 16384 2024-05-20T20:04:17.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 Processor -- start 2024-05-20T20:04:17.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 -- start start 2024-05-20T20:04:17.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d464830 0x55ec2d464210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d469e60 0x55ec2d46dca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d45e990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d47d390 con 0x55ec2d464830 2024-05-20T20:04:17.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d47da60 con 0x55ec2d469e60 2024-05-20T20:04:17.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.480+0000 7efed9db4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d47e130 con 0x55ec2d46a780 2024-05-20T20:04:17.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d469e60 0x55ec2d46dca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d469e60 0x55ec2d46dca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43200/0 (socket says 172.21.15.28:43200) 2024-05-20T20:04:17.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 -- 172.21.15.28:0/3274987318 learned_addr learned my addr 172.21.15.28:0/3274987318 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:17.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efecd400700 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d45e990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efecca00700 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d464830 0x55ec2d464210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 -- 172.21.15.28:0/3274987318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 msgr2=0x55ec2d45e990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d45e990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 -- 172.21.15.28:0/3274987318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d464830 msgr2=0x55ec2d464210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d464830 0x55ec2d464210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.481+0000 7efec7e00700 1 -- 172.21.15.28:0/3274987318 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ec2d47a430 con 0x55ec2d469e60 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.482+0000 7efec7e00700 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d469e60 0x55ec2d46dca0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7efeb400b580 tx=0x7efeb4009f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ee0c91153e1785 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.482+0000 7efeb9600700 1 -- 172.21.15.28:0/3274987318 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efeb400b9d0 con 0x55ec2d469e60 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.482+0000 7efeb9600700 1 -- 172.21.15.28:0/3274987318 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efeb4030070 con 0x55ec2d469e60 2024-05-20T20:04:17.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.482+0000 7efeb9600700 1 -- 172.21.15.28:0/3274987318 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efeb4032710 con 0x55ec2d469e60 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.482+0000 7efed9db4b80 1 -- 172.21.15.28:0/3274987318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d469e60 msgr2=0x55ec2d46dca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.482+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d469e60 0x55ec2d46dca0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7efeb400b580 tx=0x7efeb4009f90 comp rx=0 tx=0).stop 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 -- 172.21.15.28:0/3274987318 shutdown_connections 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d46a780 0x55ec2d45e990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d464830 0x55ec2d464210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 --2- 172.21.15.28:0/3274987318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d469e60 0x55ec2d46dca0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 -- 172.21.15.28:0/3274987318 >> 172.21.15.28:0/3274987318 conn(0x55ec2d392b90 msgr2=0x55ec2d46d590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:17.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 -- 172.21.15.28:0/3274987318 shutdown_connections 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 -- 172.21.15.28:0/3274987318 wait complete. 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.483+0000 7efed9db4b80 1 -- start start 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 1 Processor -- start 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d464830 0x55ec2d4884d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d46a780 0x55ec2d488a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d483480 0x55ec2d483900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d47eed0 con 0x55ec2d483480 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d47e800 con 0x55ec2d46a780 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ec2d47f5a0 con 0x55ec2d464830 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efed9db4b80 10 client.0 _ll_register_callbacks cb 0x55ec2d35cc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efecd400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d483480 0x55ec2d483900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efecd400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d483480 0x55ec2d483900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54032/0 (socket says 172.21.15.28:54032) 2024-05-20T20:04:17.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efecd400700 1 -- 172.21.15.28:0/2198085315 learned_addr learned my addr 172.21.15.28:0/2198085315 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.484+0000 7efec7e00700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d46a780 0x55ec2d488a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efecca00700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d464830 0x55ec2d4884d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efecd400700 1 -- 172.21.15.28:0/2198085315 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d464830 msgr2=0x55ec2d4884d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efecd400700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d464830 0x55ec2d4884d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efecd400700 1 -- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d46a780 msgr2=0x55ec2d488a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efecd400700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d46a780 0x55ec2d488a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efecd400700 1 -- 172.21.15.28:0/2198085315 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ec2d483e40 con 0x55ec2d483480 2024-05-20T20:04:17.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efec7e00700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d46a780 0x55ec2d488a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:17.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efecd400700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d483480 0x55ec2d483900 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7efec001a6f0 tx=0x7efec0004b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ba2783c007da21a3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:17.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.485+0000 7efeaea00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:04:17.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.486+0000 7efeaea00700 1 -- 172.21.15.28:0/2198085315 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efec0008870 con 0x55ec2d483480 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.486+0000 7efeaea00700 1 -- 172.21.15.28:0/2198085315 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7efec0008a10 con 0x55ec2d483480 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.486+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efeb4030070 con 0x55ec2d483480 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.486+0000 7efeaea00700 1 -- 172.21.15.28:0/2198085315 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7efec0008c10 con 0x55ec2d483480 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.486+0000 7efed9db4b80 10 client.14775 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.486+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55ec2d4e9f50 con 0x55ec2d483480 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.486+0000 7efeaea00700 1 -- 172.21.15.28:0/2198085315 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efeb4030070 con 0x55ec2d483480 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efeaea00700 1 -- 172.21.15.28:0/2198085315 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7efec0037020 con 0x55ec2d483480 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efed9db4b80 10 client.14775 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efed9db4b80 10 client.14775 resolve_mds: resolved name 'e' to daemon mds.e 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efed9db4b80 20 client.14775 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efed9db4b80 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d4eeb80 0x55ec2d4ed890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:17.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efed9db4b80 4 client.14775 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "status"}) -- 0x55ec2d3be520 con 0x55ec2d4eeb80 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.487+0000 7efecca00700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d4eeb80 0x55ec2d4ed890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efecca00700 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d4eeb80 0x55ec2d4ed890 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=cd6416b3968fcd8e server_cookie=11178ec2fedd608d in_seq=0 out_seq=0 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efeaea00700 10 client.14775 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efeaea00700 1 -- 172.21.15.28:0/2198085315 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55ec2d3be520 con 0x55ec2d4eeb80 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efeaea00700 10 client.14775 handle_command_reply: tid=0 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 client.14775 shutdown 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d483480 msgr2=0x55ec2d483900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d483480 0x55ec2d483900 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7efec001a6f0 tx=0x7efec0004b40 comp rx=0 tx=0).stop 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 shutdown_connections 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55ec2d4eeb80 0x55ec2d4ed890 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ec2d483480 0x55ec2d483900 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ec2d464830 0x55ec2d4884d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 --2- 172.21.15.28:0/2198085315 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ec2d46a780 0x55ec2d488a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 >> 172.21.15.28:0/2198085315 conn(0x55ec2d392b90 msgr2=0x55ec2d45e990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.488+0000 7efeaea00700 0 client.14775 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.489+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 shutdown_connections 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.489+0000 7efed9db4b80 1 -- 172.21.15.28:0/2198085315 wait complete. 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.489+0000 7efed9db4b80 20 client.14775 trim_cache size 0 max 16384 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:04:17.493 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 3, 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24482, 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 152.85645072400001, 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 218.90266058899999 2024-05-20T20:04:17.494 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efeb00b2880 msgr2=0x7efeb00b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efeb00b2880 0x7efeb00b4d40 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7efec0009c90 tx=0x7efec0004320 comp rx=0 tx=0).stop 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec806b220 msgr2=0x7efec819be90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec806b220 0x7efec819be90 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7efebc00d980 tx=0x7efebc00dc90 comp rx=0 tx=0).stop 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 shutdown_connections 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7efeb00b2880 0x7efeb00b4d40 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efec81095b0 0x7efec81a0760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efec806b220 0x7efec819be90 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 --2- 172.21.15.28:0/4055460505 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efec806bbe0 0x7efec819c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:17.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 >> 172.21.15.28:0/4055460505 conn(0x7efec80793f0 msgr2=0x7efec810e700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:17.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 shutdown_connections 2024-05-20T20:04:17.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:17.490+0000 7efecf400700 1 -- 172.21.15.28:0/4055460505 wait complete. 2024-05-20T20:04:18.480 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.i status 2024-05-20T20:04:18.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:18 smithi028 ceph-mon[31178]: pgmap v407: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:04:18.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:18 smithi154 ceph-mon[36314]: pgmap v407: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:04:18.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:18 smithi033 ceph-mon[36179]: pgmap v407: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:04:18.968 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:20.702 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:20 smithi028 ceph-mon[31178]: pgmap v408: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:20.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:20 smithi154 ceph-mon[36314]: pgmap v408: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:20.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:20 smithi033 ceph-mon[36179]: pgmap v408: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 -- 172.21.15.28:0/1902469852 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 msgr2=0x7f30841141d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 --2- 172.21.15.28:0/1902469852 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 0x7f30841141d0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f30700098f0 tx=0x7f3070009c00 comp rx=0 tx=0).stop 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 -- 172.21.15.28:0/1902469852 shutdown_connections 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 --2- 172.21.15.28:0/1902469852 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084114710 0x7f3084116b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 --2- 172.21.15.28:0/1902469852 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3084075770 0x7f3084075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 --2- 172.21.15.28:0/1902469852 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 0x7f30841141d0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 -- 172.21.15.28:0/1902469852 >> 172.21.15.28:0/1902469852 conn(0x7f308407a6a0 msgr2=0x7f308407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 -- 172.21.15.28:0/1902469852 shutdown_connections 2024-05-20T20:04:21.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.263+0000 7f308b400700 1 -- 172.21.15.28:0/1902469852 wait complete. 2024-05-20T20:04:21.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.264+0000 7f308b400700 1 Processor -- start 2024-05-20T20:04:21.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.264+0000 7f308b400700 1 -- start start 2024-05-20T20:04:21.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f308b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084075770 0x7f30841a0310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f308b400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 0x7f30841a0850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f308b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3084114710 0x7f30841a4be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f308b400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3084119d00 con 0x7f3084114710 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f308b400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3084119b80 con 0x7f3084076090 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f308b400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3084119e80 con 0x7f3084075770 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f3088a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084075770 0x7f30841a0310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f3088a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084075770 0x7f30841a0310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60056/0 (socket says 172.21.15.28:60056) 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f3088a00700 1 -- 172.21.15.28:0/4256969875 learned_addr learned my addr 172.21.15.28:0/4256969875 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f3089400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3084114710 0x7f30841a4be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.265+0000 7f3083e00700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 0x7f30841a0850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3088a00700 1 -- 172.21.15.28:0/4256969875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 msgr2=0x7f30841a0850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3088a00700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 0x7f30841a0850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3088a00700 1 -- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3084114710 msgr2=0x7f30841a4be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3088a00700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3084114710 0x7f30841a4be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3088a00700 1 -- 172.21.15.28:0/4256969875 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f30841a5320 con 0x7f3084075770 2024-05-20T20:04:21.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3088a00700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084075770 0x7f30841a0310 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f307800e9c0 tx=0x7f307800ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:21.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3081600700 1 -- 172.21.15.28:0/4256969875 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3078004a30 con 0x7f3084075770 2024-05-20T20:04:21.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3081600700 1 -- 172.21.15.28:0/4256969875 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f307800cc70 con 0x7f3084075770 2024-05-20T20:04:21.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f30841a5610 con 0x7f3084075770 2024-05-20T20:04:21.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f3081600700 1 -- 172.21.15.28:0/4256969875 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3078005580 con 0x7f3084075770 2024-05-20T20:04:21.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.266+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f30841ace90 con 0x7f3084075770 2024-05-20T20:04:21.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.271+0000 7f3081600700 1 -- 172.21.15.28:0/4256969875 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f307800c7c0 con 0x7f3084075770 2024-05-20T20:04:21.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.271+0000 7f3095e8db80 1 Processor -- start 2024-05-20T20:04:21.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.271+0000 7f3095e8db80 1 -- start start 2024-05-20T20:04:21.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0897240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0891210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0891c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3095e8db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555df07efdf0 con 0x555df0896e60 2024-05-20T20:04:21.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3095e8db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555df07df990 con 0x555df0897780 2024-05-20T20:04:21.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3095e8db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555df07e39d0 con 0x555df0891830 2024-05-20T20:04:21.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3089400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0897240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3088a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0891210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3089400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0897240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54060/0 (socket says 172.21.15.28:54060) 2024-05-20T20:04:21.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3088a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0891210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43228/0 (socket says 172.21.15.28:43228) 2024-05-20T20:04:21.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3089400700 1 -- 172.21.15.28:0/3480622645 learned_addr learned my addr 172.21.15.28:0/3480622645 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:21.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.272+0000 7f3083e00700 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0891c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3083e00700 1 -- 172.21.15.28:0/3480622645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 msgr2=0x555df0891210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3083e00700 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0891210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3083e00700 1 -- 172.21.15.28:0/3480622645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 msgr2=0x555df0897240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3083e00700 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0897240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3083e00700 1 -- 172.21.15.28:0/3480622645 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555df07a83d0 con 0x555df0891830 2024-05-20T20:04:21.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3089400700 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0897240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:21.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3081600700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f306c0b2880 0x7f306c0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3083e00700 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0891c90 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f30700098c0 tx=0x7f3070005dc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=955e9e821878bc56 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:21.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.273+0000 7f3081600700 1 -- 172.21.15.28:0/4256969875 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f30780e2390 con 0x7f3084075770 2024-05-20T20:04:21.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.274+0000 7f3089400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f306c0b2880 0x7f306c0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.274+0000 7f3076000700 1 -- 172.21.15.28:0/3480622645 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f307000fe40 con 0x555df0891830 2024-05-20T20:04:21.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.274+0000 7f3076000700 1 -- 172.21.15.28:0/3480622645 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f307000f910 con 0x555df0891830 2024-05-20T20:04:21.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.274+0000 7f3076000700 1 -- 172.21.15.28:0/3480622645 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f30700038e0 con 0x555df0891830 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3089400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f306c0b2880 0x7f306c0b4d40 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f307c007610 tx=0x7f307c00b040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 -- 172.21.15.28:0/3480622645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 msgr2=0x555df0891c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0891c90 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f30700098c0 tx=0x7f3070005dc0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 -- 172.21.15.28:0/3480622645 shutdown_connections 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0891c90 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0897240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3480622645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0891210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 -- 172.21.15.28:0/3480622645 >> 172.21.15.28:0/3480622645 conn(0x555df07bfb90 msgr2=0x555df088bfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 -- 172.21.15.28:0/3480622645 shutdown_connections 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.275+0000 7f3095e8db80 1 -- 172.21.15.28:0/3480622645 wait complete. 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 -- start start 2024-05-20T20:04:21.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 Processor -- start 2024-05-20T20:04:21.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0891830 0x555df089d4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0896e60 0x555df089a850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0897780 0x555df089ad90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555df06e5f50 con 0x555df0891830 2024-05-20T20:04:21.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555df0780710 con 0x555df0896e60 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.276+0000 7f3095e8db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555df089e190 con 0x555df0897780 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3095e8db80 10 client.0 _ll_register_callbacks cb 0x555df0789c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0891830 0x555df089d4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0891830 0x555df089d4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54066/0 (socket says 172.21.15.28:54066) 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 -- 172.21.15.28:0/1820503048 learned_addr learned my addr 172.21.15.28:0/1820503048 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3083e00700 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0896e60 0x555df089a850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3089400700 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0897780 0x555df089ad90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 -- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0897780 msgr2=0x555df089ad90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0897780 0x555df089ad90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 -- 172.21.15.28:0/1820503048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0896e60 msgr2=0x555df089a850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0896e60 0x555df089a850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.277+0000 7f3088a00700 1 -- 172.21.15.28:0/1820503048 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555df072f880 con 0x555df0891830 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f3088a00700 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0891830 0x555df089d4b0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f307800c5b0 tx=0x7f30780a5830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c825dc293ab5945 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f306aa00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:04:21.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f306aa00700 1 -- 172.21.15.28:0/1820503048 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f30780a92c0 con 0x555df0891830 2024-05-20T20:04:21.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f306aa00700 1 -- 172.21.15.28:0/1820503048 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f30780a9bd0 con 0x555df0891830 2024-05-20T20:04:21.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f307000f910 con 0x555df0891830 2024-05-20T20:04:21.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f306aa00700 1 -- 172.21.15.28:0/1820503048 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f30780a5100 con 0x555df0891830 2024-05-20T20:04:21.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f3095e8db80 10 client.14787 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:21.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.278+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x555df07b0010 con 0x555df0891830 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f306aa00700 1 -- 172.21.15.28:0/1820503048 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f307000f910 con 0x555df0891830 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f306aa00700 1 -- 172.21.15.28:0/1820503048 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f30780b2340 con 0x555df0891830 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f3095e8db80 10 client.14787 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f3095e8db80 10 client.14787 resolve_mds: resolved name 'i' to daemon mds.i 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f3095e8db80 20 client.14787 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f3095e8db80 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df08aad10 0x555df08ac180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f3095e8db80 4 client.14787 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.279+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555df07f1d40 con 0x555df08aad10 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.280+0000 7f3083e00700 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df08aad10 0x555df08ac180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.280+0000 7f3083e00700 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df08aad10 0x555df08ac180 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=134653678f0267d7 server_cookie=6e7b4d0e88146b43 in_seq=0 out_seq=0 2024-05-20T20:04:21.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.280+0000 7f306aa00700 10 client.14787 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:04:21.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.281+0000 7f306aa00700 1 -- 172.21.15.28:0/1820503048 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x555df07f1d40 con 0x555df08aad10 2024-05-20T20:04:21.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.281+0000 7f306aa00700 10 client.14787 handle_command_reply: tid=0 2024-05-20T20:04:21.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 client.14787 shutdown 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0891830 msgr2=0x555df089d4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0891830 0x555df089d4b0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f307800c5b0 tx=0x7f30780a5830 comp rx=0 tx=0).stop 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 shutdown_connections 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0897780 0x555df089ad90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df08aad10 0x555df08ac180 crc :-1 s=READY pgs=15 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0891830 0x555df089d4b0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 --2- 172.21.15.28:0/1820503048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0896e60 0x555df089a850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 >> 172.21.15.28:0/1820503048 conn(0x555df07bfb90 msgr2=0x555df0890200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f306aa00700 0 client.14787 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 shutdown_connections 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 1 -- 172.21.15.28:0/1820503048 wait complete. 2024-05-20T20:04:21.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.282+0000 7f3095e8db80 20 client.14787 trim_cache size 0 max 16384 2024-05-20T20:04:21.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.306+0000 7f3095e8db80 1 Processor -- start 2024-05-20T20:04:21.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.306+0000 7f3095e8db80 1 -- start start 2024-05-20T20:04:21.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0891830 0x555df08aa640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0896e60 0x555df0891210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0897780 0x555df0894970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3095e8db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555df08aa4c0 con 0x555df0897780 2024-05-20T20:04:21.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3095e8db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555df08aab90 con 0x555df0891830 2024-05-20T20:04:21.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3095e8db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555df08ab260 con 0x555df0896e60 2024-05-20T20:04:21.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3083e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0896e60 0x555df0891210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3088a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0891830 0x555df08aa640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3089400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0897780 0x555df0894970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3083e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0896e60 0x555df0891210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60088/0 (socket says 172.21.15.28:60088) 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3083e00700 1 -- 172.21.15.28:0/3451968540 learned_addr learned my addr 172.21.15.28:0/3451968540 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3088a00700 1 -- 172.21.15.28:0/3451968540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0896e60 msgr2=0x555df0891210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3088a00700 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0896e60 0x555df0891210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3088a00700 1 -- 172.21.15.28:0/3451968540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0897780 msgr2=0x555df0894970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3088a00700 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0897780 0x555df0894970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3088a00700 1 -- 172.21.15.28:0/3451968540 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555df089d0f0 con 0x555df0891830 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.307+0000 7f3089400700 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0897780 0x555df0894970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.308+0000 7f3088a00700 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0891830 0x555df08aa640 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f30780e5b70 tx=0x7f3078009f00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=201357fe2b9f0d9f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.308+0000 7f3075600700 1 -- 172.21.15.28:0/3451968540 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f30780a90e0 con 0x555df0891830 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.308+0000 7f3075600700 1 -- 172.21.15.28:0/3451968540 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f307800cab0 con 0x555df0891830 2024-05-20T20:04:21.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.308+0000 7f3075600700 1 -- 172.21.15.28:0/3451968540 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f307800ce20 con 0x555df0891830 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 -- 172.21.15.28:0/3451968540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0891830 msgr2=0x555df08aa640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0891830 0x555df08aa640 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f30780e5b70 tx=0x7f3078009f00 comp rx=0 tx=0).stop 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 -- 172.21.15.28:0/3451968540 shutdown_connections 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0897780 0x555df0894970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0891830 0x555df08aa640 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3451968540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0896e60 0x555df0891210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 -- 172.21.15.28:0/3451968540 >> 172.21.15.28:0/3451968540 conn(0x555df07bfb90 msgr2=0x555df0885e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 -- 172.21.15.28:0/3451968540 shutdown_connections 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 -- 172.21.15.28:0/3451968540 wait complete. 2024-05-20T20:04:21.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.309+0000 7f3095e8db80 1 -- start start 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 1 Processor -- start 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0890400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0890940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0890e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555df08ac000 con 0x555df0896e60 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555df08ab930 con 0x555df0897780 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555df08ac6d0 con 0x555df0891830 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3095e8db80 10 client.0 _ll_register_callbacks cb 0x555df0789c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.310+0000 7f3083e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0890940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0890940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54088/0 (socket says 172.21.15.28:54088) 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 -- 172.21.15.28:0/3831397074 learned_addr learned my addr 172.21.15.28:0/3831397074 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3089400700 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0890e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3088a00700 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0890400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 -- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 msgr2=0x555df0890400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0890400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 -- 172.21.15.28:0/3831397074 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 msgr2=0x555df0890e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:21.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0890e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 -- 172.21.15.28:0/3831397074 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555df08b3a20 con 0x555df0896e60 2024-05-20T20:04:21.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f3083e00700 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0890940 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f307000a010 tx=0x7f30700055f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd87cadb861269ac server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:21.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.311+0000 7f306aa00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:04:21.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f306aa00700 1 -- 172.21.15.28:0/3831397074 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f307000f460 con 0x555df0896e60 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x555df07eb520 con 0x555df0896e60 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f306aa00700 1 -- 172.21.15.28:0/3831397074 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f307000f600 con 0x555df0896e60 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f306aa00700 1 -- 172.21.15.28:0/3831397074 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f307000f800 con 0x555df0896e60 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f3095e8db80 10 client.14793 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x555df0916720 con 0x555df0896e60 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f306aa00700 1 -- 172.21.15.28:0/3831397074 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x555df07eb520 con 0x555df0896e60 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f306aa00700 1 -- 172.21.15.28:0/3831397074 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f3070035330 con 0x555df0896e60 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.312+0000 7f3095e8db80 10 client.14793 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.313+0000 7f3095e8db80 10 client.14793 resolve_mds: resolved name 'i' to daemon mds.i 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.313+0000 7f3095e8db80 20 client.14793 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.313+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df091ae90 0x555df0919ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.313+0000 7f3095e8db80 4 client.14793 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.313+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "status"}) -- 0x555df0662200 con 0x555df091ae90 2024-05-20T20:04:21.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.313+0000 7f3088a00700 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df091ae90 0x555df0919ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3088a00700 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df091ae90 0x555df0919ba0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=30bd832f10e9adf server_cookie=fdd1b423add054e6 in_seq=0 out_seq=0 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f306aa00700 10 client.14793 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f306aa00700 1 -- 172.21.15.28:0/3831397074 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x555df0662200 con 0x555df091ae90 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f306aa00700 10 client.14793 handle_command_reply: tid=0 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 client.14793 shutdown 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 msgr2=0x555df0890940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0890940 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f307000a010 tx=0x7f30700055f0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 shutdown_connections 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555df0897780 0x555df0890e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555df0891830 0x555df0890400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x555df091ae90 0x555df0919ba0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 --2- 172.21.15.28:0/3831397074 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555df0896e60 0x555df0890940 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 >> 172.21.15.28:0/3831397074 conn(0x555df07bfb90 msgr2=0x555df0894970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f306aa00700 0 client.14793 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 shutdown_connections 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.314+0000 7f3095e8db80 1 -- 172.21.15.28:0/3831397074 wait complete. 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f3095e8db80 20 client.14793 trim_cache size 0 max 16384 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 2, 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24493, 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:04:21.319 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:04:21.320 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:04:21.320 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 157.69407515699999, 2024-05-20T20:04:21.320 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:04:21.320 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:04:21.320 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:04:21.320 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 211.00489189800001 2024-05-20T20:04:21.320 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f306c0b2880 msgr2=0x7f306c0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f306c0b2880 0x7f306c0b4d40 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f307c007610 tx=0x7f307c00b040 comp rx=0 tx=0).stop 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084075770 msgr2=0x7f30841a0310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084075770 0x7f30841a0310 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f307800e9c0 tx=0x7f307800ecd0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 shutdown_connections 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f306c0b2880 0x7f306c0b4d40 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3084114710 0x7f30841a4be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3084075770 0x7f30841a0310 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 --2- 172.21.15.28:0/4256969875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3084076090 0x7f30841a0850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.315+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 >> 172.21.15.28:0/4256969875 conn(0x7f308407a6a0 msgr2=0x7f30841115b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:21.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.316+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 shutdown_connections 2024-05-20T20:04:21.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:21.316+0000 7f308b400700 1 -- 172.21.15.28:0/4256969875 wait complete. 2024-05-20T20:04:22.119 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.h status 2024-05-20T20:04:22.591 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:22 smithi154 ceph-mon[36314]: pgmap v409: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:22.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:22 smithi033 ceph-mon[36179]: pgmap v409: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:22.944 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:22 smithi028 ceph-mon[31178]: pgmap v409: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:24.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 -- 172.21.15.28:0/3430336049 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368076090 msgr2=0x7f136810fa60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 --2- 172.21.15.28:0/3430336049 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368076090 0x7f136810fa60 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f13500098f0 tx=0x7f1350009c00 comp rx=0 tx=0).stop 2024-05-20T20:04:24.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 -- 172.21.15.28:0/3430336049 shutdown_connections 2024-05-20T20:04:24.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 --2- 172.21.15.28:0/3430336049 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f136810ffa0 0x7f1368112390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 --2- 172.21.15.28:0/3430336049 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1368075770 0x7f1368075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 --2- 172.21.15.28:0/3430336049 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368076090 0x7f136810fa60 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 -- 172.21.15.28:0/3430336049 >> 172.21.15.28:0/3430336049 conn(0x7f136807a6a0 msgr2=0x7f136807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:24.822 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:24 smithi028 ceph-mon[31178]: pgmap v410: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:24.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 -- 172.21.15.28:0/3430336049 shutdown_connections 2024-05-20T20:04:24.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.820+0000 7f136e000700 1 -- 172.21.15.28:0/3430336049 wait complete. 2024-05-20T20:04:24.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.821+0000 7f136e000700 1 Processor -- start 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.821+0000 7f136e000700 1 -- start start 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f136e000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368075770 0x7f136819be50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f136e000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1368076090 0x7f136819c390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f136e000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f136810ffa0 0x7f13681a06d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f136e000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1368115940 con 0x7f1368076090 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f136e000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f13681157c0 con 0x7f1368075770 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f136e000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1368115ac0 con 0x7f136810ffa0 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f1367400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368075770 0x7f136819be50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f135ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1368076090 0x7f136819c390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f1367400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368075770 0x7f136819be50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43282/0 (socket says 172.21.15.28:43282) 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f1367400700 1 -- 172.21.15.28:0/3214868308 learned_addr learned my addr 172.21.15.28:0/3214868308 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:24.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.822+0000 7f1367e00700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f136810ffa0 0x7f13681a06d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1367400700 1 -- 172.21.15.28:0/3214868308 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f136810ffa0 msgr2=0x7f13681a06d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1367400700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f136810ffa0 0x7f13681a06d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1367400700 1 -- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1368076090 msgr2=0x7f136819c390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1367400700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1368076090 0x7f136819c390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1367400700 1 -- 172.21.15.28:0/3214868308 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f13681a0e10 con 0x7f1368075770 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f135ea00700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1368076090 0x7f136819c390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1367400700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368075770 0x7f136819be50 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f135800e9c0 tx=0x7f135800ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1364c00700 1 -- 172.21.15.28:0/3214868308 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1358004a30 con 0x7f1368075770 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f1364c00700 1 -- 172.21.15.28:0/3214868308 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f135800cc70 con 0x7f1368075770 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.823+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f13681a1100 con 0x7f1368075770 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.824+0000 7f1364c00700 1 -- 172.21.15.28:0/3214868308 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1358005490 con 0x7f1368075770 2024-05-20T20:04:24.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.824+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f13681a8980 con 0x7f1368075770 2024-05-20T20:04:24.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.827+0000 7f1364c00700 1 -- 172.21.15.28:0/3214868308 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f1358004bd0 con 0x7f1368075770 2024-05-20T20:04:24.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.829+0000 7f137896cb80 1 Processor -- start 2024-05-20T20:04:24.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f137896cb80 1 -- start start 2024-05-20T20:04:24.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f1364c00700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f134c0b2900 0x7f134c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f137896cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a4ff09a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f137896cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4feaad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f137896cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a4feb570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f137896cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5566a4f48df0 con 0x5566a4ff0ee0 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f137896cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5566a4f38990 con 0x5566a4feb0f0 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f137896cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5566a4f3c9d0 con 0x5566a4ff05c0 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f1364c00700 1 -- 172.21.15.28:0/3214868308 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f13580e22d0 con 0x7f1368075770 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f1367e00700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f134c0b2900 0x7f134c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.830+0000 7f135ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4feaad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a4feb570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a4ff09a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a4feb570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43292/0 (socket says 172.21.15.28:43292) 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 -- 172.21.15.28:0/3835694518 learned_addr learned my addr 172.21.15.28:0/3835694518 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 -- 172.21.15.28:0/3835694518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 msgr2=0x5566a4ff09a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a4ff09a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 -- 172.21.15.28:0/3835694518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 msgr2=0x5566a4feaad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4feaad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367e00700 1 -- 172.21.15.28:0/3835694518 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5566a4f013d0 con 0x5566a4feb0f0 2024-05-20T20:04:24.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f135ea00700 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4feaad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:24.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.831+0000 7f1367400700 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a4ff09a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:24.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.832+0000 7f1367e00700 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a4feb570 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f1360007d30 tx=0x7f1360019650 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2369e492ec93b178 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:24.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.832+0000 7f1367e00700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f134c0b2900 0x7f134c0b4dc0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f13600199a0 tx=0x7f1360012000 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:24.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.832+0000 7f135cc00700 1 -- 172.21.15.28:0/3835694518 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1360028070 con 0x5566a4feb0f0 2024-05-20T20:04:24.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.832+0000 7f135cc00700 1 -- 172.21.15.28:0/3835694518 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1360007810 con 0x5566a4feb0f0 2024-05-20T20:04:24.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.832+0000 7f135cc00700 1 -- 172.21.15.28:0/3835694518 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f136001f070 con 0x5566a4feb0f0 2024-05-20T20:04:24.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 -- 172.21.15.28:0/3835694518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 msgr2=0x5566a4feb570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a4feb570 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f1360007d30 tx=0x7f1360019650 comp rx=0 tx=0).stop 2024-05-20T20:04:24.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 -- 172.21.15.28:0/3835694518 shutdown_connections 2024-05-20T20:04:24.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a4feb570 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a4ff09a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 --2- 172.21.15.28:0/3835694518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4feaad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 -- 172.21.15.28:0/3835694518 >> 172.21.15.28:0/3835694518 conn(0x5566a4f18b90 msgr2=0x5566a4fe5640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:24.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 -- 172.21.15.28:0/3835694518 shutdown_connections 2024-05-20T20:04:24.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.833+0000 7f137896cb80 1 -- 172.21.15.28:0/3835694518 wait complete. 2024-05-20T20:04:24.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 -- start start 2024-05-20T20:04:24.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 Processor -- start 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4feb0f0 0x5566a4ff6bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4ff3f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff0ee0 0x5566a4ff44b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5566a4e3ef50 con 0x5566a4feb0f0 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5566a4ed9710 con 0x5566a4ff05c0 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.834+0000 7f137896cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5566a4ff7930 con 0x5566a4ff0ee0 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f137896cb80 10 client.0 _ll_register_callbacks cb 0x5566a4ee2c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4feb0f0 0x5566a4ff6bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f135ea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4ff3f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4feb0f0 0x5566a4ff6bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54120/0 (socket says 172.21.15.28:54120) 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f135ea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4ff3f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43306/0 (socket says 172.21.15.28:43306) 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 -- 172.21.15.28:0/1821703322 learned_addr learned my addr 172.21.15.28:0/1821703322 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367e00700 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff0ee0 0x5566a4ff44b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 -- 172.21.15.28:0/1821703322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff0ee0 msgr2=0x5566a4ff44b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff0ee0 0x5566a4ff44b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 -- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 msgr2=0x5566a4ff3f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4ff3f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f1367400700 1 -- 172.21.15.28:0/1821703322 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5566a4e88880 con 0x5566a4feb0f0 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.835+0000 7f135ea00700 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4ff3f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f1367400700 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4feb0f0 0x5566a4ff6bd0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f13580dfd90 tx=0x7f13580cc330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c2a9f9c74009de0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f1355600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f1355600700 1 -- 172.21.15.28:0/1821703322 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f135800ce20 con 0x5566a4feb0f0 2024-05-20T20:04:24.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f1355600700 1 -- 172.21.15.28:0/1821703322 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f13580b2280 con 0x5566a4feb0f0 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1360007810 con 0x5566a4feb0f0 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f1355600700 1 -- 172.21.15.28:0/1821703322 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f13580a93c0 con 0x5566a4feb0f0 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f137896cb80 10 client.14805 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.836+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x5566a4f09010 con 0x5566a4feb0f0 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f1355600700 1 -- 172.21.15.28:0/1821703322 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1360007810 con 0x5566a4feb0f0 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f1355600700 1 -- 172.21.15.28:0/1821703322 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f13580cab60 con 0x5566a4feb0f0 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f137896cb80 10 client.14805 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f137896cb80 10 client.14805 resolve_mds: resolved name 'h' to daemon mds.h 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f137896cb80 20 client.14805 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f137896cb80 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a50043f0 0x5566a5005860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f137896cb80 4 client.14805 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:04:24.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5566a4f4ad40 con 0x5566a50043f0 2024-05-20T20:04:24.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.837+0000 7f135ea00700 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a50043f0 0x5566a5005860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.838+0000 7f135ea00700 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a50043f0 0x5566a5005860 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=df8e20f9d9f8ec11 server_cookie=1f227ef226ef2b30 in_seq=0 out_seq=0 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.838+0000 7f1355600700 10 client.14805 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f1355600700 1 -- 172.21.15.28:0/1821703322 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5566a4f4ad40 con 0x5566a50043f0 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f1355600700 10 client.14805 handle_command_reply: tid=0 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 client.14805 shutdown 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4feb0f0 msgr2=0x5566a4ff6bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4feb0f0 0x5566a4ff6bd0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f13580dfd90 tx=0x7f13580cc330 comp rx=0 tx=0).stop 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 shutdown_connections 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a50043f0 0x5566a5005860 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff0ee0 0x5566a4ff44b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4feb0f0 0x5566a4ff6bd0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 --2- 172.21.15.28:0/1821703322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4ff3f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 >> 172.21.15.28:0/1821703322 conn(0x5566a4f18b90 msgr2=0x5566a4fe4d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:24.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f1355600700 0 client.14805 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:04:24.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 shutdown_connections 2024-05-20T20:04:24.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 1 -- 172.21.15.28:0/1821703322 wait complete. 2024-05-20T20:04:24.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.839+0000 7f137896cb80 20 client.14805 trim_cache size 0 max 16384 2024-05-20T20:04:24.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.863+0000 7f137896cb80 1 Processor -- start 2024-05-20T20:04:24.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.863+0000 7f137896cb80 1 -- start start 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f137896cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4feb0f0 0x5566a4feaad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f137896cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4feb4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f137896cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4fe54a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f137896cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5566a5003ba0 con 0x5566a4ff0ee0 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f137896cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5566a5004270 con 0x5566a4ff05c0 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f137896cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5566a5004940 con 0x5566a4feb0f0 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f135ea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4feb4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f1367400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4feb0f0 0x5566a4feaad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f135ea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4feb4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43316/0 (socket says 172.21.15.28:43316) 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f135ea00700 1 -- 172.21.15.28:0/16147232 learned_addr learned my addr 172.21.15.28:0/16147232 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.864+0000 7f1367e00700 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4fe54a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1367400700 1 -- 172.21.15.28:0/16147232 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 msgr2=0x5566a4feb4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1367400700 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4feb4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1367400700 1 -- 172.21.15.28:0/16147232 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 msgr2=0x5566a4fe54a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1367400700 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4fe54a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1367400700 1 -- 172.21.15.28:0/16147232 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5566a5005bd0 con 0x5566a4feb0f0 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1367400700 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4feb0f0 0x5566a4feaad0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f13580caf00 tx=0x7f1358004f70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b51eef412f627469 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1357e00700 1 -- 172.21.15.28:0/16147232 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f13580a5770 con 0x5566a4feb0f0 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1357e00700 1 -- 172.21.15.28:0/16147232 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f135800ce20 con 0x5566a4feb0f0 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.865+0000 7f1357e00700 1 -- 172.21.15.28:0/16147232 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f13580b2280 con 0x5566a4feb0f0 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 -- 172.21.15.28:0/16147232 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4feb0f0 msgr2=0x5566a4feaad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4feb0f0 0x5566a4feaad0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f13580caf00 tx=0x7f1358004f70 comp rx=0 tx=0).stop 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 -- 172.21.15.28:0/16147232 shutdown_connections 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a4fe54a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4feb0f0 0x5566a4feaad0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 --2- 172.21.15.28:0/16147232 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4ff05c0 0x5566a4feb4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 -- 172.21.15.28:0/16147232 >> 172.21.15.28:0/16147232 conn(0x5566a4f18b90 msgr2=0x5566a4ff2860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:24.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.866+0000 7f137896cb80 1 -- 172.21.15.28:0/16147232 shutdown_connections 2024-05-20T20:04:24.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.867+0000 7f137896cb80 1 -- 172.21.15.28:0/16147232 wait complete. 2024-05-20T20:04:24.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.867+0000 7f137896cb80 1 -- start start 2024-05-20T20:04:24.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.867+0000 7f137896cb80 1 Processor -- start 2024-05-20T20:04:24.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f137896cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a500ece0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f137896cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a500f220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f137896cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a5009c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f137896cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5566a50056e0 con 0x5566a4ff0ee0 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f137896cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5566a5005010 con 0x5566a4feb0f0 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f137896cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5566a5005db0 con 0x5566a4ff05c0 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f137896cb80 10 client.0 _ll_register_callbacks cb 0x5566a4ee2c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f1367400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a500ece0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f1367400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a500ece0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43324/0 (socket says 172.21.15.28:43324) 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f1367400700 1 -- 172.21.15.28:0/4137572440 learned_addr learned my addr 172.21.15.28:0/4137572440 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f135ea00700 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a500f220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.868+0000 7f1367e00700 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a5009c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f135ea00700 1 -- 172.21.15.28:0/4137572440 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 msgr2=0x5566a500ece0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f135ea00700 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a500ece0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f135ea00700 1 -- 172.21.15.28:0/4137572440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 msgr2=0x5566a5009c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f135ea00700 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a5009c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f135ea00700 1 -- 172.21.15.28:0/4137572440 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5566a500a1d0 con 0x5566a4ff05c0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f135ea00700 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a500f220 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x5566a4ff5350 tx=0x7f1350009520 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6e2aac442e7e50e7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f1355600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f1355600700 1 -- 172.21.15.28:0/4137572440 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1350005510 con 0x5566a4ff05c0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5566a4f44520 con 0x5566a4ff05c0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.869+0000 7f1355600700 1 -- 172.21.15.28:0/4137572440 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f1350003680 con 0x5566a4ff05c0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.870+0000 7f1355600700 1 -- 172.21.15.28:0/4137572440 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f13500056b0 con 0x5566a4ff05c0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.870+0000 7f137896cb80 10 client.24931 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.870+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x5566a50703b0 con 0x5566a4ff05c0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.870+0000 7f1355600700 1 -- 172.21.15.28:0/4137572440 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x5566a4f44520 con 0x5566a4ff05c0 2024-05-20T20:04:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.870+0000 7f1355600700 1 -- 172.21.15.28:0/4137572440 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f1350037050 con 0x5566a4ff05c0 2024-05-20T20:04:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.870+0000 7f137896cb80 10 client.24931 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.871+0000 7f137896cb80 10 client.24931 resolve_mds: resolved name 'h' to daemon mds.h 2024-05-20T20:04:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.871+0000 7f137896cb80 20 client.24931 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.871+0000 7f137896cb80 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a5074f10 0x5566a5073c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.871+0000 7f137896cb80 4 client.24931 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:04:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.871+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "status"}) -- 0x5566a4dbb200 con 0x5566a5074f10 2024-05-20T20:04:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.871+0000 7f1367400700 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a5074f10 0x5566a5073c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f1355600700 10 client.24931 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:04:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f1367400700 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a5074f10 0x5566a5073c20 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=683b38b0ef22a953 server_cookie=615d97ab89ac4631 in_seq=0 out_seq=0 2024-05-20T20:04:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f1355600700 1 -- 172.21.15.28:0/4137572440 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x5566a4dbb200 con 0x5566a5074f10 2024-05-20T20:04:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f1355600700 10 client.24931 handle_command_reply: tid=0 2024-05-20T20:04:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 client.24931 shutdown 2024-05-20T20:04:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 msgr2=0x5566a500f220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a500f220 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x5566a4ff5350 tx=0x7f1350009520 comp rx=0 tx=0).stop 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 shutdown_connections 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5566a5074f10 0x5566a5073c20 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566a4ff0ee0 0x5566a5009c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566a4feb0f0 0x5566a500ece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 --2- 172.21.15.28:0/4137572440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566a4ff05c0 0x5566a500f220 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f1355600700 0 client.24931 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 >> 172.21.15.28:0/4137572440 conn(0x5566a4f18b90 msgr2=0x5566a4ff27b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.872+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 shutdown_connections 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.873+0000 7f137896cb80 1 -- 172.21.15.28:0/4137572440 wait complete. 2024-05-20T20:04:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.873+0000 7f137896cb80 20 client.24931 trim_cache size 0 max 16384 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 1, 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24500, 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 162.26563258199999, 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 217.46368368399999 2024-05-20T20:04:24.876 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:04:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.873+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f134c0b2900 msgr2=0x7f134c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.873+0000 7f136e000700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f134c0b2900 0x7f134c0b4dc0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f13600199a0 tx=0x7f1360012000 comp rx=0 tx=0).stop 2024-05-20T20:04:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.873+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368075770 msgr2=0x7f136819be50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.873+0000 7f136e000700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368075770 0x7f136819be50 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f135800e9c0 tx=0x7f135800ecd0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 shutdown_connections 2024-05-20T20:04:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f134c0b2900 0x7f134c0b4dc0 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f136810ffa0 0x7f13681a06d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1368075770 0x7f136819be50 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 --2- 172.21.15.28:0/3214868308 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1368076090 0x7f136819c390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 >> 172.21.15.28:0/3214868308 conn(0x7f136807a6a0 msgr2=0x7f13681007b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 shutdown_connections 2024-05-20T20:04:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:24.874+0000 7f136e000700 1 -- 172.21.15.28:0/3214868308 wait complete. 2024-05-20T20:04:24.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:24 smithi033 ceph-mon[36179]: pgmap v410: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:24.879 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:24 smithi154 ceph-mon[36314]: pgmap v410: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:25.682 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.b status 2024-05-20T20:04:26.165 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:26.664 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:26 smithi028 ceph-mon[31178]: pgmap v411: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:26.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:26 smithi154 ceph-mon[36314]: pgmap v411: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:26.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:26 smithi033 ceph-mon[36179]: pgmap v411: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 -- 172.21.15.28:0/3494250540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76101035f0 msgr2=0x7f7610103a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 --2- 172.21.15.28:0/3494250540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76101035f0 0x7f7610103a50 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f75f8009990 tx=0x7f75f8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 -- 172.21.15.28:0/3494250540 shutdown_connections 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 --2- 172.21.15.28:0/3494250540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7610103f90 0x7f7610110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 --2- 172.21.15.28:0/3494250540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7610108500 0x7f76101088e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 --2- 172.21.15.28:0/3494250540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76101035f0 0x7f7610103a50 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 -- 172.21.15.28:0/3494250540 >> 172.21.15.28:0/3494250540 conn(0x7f761007a6a0 msgr2=0x7f761007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:28.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.314+0000 7f7615200700 1 -- 172.21.15.28:0/3494250540 shutdown_connections 2024-05-20T20:04:28.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.315+0000 7f7615200700 1 -- 172.21.15.28:0/3494250540 wait complete. 2024-05-20T20:04:28.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 Processor -- start 2024-05-20T20:04:28.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 -- start start 2024-05-20T20:04:28.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76101035f0 0x7f761019f890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7610103f90 0x7f761019fdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7610108500 0x7f7610199aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f76101137e0 con 0x7f7610108500 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7610113660 con 0x7f7610103f90 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f7615200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7610113960 con 0x7f76101035f0 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.316+0000 7f760e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7610108500 0x7f7610199aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7610108500 0x7f7610199aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34032/0 (socket says 172.21.15.28:34032) 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 -- 172.21.15.28:0/2493960572 learned_addr learned my addr 172.21.15.28:0/2493960572 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760de00700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76101035f0 0x7f761019f890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f7605400700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7610103f90 0x7f761019fdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 -- 172.21.15.28:0/2493960572 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76101035f0 msgr2=0x7f761019f890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76101035f0 0x7f761019f890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 -- 172.21.15.28:0/2493960572 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7610103f90 msgr2=0x7f761019fdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7610103f90 0x7f761019fdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 -- 172.21.15.28:0/2493960572 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f761019a200 con 0x7f7610108500 2024-05-20T20:04:28.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.317+0000 7f760e800700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7610108500 0x7f7610199aa0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f760800efc0 tx=0x7f760800c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:28.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.318+0000 7f7607400700 1 -- 172.21.15.28:0/2493960572 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76080092e0 con 0x7f7610108500 2024-05-20T20:04:28.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.318+0000 7f7607400700 1 -- 172.21.15.28:0/2493960572 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7608004bb0 con 0x7f7610108500 2024-05-20T20:04:28.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.318+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f761019a490 con 0x7f7610108500 2024-05-20T20:04:28.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.318+0000 7f7607400700 1 -- 172.21.15.28:0/2493960572 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7608004020 con 0x7f7610108500 2024-05-20T20:04:28.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.318+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f76101a6760 con 0x7f7610108500 2024-05-20T20:04:28.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.321+0000 7f7607400700 1 -- 172.21.15.28:0/2493960572 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f760801b030 con 0x7f7610108500 2024-05-20T20:04:28.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 Processor -- start 2024-05-20T20:04:28.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 -- start start 2024-05-20T20:04:28.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f7607400700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f75f40b2900 0x7f75f40b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda039a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecd9fdad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecd9fe570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eecd95bdf0 con 0x55eecd9fe0f0 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eecd94b990 con 0x55eecda03ee0 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f7607400700 1 -- 172.21.15.28:0/2493960572 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f76080e1ae0 con 0x7f7610108500 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f761fc32b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eecd94f9d0 con 0x55eecda035c0 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.323+0000 7f7605400700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f75f40b2900 0x7f75f40b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda039a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda039a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39166/0 (socket says 172.21.15.28:39166) 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f7605400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecd9fdad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 -- 172.21.15.28:0/2144977700 learned_addr learned my addr 172.21.15.28:0/2144977700 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 -- 172.21.15.28:0/2144977700 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 msgr2=0x55eecd9fdad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 --2- 172.21.15.28:0/2144977700 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecd9fdad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 -- 172.21.15.28:0/2144977700 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 msgr2=0x55eecd9fe570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 --2- 172.21.15.28:0/2144977700 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecd9fe570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 -- 172.21.15.28:0/2144977700 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55eecd9143d0 con 0x55eecda035c0 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.324+0000 7f760de00700 1 --2- 172.21.15.28:0/2144977700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda039a0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f7600009cb0 tx=0x7f7600011440 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2088683f34c703e1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.325+0000 7f7605400700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f75f40b2900 0x7f75f40b4dc0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f75f80098e0 tx=0x7f75f8005790 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.325+0000 7f75ff400700 1 -- 172.21.15.28:0/2144977700 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7600011d20 con 0x55eecda035c0 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.325+0000 7f75ff400700 1 -- 172.21.15.28:0/2144977700 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f76000044d0 con 0x55eecda035c0 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.325+0000 7f75ff400700 1 -- 172.21.15.28:0/2144977700 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7600005280 con 0x55eecda035c0 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 -- 172.21.15.28:0/2144977700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 msgr2=0x55eecda039a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2144977700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda039a0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f7600009cb0 tx=0x7f7600011440 comp rx=0 tx=0).stop 2024-05-20T20:04:28.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 -- 172.21.15.28:0/2144977700 shutdown_connections 2024-05-20T20:04:28.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2144977700 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecd9fe570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2144977700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda039a0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2144977700 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecd9fdad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 -- 172.21.15.28:0/2144977700 >> 172.21.15.28:0/2144977700 conn(0x55eecd92bb90 msgr2=0x55eecd9f8640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:28.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 -- 172.21.15.28:0/2144977700 shutdown_connections 2024-05-20T20:04:28.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.326+0000 7f761fc32b80 1 -- 172.21.15.28:0/2144977700 wait complete. 2024-05-20T20:04:28.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 -- start start 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 Processor -- start 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecd9fe0f0 0x55eecda09c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda06ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda03ee0 0x55eecda07530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eecd851f50 con 0x55eecda03ee0 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eecd8ec710 con 0x55eecd9fe0f0 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eecda0eda0 con 0x55eecda035c0 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.327+0000 7f761fc32b80 10 client.0 _ll_register_callbacks cb 0x55eecd8f5c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda03ee0 0x55eecda07530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda03ee0 0x55eecda07530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34048/0 (socket says 172.21.15.28:34048) 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecd9fe0f0 0x55eecda09c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 -- 172.21.15.28:0/1665548577 learned_addr learned my addr 172.21.15.28:0/1665548577 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f7605400700 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda06ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 -- 172.21.15.28:0/1665548577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 msgr2=0x55eecda06ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda06ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 -- 172.21.15.28:0/1665548577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecd9fe0f0 msgr2=0x55eecda09c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecd9fe0f0 0x55eecda09c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 -- 172.21.15.28:0/1665548577 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55eecd89b880 con 0x55eecda03ee0 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f760e800700 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda03ee0 0x55eecda07530 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f76080a51d0 tx=0x7f76080a9690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2c3c4f2025ecefec server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.328+0000 7f75f3e00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.329+0000 7f75f3e00700 1 -- 172.21.15.28:0/1665548577 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76080a9150 con 0x55eecda03ee0 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.329+0000 7f75f3e00700 1 -- 172.21.15.28:0/1665548577 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f76080ad170 con 0x55eecda03ee0 2024-05-20T20:04:28.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.329+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f76000044d0 con 0x55eecda03ee0 2024-05-20T20:04:28.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.329+0000 7f75f3e00700 1 -- 172.21.15.28:0/1665548577 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f76080ad500 con 0x55eecda03ee0 2024-05-20T20:04:28.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.329+0000 7f761fc32b80 10 client.14820 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:28.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.329+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55eecd91c010 con 0x55eecda03ee0 2024-05-20T20:04:28.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.329+0000 7f75f3e00700 1 -- 172.21.15.28:0/1665548577 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f76000044d0 con 0x55eecda03ee0 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f75f3e00700 1 -- 172.21.15.28:0/1665548577 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f76080a7e20 con 0x55eecda03ee0 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f761fc32b80 10 client.14820 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f761fc32b80 10 client.14820 resolve_mds: resolved name 'b' to daemon mds.b 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f761fc32b80 20 client.14820 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda1b870 0x55eecda1cce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f761fc32b80 4 client.14820 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55eecd95dd40 con 0x55eecda1b870 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.330+0000 7f760de00700 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda1b870 0x55eecda1cce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.331+0000 7f760de00700 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda1b870 0x55eecda1cce0 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4bf786cb9f886dd8 server_cookie=2f1074a50672da45 in_seq=0 out_seq=0 2024-05-20T20:04:28.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.331+0000 7f75f3e00700 10 client.14820 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f75f3e00700 1 -- 172.21.15.28:0/1665548577 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55eecd95dd40 con 0x55eecda1b870 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f75f3e00700 10 client.14820 handle_command_reply: tid=0 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 client.14820 shutdown 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda03ee0 msgr2=0x55eecda07530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda03ee0 0x55eecda07530 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f76080a51d0 tx=0x7f76080a9690 comp rx=0 tx=0).stop 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 shutdown_connections 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda1b870 0x55eecda1cce0 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda03ee0 0x55eecda07530 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecd9fe0f0 0x55eecda09c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1665548577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda035c0 0x55eecda06ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f75f3e00700 0 client.14820 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:04:28.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.332+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 >> 172.21.15.28:0/1665548577 conn(0x55eecd92bb90 msgr2=0x55eecd9f7d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:28.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.333+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 shutdown_connections 2024-05-20T20:04:28.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.333+0000 7f761fc32b80 1 -- 172.21.15.28:0/1665548577 wait complete. 2024-05-20T20:04:28.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.333+0000 7f761fc32b80 20 client.14820 trim_cache size 0 max 16384 2024-05-20T20:04:28.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.356+0000 7f761fc32b80 1 Processor -- start 2024-05-20T20:04:28.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f761fc32b80 1 -- start start 2024-05-20T20:04:28.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecda01770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda035c0 0x55eecd9f84a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda03ee0 0x55eecda26140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f761fc32b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eecda1b020 con 0x55eecd9fe0f0 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f761fc32b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eecda1b6f0 con 0x55eecda035c0 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f761fc32b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eecda1bdc0 con 0x55eecda03ee0 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f760de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecda01770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f7605400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda035c0 0x55eecd9f84a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f760e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda03ee0 0x55eecda26140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f7605400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda035c0 0x55eecd9f84a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53572/0 (socket says 172.21.15.28:53572) 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f760de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecda01770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34064/0 (socket says 172.21.15.28:34064) 2024-05-20T20:04:28.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.357+0000 7f7605400700 1 -- 172.21.15.28:0/1287439827 learned_addr learned my addr 172.21.15.28:0/1287439827 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:28.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f7605400700 1 -- 172.21.15.28:0/1287439827 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda03ee0 msgr2=0x55eecda26140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f7605400700 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda03ee0 0x55eecda26140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f7605400700 1 -- 172.21.15.28:0/1287439827 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 msgr2=0x55eecda01770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f7605400700 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecda01770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f7605400700 1 -- 172.21.15.28:0/1287439827 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55eecd9fe8e0 con 0x55eecda035c0 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f760e800700 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda03ee0 0x55eecda26140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f7605400700 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda035c0 0x55eecd9f84a0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f75f8000c00 tx=0x7f75f8005960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=776be976bdae79a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.358+0000 7f75fea00700 1 -- 172.21.15.28:0/1287439827 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f75f8038070 con 0x55eecda035c0 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f75fea00700 1 -- 172.21.15.28:0/1287439827 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f75f800f910 con 0x55eecda035c0 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f75fea00700 1 -- 172.21.15.28:0/1287439827 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f75f8033680 con 0x55eecda035c0 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 -- 172.21.15.28:0/1287439827 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda035c0 msgr2=0x55eecd9f84a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda035c0 0x55eecd9f84a0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f75f8000c00 tx=0x7f75f8005960 comp rx=0 tx=0).stop 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 -- 172.21.15.28:0/1287439827 shutdown_connections 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecda03ee0 0x55eecda26140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecd9fe0f0 0x55eecda01770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 --2- 172.21.15.28:0/1287439827 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda035c0 0x55eecd9f84a0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 -- 172.21.15.28:0/1287439827 >> 172.21.15.28:0/1287439827 conn(0x55eecd92bb90 msgr2=0x55eecda01060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:28.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 -- 172.21.15.28:0/1287439827 shutdown_connections 2024-05-20T20:04:28.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.359+0000 7f761fc32b80 1 -- 172.21.15.28:0/1287439827 wait complete. 2024-05-20T20:04:28.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.360+0000 7f761fc32b80 1 -- start start 2024-05-20T20:04:28.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.360+0000 7f761fc32b80 1 Processor -- start 2024-05-20T20:04:28.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.360+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecd9fe0f0 0x55eecda265e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda035c0 0x55eecda21110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f761fc32b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecda21650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f761fc32b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eecda1cb60 con 0x55eecda035c0 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f761fc32b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eecda1c490 con 0x55eecda03ee0 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f761fc32b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eecda1d230 con 0x55eecd9fe0f0 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f761fc32b80 10 client.0 _ll_register_callbacks cb 0x55eecd8f5c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f7605400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda035c0 0x55eecda21110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f7605400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda035c0 0x55eecda21110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34066/0 (socket says 172.21.15.28:34066) 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f7605400700 1 -- 172.21.15.28:0/2190942012 learned_addr learned my addr 172.21.15.28:0/2190942012 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f760de00700 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecd9fe0f0 0x55eecda265e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f760e800700 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecda21650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f760de00700 1 -- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 msgr2=0x55eecda21650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f760de00700 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecda21650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f760de00700 1 -- 172.21.15.28:0/2190942012 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda035c0 msgr2=0x55eecda21110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f760de00700 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda035c0 0x55eecda21110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.361+0000 7f760de00700 1 -- 172.21.15.28:0/2190942012 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55eecda21c90 con 0x55eecd9fe0f0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f760de00700 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecd9fe0f0 0x55eecda265e0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f7600009230 tx=0x7f7600004b90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=853324cd2c9fde6e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f75f3e00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f75f3e00700 1 -- 172.21.15.28:0/2190942012 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f760001d040 con 0x55eecd9fe0f0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f75f800f910 con 0x55eecd9fe0f0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f75f3e00700 1 -- 172.21.15.28:0/2190942012 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f7600004450 con 0x55eecd9fe0f0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f75f3e00700 1 -- 172.21.15.28:0/2190942012 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f7600004650 con 0x55eecd9fe0f0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f761fc32b80 10 client.24949 fetch_fsmap learned FSMap version 28 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.362+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55eecda87e50 con 0x55eecd9fe0f0 2024-05-20T20:04:28.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f75f3e00700 1 -- 172.21.15.28:0/2190942012 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f75f800f910 con 0x55eecd9fe0f0 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f75f3e00700 1 -- 172.21.15.28:0/2190942012 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f7600023040 con 0x55eecd9fe0f0 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f761fc32b80 10 client.24949 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f761fc32b80 10 client.24949 resolve_mds: resolved name 'b' to daemon mds.b 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f761fc32b80 20 client.24949 populate_metadata read hostname 'smithi028' 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda8c830 0x55eecda8b540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f761fc32b80 4 client.24949 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "status"}) -- 0x55eecd957520 con 0x55eecda8c830 2024-05-20T20:04:28.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.363+0000 7f7605400700 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda8c830 0x55eecda8b540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f7605400700 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda8c830 0x55eecda8b540 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c44b6c3b5632d3f server_cookie=f98c493fec0cbe9c in_seq=0 out_seq=0 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f75f3e00700 10 client.24949 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f75f3e00700 1 -- 172.21.15.28:0/2190942012 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x55eecd957520 con 0x55eecda8c830 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f75f3e00700 10 client.24949 handle_command_reply: tid=0 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 client.24949 shutdown 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecd9fe0f0 msgr2=0x55eecda265e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecd9fe0f0 0x55eecda265e0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f7600009230 tx=0x7f7600004b90 comp rx=0 tx=0).stop 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 shutdown_connections 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eecda8c830 0x55eecda8b540 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eecda03ee0 0x55eecda21650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eecd9fe0f0 0x55eecda265e0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 --2- 172.21.15.28:0/2190942012 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eecda035c0 0x55eecda21110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 >> 172.21.15.28:0/2190942012 conn(0x55eecd92bb90 msgr2=0x55eecda06dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f75f3e00700 0 client.24949 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:04:28.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.364+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 shutdown_connections 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f761fc32b80 1 -- 172.21.15.28:0/2190942012 wait complete. 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f761fc32b80 20 client.24949 trim_cache size 0 max 16384 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 0, 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24542, 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 181.848662983, 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:04:28.369 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:04:28.370 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:04:28.370 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 205.34123210499999 2024-05-20T20:04:28.370 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:04:28.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f75f40b2900 msgr2=0x7f75f40b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f7615200700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f75f40b2900 0x7f75f40b4dc0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f75f80098e0 tx=0x7f75f8005790 comp rx=0 tx=0).stop 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7610108500 msgr2=0x7f7610199aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f7615200700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7610108500 0x7f7610199aa0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f760800efc0 tx=0x7f760800c3a0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 shutdown_connections 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.365+0000 7f7615200700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f75f40b2900 0x7f75f40b4dc0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.366+0000 7f7615200700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7610108500 0x7f7610199aa0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.366+0000 7f7615200700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76101035f0 0x7f761019f890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.366+0000 7f7615200700 1 --2- 172.21.15.28:0/2493960572 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7610103f90 0x7f761019fdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.366+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 >> 172.21.15.28:0/2493960572 conn(0x7f761007a6a0 msgr2=0x7f76101016e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.366+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 shutdown_connections 2024-05-20T20:04:28.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:28.366+0000 7f7615200700 1 -- 172.21.15.28:0/2493960572 wait complete. 2024-05-20T20:04:28.776 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:28 smithi028 ceph-mon[31178]: pgmap v412: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:28 smithi154 ceph-mon[36314]: pgmap v412: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:28 smithi033 ceph-mon[36179]: pgmap v412: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:29.190 INFO:tasks.cephadm:Setup complete, yielding 2024-05-20T20:04:29.190 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T20:04:29.199 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:04:29.200 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch status' 2024-05-20T20:04:29.684 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:30.872 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:30 smithi028 ceph-mon[31178]: pgmap v413: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:30.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:30 smithi154 ceph-mon[36314]: pgmap v413: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:30.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:30 smithi033 ceph-mon[36179]: pgmap v413: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:31.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 -- 172.21.15.28:0/3875652234 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 msgr2=0x7f0040076420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:31.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 --2- 172.21.15.28:0/3875652234 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 0x7f0040076420 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f002c009990 tx=0x7f002c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:31.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 -- 172.21.15.28:0/3875652234 shutdown_connections 2024-05-20T20:04:31.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 --2- 172.21.15.28:0/3875652234 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f004010fbd0 0x7f0040111fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:31.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 --2- 172.21.15.28:0/3875652234 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0040077570 0x7f00400759d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:31.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 --2- 172.21.15.28:0/3875652234 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 0x7f0040076420 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:31.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 -- 172.21.15.28:0/3875652234 >> 172.21.15.28:0/3875652234 conn(0x7f00400feb90 msgr2=0x7f0040100fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:31.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 -- 172.21.15.28:0/3875652234 shutdown_connections 2024-05-20T20:04:31.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.793+0000 7f0044c00700 1 -- 172.21.15.28:0/3875652234 wait complete. 2024-05-20T20:04:31.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.794+0000 7f0044c00700 1 Processor -- start 2024-05-20T20:04:31.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.794+0000 7f0044c00700 1 -- start start 2024-05-20T20:04:31.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f0044c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 0x7f004019fd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:31.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f0044c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0040077570 0x7f00401a0290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f0044c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f004010fbd0 0x7f00401a7310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f0044c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0040115680 con 0x7f004010fbd0 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f0044c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0040115500 con 0x7f0040077570 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f0044c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0040115800 con 0x7f0040075fa0 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0040077570 0x7f00401a0290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f004010fbd0 0x7f00401a7310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0040077570 0x7f00401a0290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53610/0 (socket says 172.21.15.28:53610) 2024-05-20T20:04:31.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 -- 172.21.15.28:0/1022759847 learned_addr learned my addr 172.21.15.28:0/1022759847 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 0x7f004019fd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 -- 172.21.15.28:0/1022759847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 msgr2=0x7f004019fd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 0x7f004019fd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 -- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f004010fbd0 msgr2=0x7f00401a7310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f004010fbd0 0x7f00401a7310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.795+0000 7f003d400700 1 -- 172.21.15.28:0/1022759847 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f00401a79b0 con 0x7f0040077570 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.796+0000 7f003d400700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0040077570 0x7f00401a0290 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f002c006010 tx=0x7f002c0057c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.796+0000 7f0036a00700 1 -- 172.21.15.28:0/1022759847 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f002c027070 con 0x7f0040077570 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.796+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f00401a7c40 con 0x7f0040077570 2024-05-20T20:04:31.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.796+0000 7f0036a00700 1 -- 172.21.15.28:0/1022759847 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f002c022470 con 0x7f0040077570 2024-05-20T20:04:31.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.796+0000 7f0036a00700 1 -- 172.21.15.28:0/1022759847 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f002c017650 con 0x7f0040077570 2024-05-20T20:04:31.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.796+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f00401a80f0 con 0x7f0040077570 2024-05-20T20:04:31.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.797+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f004006a230 con 0x7f0040077570 2024-05-20T20:04:31.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.800+0000 7f0036a00700 1 -- 172.21.15.28:0/1022759847 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f002c02a070 con 0x7f0040077570 2024-05-20T20:04:31.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.801+0000 7f0036a00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f00240b2880 0x7f00240b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:31.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.801+0000 7f0036a00700 1 -- 172.21.15.28:0/1022759847 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f002c0e3050 con 0x7f0040077570 2024-05-20T20:04:31.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.802+0000 7f003de00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f00240b2880 0x7f00240b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:31.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.802+0000 7f003de00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f00240b2880 0x7f00240b4d40 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f0040077950 tx=0x7f0030009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:31.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:31.806+0000 7f0036a00700 1 -- 172.21.15.28:0/1022759847 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f002c013070 con 0x7f0040077570 2024-05-20T20:04:32.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.065+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch status", "target": ["mon-mgr", ""]}) -- 0x7f0040063a40 con 0x7f00240b2880 2024-05-20T20:04:32.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.068+0000 7f0036a00700 1 -- 172.21.15.28:0/1022759847 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+43 (secure 0 0 0) 0x7f0040063a40 con 0x7f00240b2880 2024-05-20T20:04:32.069 INFO:teuthology.orchestra.run.smithi028.stdout:Backend: cephadm 2024-05-20T20:04:32.069 INFO:teuthology.orchestra.run.smithi028.stdout:Available: Yes 2024-05-20T20:04:32.069 INFO:teuthology.orchestra.run.smithi028.stdout:Paused: No 2024-05-20T20:04:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f00240b2880 msgr2=0x7f00240b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f00240b2880 0x7f00240b4d40 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f0040077950 tx=0x7f0030009260 comp rx=0 tx=0).stop 2024-05-20T20:04:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0040077570 msgr2=0x7f00401a0290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0040077570 0x7f00401a0290 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f002c006010 tx=0x7f002c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:04:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 shutdown_connections 2024-05-20T20:04:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f00240b2880 0x7f00240b4d40 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f004010fbd0 0x7f00401a7310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0040075fa0 0x7f004019fd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 --2- 172.21.15.28:0/1022759847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0040077570 0x7f00401a0290 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 >> 172.21.15.28:0/1022759847 conn(0x7f00400feb90 msgr2=0x7f0040102f70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 shutdown_connections 2024-05-20T20:04:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:32.071+0000 7f0044c00700 1 -- 172.21.15.28:0/1022759847 wait complete. 2024-05-20T20:04:32.776 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:32 smithi028 ceph-mon[31178]: pgmap v414: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:32.825 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch ps' 2024-05-20T20:04:32.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:32 smithi154 ceph-mon[36314]: pgmap v414: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:32.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:32 smithi033 ceph-mon[36179]: pgmap v414: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:33.285 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:33.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:33 smithi028 ceph-mon[31178]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:33.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:33 smithi154 ceph-mon[36314]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:33.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:33 smithi033 ceph-mon[36179]: from='client.24971 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:34.701 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:34 smithi028 ceph-mon[31178]: pgmap v415: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:34.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:34 smithi154 ceph-mon[36314]: pgmap v415: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:34.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:34 smithi033 ceph-mon[36179]: pgmap v415: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:35.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 -- 172.21.15.28:0/2885471737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f650410b480 msgr2=0x7f650410d910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:35.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 --2- 172.21.15.28:0/2885471737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f650410b480 0x7f650410d910 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f64fc00b0a0 tx=0x7f64fc00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 -- 172.21.15.28:0/2885471737 shutdown_connections 2024-05-20T20:04:35.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 --2- 172.21.15.28:0/2885471737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f650410b480 0x7f650410d910 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 --2- 172.21.15.28:0/2885471737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6504077570 0x7f65040759d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 --2- 172.21.15.28:0/2885471737 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504075fa0 0x7f6504076420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 -- 172.21.15.28:0/2885471737 >> 172.21.15.28:0/2885471737 conn(0x7f65040feb10 msgr2=0x7f6504100f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:35.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 -- 172.21.15.28:0/2885471737 shutdown_connections 2024-05-20T20:04:35.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.574+0000 7f6508a00700 1 -- 172.21.15.28:0/2885471737 wait complete. 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 Processor -- start 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 -- start start 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6504075fa0 0x7f650419b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504077570 0x7f650419be50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f650410b480 0x7f65041a2ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f65041112b0 con 0x7f6504075fa0 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6504111130 con 0x7f6504077570 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.576+0000 7f6508a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6504111430 con 0x7f650410b480 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504077570 0x7f650419be50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6504075fa0 0x7f650419b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504077570 0x7f650419be50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53628/0 (socket says 172.21.15.28:53628) 2024-05-20T20:04:35.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 -- 172.21.15.28:0/2023085656 learned_addr learned my addr 172.21.15.28:0/2023085656 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:35.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6502800700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f650410b480 0x7f65041a2ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 -- 172.21.15.28:0/2023085656 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f650410b480 msgr2=0x7f65041a2ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f650410b480 0x7f65041a2ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 -- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6504075fa0 msgr2=0x7f650419b910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6504075fa0 0x7f650419b910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6501400700 1 -- 172.21.15.28:0/2023085656 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f65041a3570 con 0x7f6504077570 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.577+0000 7f6502800700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f650410b480 0x7f65041a2ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.578+0000 7f6501400700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504077570 0x7f650419be50 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f64f000e970 tx=0x7f64f000ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.578+0000 7f64faa00700 1 -- 172.21.15.28:0/2023085656 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f64f0004220 con 0x7f6504077570 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.578+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f65041a3860 con 0x7f6504077570 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.578+0000 7f64faa00700 1 -- 172.21.15.28:0/2023085656 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f64f00043c0 con 0x7f6504077570 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.578+0000 7f64faa00700 1 -- 172.21.15.28:0/2023085656 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f64f0003ce0 con 0x7f6504077570 2024-05-20T20:04:35.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.578+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f65041a3df0 con 0x7f6504077570 2024-05-20T20:04:35.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.579+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f650406a230 con 0x7f6504077570 2024-05-20T20:04:35.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.583+0000 7f64faa00700 1 -- 172.21.15.28:0/2023085656 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f64f000c7e0 con 0x7f6504077570 2024-05-20T20:04:35.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.584+0000 7f64faa00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f64e80b2900 0x7f64e80b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:35.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.585+0000 7f64faa00700 1 -- 172.21.15.28:0/2023085656 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f64f00e23e0 con 0x7f6504077570 2024-05-20T20:04:35.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.585+0000 7f6501e00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f64e80b2900 0x7f64e80b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:35.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.586+0000 7f6501e00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f64e80b2900 0x7f64e80b4dc0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f6504077950 tx=0x7f64f4009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:35.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.586+0000 7f64faa00700 1 -- 172.21.15.28:0/2023085656 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f64f00aa0c0 con 0x7f6504077570 2024-05-20T20:04:35.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.841+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) -- 0x7f6504063980 con 0x7f64e80b2900 2024-05-20T20:04:35.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.858+0000 7f64faa00700 1 -- 172.21.15.28:0/2023085656 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+4123 (secure 0 0 0) 0x7f6504063980 con 0x7f64e80b2900 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:mds.a smithi028 running (3m) 53s ago 3m 17.7M - 19.0.0-3830-g4ebd1d53 559f9f297837 ee8975d9c7e7 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:mds.b smithi033 running (3m) 74s ago 3m 26.0M - 19.0.0-3830-g4ebd1d53 559f9f297837 ee0c30e87bae 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:mds.c smithi154 running (3m) 74s ago 3m 21.3M - 19.0.0-3830-g4ebd1d53 559f9f297837 f549f9ff21e3 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:mds.d smithi028 running (4m) 53s ago 4m 13.0M - 19.0.0-3830-g4ebd1d53 559f9f297837 e1a49fd899fd 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:mds.e smithi033 running (3m) 74s ago 3m 19.0M - 19.0.0-3830-g4ebd1d53 559f9f297837 4896299ff43d 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:mds.f smithi154 running (3m) 74s ago 3m 19.3M - 19.0.0-3830-g4ebd1d53 559f9f297837 d4e4b2e108ec 2024-05-20T20:04:35.859 INFO:teuthology.orchestra.run.smithi028.stdout:mds.g smithi028 running (3m) 53s ago 3m 12.9M - 19.0.0-3830-g4ebd1d53 559f9f297837 42c50cb52246 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mds.h smithi033 running (3m) 74s ago 3m 20.7M - 19.0.0-3830-g4ebd1d53 559f9f297837 ea89059c62bc 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mds.i smithi154 running (3m) 74s ago 3m 20.6M - 19.0.0-3830-g4ebd1d53 559f9f297837 a1ee9ca81892 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mds.j smithi028 running (4m) 53s ago 4m 17.9M - 19.0.0-3830-g4ebd1d53 559f9f297837 287c92d37fbb 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mds.k smithi033 running (4m) 74s ago 4m 18.1M - 19.0.0-3830-g4ebd1d53 559f9f297837 3613a2b3e18a 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mds.l smithi154 running (4m) 74s ago 4m 21.5M - 19.0.0-3830-g4ebd1d53 559f9f297837 07de68c6d6f3 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.x smithi028 *:9283,8765 running (12m) 53s ago 12m 531M - 19.0.0-3830-g4ebd1d53 559f9f297837 f9f76a24a660 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.y smithi033 *:8443,8765 running (10m) 74s ago 10m 469M - 19.0.0-3830-g4ebd1d53 559f9f297837 466c25eca87c 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mgr.z smithi154 *:8443,8765 running (10m) 74s ago 10m 470M - 19.0.0-3830-g4ebd1d53 559f9f297837 445f1d9e01f4 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mon.a smithi028 running (12m) 53s ago 12m 50.6M 2048M 19.0.0-3830-g4ebd1d53 559f9f297837 ea4da3bc4815 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mon.b smithi033 running (10m) 74s ago 10m 42.0M 2048M 19.0.0-3830-g4ebd1d53 559f9f297837 63bf016a4da0 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:mon.c smithi154 running (10m) 74s ago 10m 45.4M 2048M 19.0.0-3830-g4ebd1d53 559f9f297837 7216247a8a46 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 smithi028 running (9m) 53s ago 9m 45.0M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 f0c972ccb516 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 smithi033 running (9m) 74s ago 9m 48.4M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 d0985d6330fb 2024-05-20T20:04:35.860 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 smithi154 running (8m) 74s ago 8m 48.4M 4272M 19.0.0-3830-g4ebd1d53 559f9f297837 74ea13dcc916 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 smithi028 running (8m) 53s ago 8m 42.5M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 468766d7c39c 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 smithi033 running (7m) 74s ago 7m 45.0M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 1bde32d71d4d 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 smithi154 running (7m) 74s ago 7m 56.2M 4272M 19.0.0-3830-g4ebd1d53 559f9f297837 47bf7165bac1 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 smithi028 running (6m) 53s ago 6m 41.6M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 dd1372d90864 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 smithi033 running (6m) 74s ago 6m 56.0M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 8e64544b0484 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.8 smithi154 running (6m) 74s ago 6m 46.4M 4272M 19.0.0-3830-g4ebd1d53 559f9f297837 030a52b1dd97 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.9 smithi028 running (5m) 53s ago 5m 53.0M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 aa0630cefbc9 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.10 smithi033 running (5m) 74s ago 5m 44.8M 4273M 19.0.0-3830-g4ebd1d53 559f9f297837 60e1404b42b2 2024-05-20T20:04:35.861 INFO:teuthology.orchestra.run.smithi028.stdout:osd.11 smithi154 running (4m) 74s ago 4m 41.8M 4272M 19.0.0-3830-g4ebd1d53 559f9f297837 7390fe069fd8 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.861+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f64e80b2900 msgr2=0x7f64e80b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.861+0000 7f6508a00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f64e80b2900 0x7f64e80b4dc0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f6504077950 tx=0x7f64f4009260 comp rx=0 tx=0).stop 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.861+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504077570 msgr2=0x7f650419be50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.861+0000 7f6508a00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504077570 0x7f650419be50 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f64f000e970 tx=0x7f64f000ec80 comp rx=0 tx=0).stop 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 shutdown_connections 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f64e80b2900 0x7f64e80b4dc0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f650410b480 0x7f65041a2ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6504075fa0 0x7f650419b910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 --2- 172.21.15.28:0/2023085656 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6504077570 0x7f650419be50 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:35.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 >> 172.21.15.28:0/2023085656 conn(0x7f65040feb10 msgr2=0x7f6504107380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:35.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 shutdown_connections 2024-05-20T20:04:35.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:35.862+0000 7f6508a00700 1 -- 172.21.15.28:0/2023085656 wait complete. 2024-05-20T20:04:36.613 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch ls' 2024-05-20T20:04:36.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:36 smithi154 ceph-mon[36314]: pgmap v416: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:36.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:36 smithi154 ceph-mon[36314]: from='client.24977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:36.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:36 smithi033 ceph-mon[36179]: pgmap v416: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:36.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:36 smithi033 ceph-mon[36179]: from='client.24977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:37.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:36 smithi028 ceph-mon[31178]: pgmap v416: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:37.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:36 smithi028 ceph-mon[31178]: from='client.24977 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:37.080 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:38.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:38 smithi154 ceph-mon[36314]: pgmap v417: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:38.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:38 smithi033 ceph-mon[36179]: pgmap v417: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:38.896 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:38 smithi028 ceph-mon[31178]: pgmap v417: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:39.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.408+0000 7f398a600700 1 -- 172.21.15.28:0/3725924518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 msgr2=0x7f398410db10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:39.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.408+0000 7f398a600700 1 --2- 172.21.15.28:0/3725924518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f398410db10 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f397c00b0d0 tx=0x7f397c00b3e0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.409+0000 7f398a600700 1 -- 172.21.15.28:0/3725924518 shutdown_connections 2024-05-20T20:04:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.409+0000 7f398a600700 1 --2- 172.21.15.28:0/3725924518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f398410db10 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.409+0000 7f398a600700 1 --2- 172.21.15.28:0/3725924518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39841056c0 0x7f3984105ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.409+0000 7f398a600700 1 --2- 172.21.15.28:0/3725924518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39841068c0 0x7f3984106d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.409+0000 7f398a600700 1 -- 172.21.15.28:0/3725924518 >> 172.21.15.28:0/3725924518 conn(0x7f3984100c80 msgr2=0x7f39841030a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.409+0000 7f398a600700 1 -- 172.21.15.28:0/3725924518 shutdown_connections 2024-05-20T20:04:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.409+0000 7f398a600700 1 -- 172.21.15.28:0/3725924518 wait complete. 2024-05-20T20:04:39.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 Processor -- start 2024-05-20T20:04:39.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 -- start start 2024-05-20T20:04:39.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39841056c0 0x7f398419b7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39841068c0 0x7f398419bce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f39841a2d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f39841111b0 con 0x7f3984107260 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3984111030 con 0x7f39841068c0 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.410+0000 7f398a600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3984111330 con 0x7f39841056c0 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f39841a2d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f39841a2d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45904/0 (socket says 172.21.15.28:45904) 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 -- 172.21.15.28:0/1879783908 learned_addr learned my addr 172.21.15.28:0/1879783908 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3983e00700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39841068c0 0x7f398419bce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989200700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39841056c0 0x7f398419b7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 -- 172.21.15.28:0/1879783908 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39841056c0 msgr2=0x7f398419b7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39841056c0 0x7f398419b7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 -- 172.21.15.28:0/1879783908 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39841068c0 msgr2=0x7f398419bce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:39.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39841068c0 0x7f398419bce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 -- 172.21.15.28:0/1879783908 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f39841a3400 con 0x7f3984107260 2024-05-20T20:04:39.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3983e00700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39841068c0 0x7f398419bce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:39.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.411+0000 7f3989c00700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f39841a2d60 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f397c004cb0 tx=0x7f397c004ce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:39.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.412+0000 7f3981600700 1 -- 172.21.15.28:0/1879783908 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f397c01f030 con 0x7f3984107260 2024-05-20T20:04:39.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.412+0000 7f3981600700 1 -- 172.21.15.28:0/1879783908 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f397c005040 con 0x7f3984107260 2024-05-20T20:04:39.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.412+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f39841a3690 con 0x7f3984107260 2024-05-20T20:04:39.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.412+0000 7f3981600700 1 -- 172.21.15.28:0/1879783908 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f397c01b690 con 0x7f3984107260 2024-05-20T20:04:39.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.412+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f39841a3b90 con 0x7f3984107260 2024-05-20T20:04:39.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.413+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f398406a230 con 0x7f3984107260 2024-05-20T20:04:39.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.416+0000 7f3981600700 1 -- 172.21.15.28:0/1879783908 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f397c005480 con 0x7f3984107260 2024-05-20T20:04:39.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.417+0000 7f3981600700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f396c0b2900 0x7f396c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:39.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.417+0000 7f3981600700 1 -- 172.21.15.28:0/1879783908 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f397c00f020 con 0x7f3984107260 2024-05-20T20:04:39.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.417+0000 7f3989200700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f396c0b2900 0x7f396c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:39.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.418+0000 7f3989200700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f396c0b2900 0x7f396c0b4dc0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f3984106720 tx=0x7f3978009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:39.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.420+0000 7f3981600700 1 -- 172.21.15.28:0/1879783908 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f397c029020 con 0x7f3984107260 2024-05-20T20:04:39.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.673+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch ls", "target": ["mon-mgr", ""]}) -- 0x7f3984063980 con 0x7f396c0b2900 2024-05-20T20:04:39.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.682+0000 7f3981600700 1 -- 172.21.15.28:0/1879783908 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+980 (secure 0 0 0) 0x7f3984063980 con 0x7f396c0b2900 2024-05-20T20:04:39.683 INFO:teuthology.orchestra.run.smithi028.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-20T20:04:39.683 INFO:teuthology.orchestra.run.smithi028.stdout:mds.all 12/12 78s ago 4m smithi028=a;smithi028=d;smithi028=g;smithi028=j;smithi033=b;smithi033=e;smithi033=h;smithi033=k;smithi154=c;smithi154=f;smithi154=i;smithi154=l;count:12 2024-05-20T20:04:39.683 INFO:teuthology.orchestra.run.smithi028.stdout:mgr 3/3 78s ago 10m smithi028=x;smithi033=y;smithi154=z;count:3 2024-05-20T20:04:39.683 INFO:teuthology.orchestra.run.smithi028.stdout:mon 3/3 78s ago 10m smithi028:172.21.15.28=a;smithi033:172.21.15.33=b;smithi154:172.21.15.154=c;count:3 2024-05-20T20:04:39.683 INFO:teuthology.orchestra.run.smithi028.stdout:osd 12 78s ago - 2024-05-20T20:04:39.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.685+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f396c0b2900 msgr2=0x7f396c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:39.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.685+0000 7f398a600700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f396c0b2900 0x7f396c0b4dc0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f3984106720 tx=0x7f3978009260 comp rx=0 tx=0).stop 2024-05-20T20:04:39.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.685+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 msgr2=0x7f39841a2d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.685+0000 7f398a600700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f39841a2d60 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f397c004cb0 tx=0x7f397c004ce0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 shutdown_connections 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f396c0b2900 0x7f396c0b4dc0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3984107260 0x7f39841a2d60 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39841056c0 0x7f398419b7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 --2- 172.21.15.28:0/1879783908 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39841068c0 0x7f398419bce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 >> 172.21.15.28:0/1879783908 conn(0x7f3984100c80 msgr2=0x7f3984109bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:39.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 shutdown_connections 2024-05-20T20:04:39.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:39.686+0000 7f398a600700 1 -- 172.21.15.28:0/1879783908 wait complete. 2024-05-20T20:04:40.502 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch host ls' 2024-05-20T20:04:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:40 smithi028 ceph-mon[31178]: pgmap v418: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:40.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:40 smithi028 ceph-mon[31178]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:40 smithi154 ceph-mon[36314]: pgmap v418: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:40 smithi154 ceph-mon[36314]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:40 smithi033 ceph-mon[36179]: pgmap v418: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:40 smithi033 ceph-mon[36179]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:42.729 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:42 smithi028 ceph-mon[31178]: pgmap v419: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:42.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:42 smithi154 ceph-mon[36314]: pgmap v419: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:42.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:42 smithi033 ceph-mon[36179]: pgmap v419: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:43.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 -- 172.21.15.28:0/64854505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 msgr2=0x7f5af4076420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:43.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 --2- 172.21.15.28:0/64854505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 0x7f5af4076420 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f5adc009990 tx=0x7f5adc009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 -- 172.21.15.28:0/64854505 shutdown_connections 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 --2- 172.21.15.28:0/64854505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5af410b480 0x7f5af410d910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 --2- 172.21.15.28:0/64854505 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 0x7f5af40759d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 --2- 172.21.15.28:0/64854505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 0x7f5af4076420 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 -- 172.21.15.28:0/64854505 >> 172.21.15.28:0/64854505 conn(0x7f5af40feb10 msgr2=0x7f5af4100f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 -- 172.21.15.28:0/64854505 shutdown_connections 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.234+0000 7f5afa800700 1 -- 172.21.15.28:0/64854505 wait complete. 2024-05-20T20:04:43.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.235+0000 7f5afa800700 1 Processor -- start 2024-05-20T20:04:43.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.236+0000 7f5afa800700 1 -- start start 2024-05-20T20:04:43.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.236+0000 7f5afa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 0x7f5af419b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:43.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.236+0000 7f5afa800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 0x7f5af419bee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:43.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.236+0000 7f5afa800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5af410b480 0x7f5af41a2f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:43.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.236+0000 7f5afa800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5af4111290 con 0x7f5af4077570 2024-05-20T20:04:43.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.236+0000 7f5afa800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5af4111110 con 0x7f5af4075fa0 2024-05-20T20:04:43.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.236+0000 7f5afa800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5af4111410 con 0x7f5af410b480 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 0x7f5af419bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 0x7f5af419bee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45928/0 (socket says 172.21.15.28:45928) 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 -- 172.21.15.28:0/3366931987 learned_addr learned my addr 172.21.15.28:0/3366931987 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5af3400700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 0x7f5af419b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5af3e00700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5af410b480 0x7f5af41a2f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 -- 172.21.15.28:0/3366931987 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5af410b480 msgr2=0x7f5af41a2f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5af410b480 0x7f5af41a2f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 -- 172.21.15.28:0/3366931987 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 msgr2=0x7f5af419b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 0x7f5af419b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 -- 172.21.15.28:0/3366931987 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5af41a3600 con 0x7f5af4077570 2024-05-20T20:04:43.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.237+0000 7f5aeaa00700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 0x7f5af419bee0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f5adc00b760 tx=0x7f5adc0057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:43.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.238+0000 7f5af0c00700 1 -- 172.21.15.28:0/3366931987 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5adc027070 con 0x7f5af4077570 2024-05-20T20:04:43.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.238+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5af41a3890 con 0x7f5af4077570 2024-05-20T20:04:43.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.238+0000 7f5af0c00700 1 -- 172.21.15.28:0/3366931987 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5adc022470 con 0x7f5af4077570 2024-05-20T20:04:43.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.238+0000 7f5af0c00700 1 -- 172.21.15.28:0/3366931987 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5adc017650 con 0x7f5af4077570 2024-05-20T20:04:43.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.238+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5af41a3cf0 con 0x7f5af4077570 2024-05-20T20:04:43.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.239+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5af406a230 con 0x7f5af4077570 2024-05-20T20:04:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.243+0000 7f5af0c00700 1 -- 172.21.15.28:0/3366931987 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f5adc02a070 con 0x7f5af4077570 2024-05-20T20:04:43.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.245+0000 7f5af0c00700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5ad80b2880 0x7f5ad80b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:43.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.245+0000 7f5af0c00700 1 -- 172.21.15.28:0/3366931987 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f5adc0e3350 con 0x7f5af4077570 2024-05-20T20:04:43.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.245+0000 7f5af3400700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5ad80b2880 0x7f5ad80b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:43.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.246+0000 7f5af3400700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5ad80b2880 0x7f5ad80b4d40 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f5af4077950 tx=0x7f5ae4009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:43.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.247+0000 7f5af0c00700 1 -- 172.21.15.28:0/3366931987 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f5adc031080 con 0x7f5af4077570 2024-05-20T20:04:43.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.495+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""]}) -- 0x7f5af4063980 con 0x7f5ad80b2880 2024-05-20T20:04:43.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.499+0000 7f5af0c00700 1 -- 172.21.15.28:0/3366931987 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+191 (secure 0 0 0) 0x7f5af4063980 con 0x7f5ad80b2880 2024-05-20T20:04:43.499 INFO:teuthology.orchestra.run.smithi028.stdout:HOST ADDR LABELS STATUS 2024-05-20T20:04:43.499 INFO:teuthology.orchestra.run.smithi028.stdout:smithi028 172.21.15.28 2024-05-20T20:04:43.499 INFO:teuthology.orchestra.run.smithi028.stdout:smithi033 172.21.15.33 2024-05-20T20:04:43.499 INFO:teuthology.orchestra.run.smithi028.stdout:smithi154 172.21.15.154 2024-05-20T20:04:43.499 INFO:teuthology.orchestra.run.smithi028.stdout:3 hosts in cluster 2024-05-20T20:04:43.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5ad80b2880 msgr2=0x7f5ad80b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:43.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5ad80b2880 0x7f5ad80b4d40 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f5af4077950 tx=0x7f5ae4009260 comp rx=0 tx=0).stop 2024-05-20T20:04:43.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 msgr2=0x7f5af419bee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:43.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 0x7f5af419bee0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f5adc00b760 tx=0x7f5adc0057c0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 shutdown_connections 2024-05-20T20:04:43.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5ad80b2880 0x7f5ad80b4d40 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5af410b480 0x7f5af41a2f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5af4075fa0 0x7f5af419b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 --2- 172.21.15.28:0/3366931987 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5af4077570 0x7f5af419bee0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:43.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.502+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 >> 172.21.15.28:0/3366931987 conn(0x7f5af40feb10 msgr2=0x7f5af4107380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:43.504 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.503+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 shutdown_connections 2024-05-20T20:04:43.504 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:43.503+0000 7f5afa800700 1 -- 172.21.15.28:0/3366931987 wait complete. 2024-05-20T20:04:44.328 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch device ls' 2024-05-20T20:04:44.830 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:44 smithi154 ceph-mon[36314]: pgmap v420: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:44 smithi154 ceph-mon[36314]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:44 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:04:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:44 smithi033 ceph-mon[36179]: pgmap v420: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:44 smithi033 ceph-mon[36179]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:44.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:44 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:04:44.994 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:44 smithi028 ceph-mon[31178]: pgmap v420: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:44.994 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:44 smithi028 ceph-mon[31178]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:44.994 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:44 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:04:46.143 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.143 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.143 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.143 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.143 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: pgmap v421: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:46.144 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:04:46.144 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:04:46.144 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:46 smithi028 ceph-mon[31178]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: pgmap v421: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:04:46.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:04:46.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:46 smithi154 ceph-mon[36314]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: pgmap v421: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:04:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:46 smithi033 ceph-mon[36179]: from='mgr.14152 172.21.15.28:0/2712975487' entity='mgr.x' 2024-05-20T20:04:47.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:46.999+0000 7fa4e0000700 1 -- 172.21.15.28:0/3946102430 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81069e0 msgr2=0x7fa4d8106e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:46.999+0000 7fa4e0000700 1 --2- 172.21.15.28:0/3946102430 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81069e0 0x7fa4d8106e40 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fa4c40099c0 tx=0x7fa4c4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.000+0000 7fa4e0000700 1 -- 172.21.15.28:0/3946102430 shutdown_connections 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.000+0000 7fa4e0000700 1 --2- 172.21.15.28:0/3946102430 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa4d8107380 0x7fa4d810dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.000+0000 7fa4e0000700 1 --2- 172.21.15.28:0/3946102430 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa4d81057e0 0x7fa4d8105c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.000+0000 7fa4e0000700 1 --2- 172.21.15.28:0/3946102430 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81069e0 0x7fa4d8106e40 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.000+0000 7fa4e0000700 1 -- 172.21.15.28:0/3946102430 >> 172.21.15.28:0/3946102430 conn(0x7fa4d8100da0 msgr2=0x7fa4d81031c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.000+0000 7fa4e0000700 1 -- 172.21.15.28:0/3946102430 shutdown_connections 2024-05-20T20:04:47.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.000+0000 7fa4e0000700 1 -- 172.21.15.28:0/3946102430 wait complete. 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 Processor -- start 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 -- start start 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81057e0 0x7fa4d819b8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa4d81069e0 0x7fa4d819bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa4d8107380 0x7fa4d81a2e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d8111250 con 0x7fa4d81057e0 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d81110d0 con 0x7fa4d8107380 2024-05-20T20:04:47.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.001+0000 7fa4e0000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa4d81113d0 con 0x7fa4d81069e0 2024-05-20T20:04:47.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81057e0 0x7fa4d819b8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:47.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81057e0 0x7fa4d819b8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:33128/0 (socket says 172.21.15.28:33128) 2024-05-20T20:04:47.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 -- 172.21.15.28:0/2173966994 learned_addr learned my addr 172.21.15.28:0/2173966994 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:47.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dcc00700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa4d81069e0 0x7fa4d819bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:47.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4de000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa4d8107380 0x7fa4d81a2e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:47.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 -- 172.21.15.28:0/2173966994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa4d81069e0 msgr2=0x7fa4d819bdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa4d81069e0 0x7fa4d819bdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 -- 172.21.15.28:0/2173966994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa4d8107380 msgr2=0x7fa4d81a2e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa4d8107380 0x7fa4d81a2e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.002+0000 7fa4dd600700 1 -- 172.21.15.28:0/2173966994 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa4d81a3510 con 0x7fa4d81057e0 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.003+0000 7fa4dd600700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81057e0 0x7fa4d819b8b0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fa4cc00e9a0 tx=0x7fa4cc00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.003+0000 7fa4d6000700 1 -- 172.21.15.28:0/2173966994 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4cc004220 con 0x7fa4d81057e0 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.003+0000 7fa4d6000700 1 -- 172.21.15.28:0/2173966994 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa4cc0043c0 con 0x7fa4d81057e0 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.003+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa4d81a3800 con 0x7fa4d81057e0 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.003+0000 7fa4d6000700 1 -- 172.21.15.28:0/2173966994 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa4cc003c70 con 0x7fa4d81057e0 2024-05-20T20:04:47.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.003+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa4d81a3d90 con 0x7fa4d81057e0 2024-05-20T20:04:47.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.004+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa4d806a230 con 0x7fa4d81057e0 2024-05-20T20:04:47.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.007+0000 7fa4d6000700 1 -- 172.21.15.28:0/2173966994 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fa4cc00c7c0 con 0x7fa4d81057e0 2024-05-20T20:04:47.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.009+0000 7fa4d6000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa4c00b2900 0x7fa4c00b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:47.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.009+0000 7fa4d6000700 1 -- 172.21.15.28:0/2173966994 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fa4cc0e2370 con 0x7fa4d81057e0 2024-05-20T20:04:47.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.009+0000 7fa4dcc00700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa4c00b2900 0x7fa4c00b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:47.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.010+0000 7fa4dcc00700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa4c00b2900 0x7fa4c00b4dc0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4000c00 tx=0x7fa4c4005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:47.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.014+0000 7fa4d6000700 1 -- 172.21.15.28:0/2173966994 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa4cc0aa090 con 0x7fa4d81057e0 2024-05-20T20:04:47.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.262+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "orch device ls", "target": ["mon-mgr", ""]}) -- 0x7fa4d8063980 con 0x7fa4c00b2900 2024-05-20T20:04:47.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.266+0000 7fa4d6000700 1 -- 172.21.15.28:0/2173966994 <== mgr.14152 v2:172.21.15.28:6800/2019900521 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+532 (secure 0 0 0) 0x7fa4d8063980 con 0x7fa4c00b2900 2024-05-20T20:04:47.267 INFO:teuthology.orchestra.run.smithi028.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2024-05-20T20:04:47.267 INFO:teuthology.orchestra.run.smithi028.stdout:smithi028 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB747401F4480DGN 447G No 5m ago Has a FileSystem, LVM detected 2024-05-20T20:04:47.267 INFO:teuthology.orchestra.run.smithi028.stdout:smithi033 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5330008Q400BGN 372G No 5m ago Has a FileSystem, LVM detected 2024-05-20T20:04:47.267 INFO:teuthology.orchestra.run.smithi028.stdout:smithi154 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6233003B400BGN 372G No 4m ago Has a FileSystem, LVM detected 2024-05-20T20:04:47.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.269+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa4c00b2900 msgr2=0x7fa4c00b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:47.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.269+0000 7fa4e0000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa4c00b2900 0x7fa4c00b4dc0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fa4c4000c00 tx=0x7fa4c4005fb0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.269+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81057e0 msgr2=0x7fa4d819b8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:47.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.269+0000 7fa4e0000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81057e0 0x7fa4d819b8b0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fa4cc00e9a0 tx=0x7fa4cc00ecb0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 shutdown_connections 2024-05-20T20:04:47.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa4c00b2900 0x7fa4c00b4dc0 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa4d8107380 0x7fa4d81a2e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa4d81057e0 0x7fa4d819b8b0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 --2- 172.21.15.28:0/2173966994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa4d81069e0 0x7fa4d819bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:47.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 >> 172.21.15.28:0/2173966994 conn(0x7fa4d8100da0 msgr2=0x7fa4d8109d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:47.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 shutdown_connections 2024-05-20T20:04:47.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:47.270+0000 7fa4e0000700 1 -- 172.21.15.28:0/2173966994 wait complete. 2024-05-20T20:04:48.113 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T20:04:48.121 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:04:48.121 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- bash -c 'ceph fs dump' 2024-05-20T20:04:48.623 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:48.773 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:48 smithi028 ceph-mon[31178]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:48.773 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:48 smithi028 ceph-mon[31178]: pgmap v422: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:48 smithi154 ceph-mon[36314]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:48 smithi154 ceph-mon[36314]: pgmap v422: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:48 smithi033 ceph-mon[36179]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:04:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:48 smithi033 ceph-mon[36179]: pgmap v422: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:50.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:50 smithi028 ceph-mon[31178]: pgmap v423: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:50.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 -- 172.21.15.28:0/928064566 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 msgr2=0x7fef78105b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:50.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 --2- 172.21.15.28:0/928064566 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef78105b00 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fef6c0099c0 tx=0x7fef6c009cd0 comp rx=0 tx=0).stop 2024-05-20T20:04:50.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 -- 172.21.15.28:0/928064566 shutdown_connections 2024-05-20T20:04:50.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 --2- 172.21.15.28:0/928064566 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fef78107280 0x7fef7810db30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:50.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 --2- 172.21.15.28:0/928064566 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef78105b00 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:50.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 --2- 172.21.15.28:0/928064566 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fef781068e0 0x7fef78106d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:50.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 -- 172.21.15.28:0/928064566 >> 172.21.15.28:0/928064566 conn(0x7fef78100c80 msgr2=0x7fef781030c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:50.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 -- 172.21.15.28:0/928064566 shutdown_connections 2024-05-20T20:04:50.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.825+0000 7fef7e800700 1 -- 172.21.15.28:0/928064566 wait complete. 2024-05-20T20:04:50.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.826+0000 7fef7e800700 1 Processor -- start 2024-05-20T20:04:50.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.826+0000 7fef7e800700 1 -- start start 2024-05-20T20:04:50.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef7819b870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:50.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fef781068e0 0x7fef7819bdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:50.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7e800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fef78107280 0x7fef781a2e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:50.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7e800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fef781111a0 con 0x7fef781056e0 2024-05-20T20:04:50.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7e800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fef78111020 con 0x7fef78107280 2024-05-20T20:04:50.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7e800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fef78111320 con 0x7fef781068e0 2024-05-20T20:04:50.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef7819b870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:50.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef7819b870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:33152/0 (socket says 172.21.15.28:33152) 2024-05-20T20:04:50.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7d400700 1 -- 172.21.15.28:0/3981800739 learned_addr learned my addr 172.21.15.28:0/3981800739 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:50.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7ca00700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fef781068e0 0x7fef7819bdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:50.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7de00700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fef78107280 0x7fef781a2e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:50.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7d400700 1 -- 172.21.15.28:0/3981800739 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fef781068e0 msgr2=0x7fef7819bdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:50.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7d400700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fef781068e0 0x7fef7819bdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:50.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7d400700 1 -- 172.21.15.28:0/3981800739 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fef78107280 msgr2=0x7fef781a2e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:50.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.827+0000 7fef7d400700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fef78107280 0x7fef781a2e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:50.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef7d400700 1 -- 172.21.15.28:0/3981800739 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fef781a34d0 con 0x7fef781056e0 2024-05-20T20:04:50.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef7ca00700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fef781068e0 0x7fef7819bdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:50.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef7d400700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef7819b870 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fef6c00b580 tx=0x7fef6c003af0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:50.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef76000700 1 -- 172.21.15.28:0/3981800739 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fef6c009e60 con 0x7fef781056e0 2024-05-20T20:04:50.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fef781a3760 con 0x7fef781056e0 2024-05-20T20:04:50.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef76000700 1 -- 172.21.15.28:0/3981800739 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fef6c005570 con 0x7fef781056e0 2024-05-20T20:04:50.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef76000700 1 -- 172.21.15.28:0/3981800739 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fef6c00fe00 con 0x7fef781056e0 2024-05-20T20:04:50.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.828+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fef781a3bc0 con 0x7fef781056e0 2024-05-20T20:04:50.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.830+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fef7806a230 con 0x7fef781056e0 2024-05-20T20:04:50.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.832+0000 7fef76000700 1 -- 172.21.15.28:0/3981800739 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fef6c00f4f0 con 0x7fef781056e0 2024-05-20T20:04:50.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.834+0000 7fef76000700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fef600b2880 0x7fef600b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:50.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.834+0000 7fef76000700 1 -- 172.21.15.28:0/3981800739 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fef6c0e3050 con 0x7fef781056e0 2024-05-20T20:04:50.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.834+0000 7fef7ca00700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fef600b2880 0x7fef600b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:50.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.835+0000 7fef7ca00700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fef600b2880 0x7fef600b4d40 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fef64011490 tx=0x7fef64009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:50.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:50.837+0000 7fef76000700 1 -- 172.21.15.28:0/3981800739 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fef6c013070 con 0x7fef781056e0 2024-05-20T20:04:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:50 smithi154 ceph-mon[36314]: pgmap v423: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:50 smithi033 ceph-mon[36179]: pgmap v423: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:51.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.125+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7fef7802e560 con 0x7fef781056e0 2024-05-20T20:04:51.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.126+0000 7fef76000700 1 -- 172.21.15.28:0/3981800739 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 28 v28) ==== 76+0+3086 (secure 0 0 0) 0x7fef6c0a4e20 con 0x7fef781056e0 2024-05-20T20:04:51.128 INFO:teuthology.orchestra.run.smithi028.stdout:e28 2024-05-20T20:04:51.128 INFO:teuthology.orchestra.run.smithi028.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T20:04:51.128 INFO:teuthology.orchestra.run.smithi028.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:legacy client fscid: 1 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:Filesystem 'cephfs' (1) 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:fs_name cephfs 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:epoch 28 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:created 2024-05-20T20:01:26.231967+0000 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:modified 2024-05-20T20:01:50.441739+0000 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:tableserver 0 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:root 0 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:session_timeout 300 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:session_autoclose 300 2024-05-20T20:04:51.129 INFO:teuthology.orchestra.run.smithi028.stdout:max_file_size 1099511627776 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:max_xattr_size 65536 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:required_client_features {} 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:last_failure 0 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:last_failure_osd_epoch 0 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:max_mds 5 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:in 0,1,2,3,4 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:up {0=24542,1=24500,2=24493,3=24482,4=24475} 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:failed 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:damaged 2024-05-20T20:04:51.130 INFO:teuthology.orchestra.run.smithi028.stdout:stopped 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:data_pools [3] 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:metadata_pool 2 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:inline_data disabled 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:balancer 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:bal_rank_mask -1 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:standby_count_wanted 0 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:qdb_cluster leader: 24542 members: 24475,24482,24493,24500,24542 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.b{0:24542} state up:active seq 7 addr [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.f{0:24463} state up:standby-replay seq 1 addr [v2:172.21.15.154:6834/3965936826,v1:172.21.15.154:6835/3965936826] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.h{1:24500} state up:active seq 15 addr [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.k{1:24455} state up:standby-replay seq 1 addr [v2:172.21.15.33:6832/260147664,v1:172.21.15.33:6833/260147664] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.i{2:24493} state up:active seq 15 addr [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.j{2:24442} state up:standby-replay seq 1 addr [v2:172.21.15.28:6834/34026174,v1:172.21.15.28:6835/34026174] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.131 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.e{3:24482} state up:active seq 18 addr [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.l{3:24427} state up:standby-replay seq 1 addr [v2:172.21.15.154:6832/2654302434,v1:172.21.15.154:6833/2654302434] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.c{4:24475} state up:active seq 17 addr [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.a{4:14400} state up:standby-replay seq 1 addr [v2:172.21.15.28:6840/1736910968,v1:172.21.15.28:6841/1736910968] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout:Standby daemons: 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.d{-1:14364} state up:standby seq 1 addr [v2:172.21.15.28:6836/1950403416,v1:172.21.15.28:6837/1950403416] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.132 INFO:teuthology.orchestra.run.smithi028.stdout:[mds.g{-1:14388} state up:standby seq 1 addr [v2:172.21.15.28:6838/892373280,v1:172.21.15.28:6839/892373280] compat {c=[1],r=[1],i=[1fff]}] 2024-05-20T20:04:51.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.129+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fef600b2880 msgr2=0x7fef600b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:51.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.129+0000 7fef7e800700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fef600b2880 0x7fef600b4d40 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fef64011490 tx=0x7fef64009260 comp rx=0 tx=0).stop 2024-05-20T20:04:51.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.129+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 msgr2=0x7fef7819b870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:51.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.129+0000 7fef7e800700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef7819b870 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fef6c00b580 tx=0x7fef6c003af0 comp rx=0 tx=0).stop 2024-05-20T20:04:51.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 shutdown_connections 2024-05-20T20:04:51.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fef600b2880 0x7fef600b4d40 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:51.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fef78107280 0x7fef781a2e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:51.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fef781056e0 0x7fef7819b870 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:51.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 --2- 172.21.15.28:0/3981800739 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fef781068e0 0x7fef7819bdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:51.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 >> 172.21.15.28:0/3981800739 conn(0x7fef78100c80 msgr2=0x7fef78109cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:51.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 shutdown_connections 2024-05-20T20:04:51.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:51.130+0000 7fef7e800700 1 -- 172.21.15.28:0/3981800739 wait complete. 2024-05-20T20:04:51.136 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 28 2024-05-20T20:04:51.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:51 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3981800739' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-05-20T20:04:51.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:51 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3981800739' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-05-20T20:04:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:51 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3981800739' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-05-20T20:04:51.976 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- bash -c 'ceph osd dump' 2024-05-20T20:04:52.438 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:52.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:52 smithi028 ceph-mon[31178]: pgmap v424: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:52.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:52 smithi154 ceph-mon[36314]: pgmap v424: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:52.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:52 smithi033 ceph-mon[36179]: pgmap v424: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 -- 172.21.15.28:0/2270493797 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f45741068e0 msgr2=0x7f4574106d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 --2- 172.21.15.28:0/2270493797 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f45741068e0 0x7f4574106d40 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f455c009990 tx=0x7f455c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 -- 172.21.15.28:0/2270493797 shutdown_connections 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 --2- 172.21.15.28:0/2270493797 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4574107280 0x7f457410db30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 --2- 172.21.15.28:0/2270493797 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f45741056e0 0x7f4574105b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 --2- 172.21.15.28:0/2270493797 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f45741068e0 0x7f4574106d40 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 -- 172.21.15.28:0/2270493797 >> 172.21.15.28:0/2270493797 conn(0x7f4574100c60 msgr2=0x7f45741030c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 -- 172.21.15.28:0/2270493797 shutdown_connections 2024-05-20T20:04:54.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.756+0000 7f4573200700 1 -- 172.21.15.28:0/2270493797 wait complete. 2024-05-20T20:04:54.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 Processor -- start 2024-05-20T20:04:54.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 -- start start 2024-05-20T20:04:54.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f45741056e0 0x7f457419b7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:54.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f45741068e0 0x7f457419bd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:54.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4574107280 0x7f45741a2d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:54.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f45741111a0 con 0x7f45741056e0 2024-05-20T20:04:54.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4574111020 con 0x7f45741068e0 2024-05-20T20:04:54.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.758+0000 7f4573200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4574111320 con 0x7f4574107280 2024-05-20T20:04:54.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f45741056e0 0x7f457419b7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:54.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f45741056e0 0x7f457419b7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:33178/0 (socket says 172.21.15.28:33178) 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 -- 172.21.15.28:0/991029730 learned_addr learned my addr 172.21.15.28:0/991029730 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4569400700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f45741068e0 0x7f457419bd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4572800700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4574107280 0x7f45741a2d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 -- 172.21.15.28:0/991029730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4574107280 msgr2=0x7f45741a2d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4574107280 0x7f45741a2d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 -- 172.21.15.28:0/991029730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f45741068e0 msgr2=0x7f457419bd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f45741068e0 0x7f457419bd00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4571e00700 1 -- 172.21.15.28:0/991029730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f45741a3420 con 0x7f45741056e0 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.759+0000 7f4569400700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f45741068e0 0x7f457419bd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.760+0000 7f4571e00700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f45741056e0 0x7f457419b7c0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f4564009f40 tx=0x7f456400c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.760+0000 7f456b400700 1 -- 172.21.15.28:0/991029730 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f45640092e0 con 0x7f45741056e0 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.760+0000 7f456b400700 1 -- 172.21.15.28:0/991029730 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f456400c9c0 con 0x7f45741056e0 2024-05-20T20:04:54.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.760+0000 7f456b400700 1 -- 172.21.15.28:0/991029730 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f45640044c0 con 0x7f45741056e0 2024-05-20T20:04:54.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.760+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f45741a36b0 con 0x7f45741056e0 2024-05-20T20:04:54.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.760+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f45741a3c40 con 0x7f45741056e0 2024-05-20T20:04:54.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.761+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f457406a230 con 0x7f45741056e0 2024-05-20T20:04:54.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.764+0000 7f456b400700 1 -- 172.21.15.28:0/991029730 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f456401b030 con 0x7f45741056e0 2024-05-20T20:04:54.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.765+0000 7f456b400700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f45580b2900 0x7f45580b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:54.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.766+0000 7f456b400700 1 -- 172.21.15.28:0/991029730 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f45640e2a60 con 0x7f45741056e0 2024-05-20T20:04:54.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.766+0000 7f4569400700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f45580b2900 0x7f45580b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:54.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.767+0000 7f4569400700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f45580b2900 0x7f45580b4dc0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f455c00b720 tx=0x7f455c005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:54.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:54.768+0000 7f456b400700 1 -- 172.21.15.28:0/991029730 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f4564022020 con 0x7f45741056e0 2024-05-20T20:04:54.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:54 smithi028 ceph-mon[31178]: pgmap v425: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:54 smithi154 ceph-mon[36314]: pgmap v425: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:54.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:54 smithi033 ceph-mon[36179]: pgmap v425: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:04:55.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.011+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd dump"} v 0) -- 0x7f457402e560 con 0x7f45741056e0 2024-05-20T20:04:55.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.013+0000 7f456b400700 1 -- 172.21.15.28:0/991029730 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump"}]=0 v80) ==== 56+0+5285 (secure 0 0 0) 0x7f4564010040 con 0x7f45741056e0 2024-05-20T20:04:55.020 INFO:teuthology.orchestra.run.smithi028.stdout:epoch 80 2024-05-20T20:04:55.020 INFO:teuthology.orchestra.run.smithi028.stdout:fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T20:04:55.020 INFO:teuthology.orchestra.run.smithi028.stdout:created 2024-05-20T19:51:52.746711+0000 2024-05-20T20:04:55.020 INFO:teuthology.orchestra.run.smithi028.stdout:modified 2024-05-20T20:02:04.742448+0000 2024-05-20T20:04:55.020 INFO:teuthology.orchestra.run.smithi028.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:crush_version 26 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:full_ratio 0.95 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:backfillfull_ratio 0.9 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:nearfull_ratio 0.85 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:require_min_compat_client luminous 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:min_compat_client luminous 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:require_osd_release squid 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:stretch_mode_enabled false 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 12.00 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:pool 2 'cephfs_metadata' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 78 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 2.25 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:pool 3 'cephfs_data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode off last_change 78 flags hashpspool stripe_width 0 pg_num_min 64 target_size_ratio 0.9 application cephfs read_balance_score 1.50 2024-05-20T20:04:55.021 INFO:teuthology.orchestra.run.smithi028.stdout:max_osd 12 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.0 up in weight 1 up_from 8 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.28:6802/3612620015,v1:172.21.15.28:6803/3612620015] [v2:172.21.15.28:6804/3612620015,v1:172.21.15.28:6805/3612620015] exists,up ee508d61-6223-4ba1-a7c6-6be05c87bad9 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.1 up in weight 1 up_from 13 up_thru 75 down_at 0 last_clean_interval [0,0) [v2:172.21.15.33:6800/3302429642,v1:172.21.15.33:6801/3302429642] [v2:172.21.15.33:6802/3302429642,v1:172.21.15.33:6803/3302429642] exists,up c93dcd4d-a863-4d99-922d-2ba54fe25033 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.2 up in weight 1 up_from 18 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.154:6800/152922770,v1:172.21.15.154:6801/152922770] [v2:172.21.15.154:6802/152922770,v1:172.21.15.154:6803/152922770] exists,up a98e172e-f3be-463f-a25d-bc9fc778ce48 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.3 up in weight 1 up_from 25 up_thru 75 down_at 0 last_clean_interval [0,0) [v2:172.21.15.28:6810/1329515601,v1:172.21.15.28:6811/1329515601] [v2:172.21.15.28:6812/1329515601,v1:172.21.15.28:6813/1329515601] exists,up 43c813b2-4e9a-4c76-9e2e-6c19825adf7a 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.4 up in weight 1 up_from 31 up_thru 75 down_at 0 last_clean_interval [0,0) [v2:172.21.15.33:6808/3112020300,v1:172.21.15.33:6809/3112020300] [v2:172.21.15.33:6810/3112020300,v1:172.21.15.33:6811/3112020300] exists,up 57abdf0f-f317-465a-99d5-1d761d481da1 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.5 up in weight 1 up_from 36 up_thru 75 down_at 0 last_clean_interval [0,0) [v2:172.21.15.154:6808/666306624,v1:172.21.15.154:6809/666306624] [v2:172.21.15.154:6810/666306624,v1:172.21.15.154:6811/666306624] exists,up 967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.6 up in weight 1 up_from 42 up_thru 75 down_at 0 last_clean_interval [0,0) [v2:172.21.15.28:6818/163025282,v1:172.21.15.28:6819/163025282] [v2:172.21.15.28:6820/163025282,v1:172.21.15.28:6821/163025282] exists,up 1fb2acac-f855-46cf-8255-833f111bdddd 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.7 up in weight 1 up_from 47 up_thru 75 down_at 0 last_clean_interval [0,0) [v2:172.21.15.33:6816/867501728,v1:172.21.15.33:6817/867501728] [v2:172.21.15.33:6818/867501728,v1:172.21.15.33:6819/867501728] exists,up a66dec84-09f9-4179-9130-3eb1e6cc5044 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.8 up in weight 1 up_from 53 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.154:6816/1056926817,v1:172.21.15.154:6817/1056926817] [v2:172.21.15.154:6818/1056926817,v1:172.21.15.154:6819/1056926817] exists,up 8eb55633-1f82-4288-bf5c-e1e988c1d9fc 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.9 up in weight 1 up_from 59 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.28:6826/3479914961,v1:172.21.15.28:6827/3479914961] [v2:172.21.15.28:6828/3479914961,v1:172.21.15.28:6829/3479914961] exists,up 4de26914-f99d-42d0-a3f9-9050e7bf73b0 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.10 up in weight 1 up_from 64 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.33:6824/924561268,v1:172.21.15.33:6825/924561268] [v2:172.21.15.33:6826/924561268,v1:172.21.15.33:6827/924561268] exists,up 28571f48-e611-45cd-9835-72989012a981 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:osd.11 up in weight 1 up_from 70 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.154:6824/2796531640,v1:172.21.15.154:6825/2796531640] [v2:172.21.15.154:6826/2796531640,v1:172.21.15.154:6827/2796531640] exists,up cb59bcc3-6dcc-42dd-8492-46bf37eb3be6 2024-05-20T20:04:55.022 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 2.6 [8,11] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 2.10 [2,4] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 2.13 [2,11] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 3.6 [4,6] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 3.1a [5,7] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 3.1d [5,9] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 3.22 [3,9] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 3.36 [4,9] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:pg_upmap_items 3.3a [4,8] 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:6801/811935446 expires 2024-05-21T19:53:03.221125+0000 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:6800/811935446 expires 2024-05-21T19:53:03.221125+0000 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:0/3213207110 expires 2024-05-21T19:53:03.221125+0000 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:0/443467971 expires 2024-05-21T19:52:27.405869+0000 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:0/3330495421 expires 2024-05-21T19:52:27.405869+0000 2024-05-20T20:04:55.023 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:6800/1506400984 expires 2024-05-21T19:52:27.405869+0000 2024-05-20T20:04:55.024 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:0/4072125252 expires 2024-05-21T19:53:03.221125+0000 2024-05-20T20:04:55.024 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:0/3854117144 expires 2024-05-21T19:52:27.405869+0000 2024-05-20T20:04:55.024 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:6801/1506400984 expires 2024-05-21T19:52:27.405869+0000 2024-05-20T20:04:55.024 INFO:teuthology.orchestra.run.smithi028.stdout:blocklist 172.21.15.28:0/1163220027 expires 2024-05-21T19:53:03.221125+0000 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.016+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f45580b2900 msgr2=0x7f45580b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.016+0000 7f4573200700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f45580b2900 0x7f45580b4dc0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f455c00b720 tx=0x7f455c005fd0 comp rx=0 tx=0).stop 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.016+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f45741056e0 msgr2=0x7f457419b7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.016+0000 7f4573200700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f45741056e0 0x7f457419b7c0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f4564009f40 tx=0x7f456400c3a0 comp rx=0 tx=0).stop 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 shutdown_connections 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f45580b2900 0x7f45580b4dc0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4574107280 0x7f45741a2d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:55.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f45741056e0 0x7f457419b7c0 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:55.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 --2- 172.21.15.28:0/991029730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f45741068e0 0x7f457419bd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:55.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 >> 172.21.15.28:0/991029730 conn(0x7f4574100c60 msgr2=0x7f4574109be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:55.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 shutdown_connections 2024-05-20T20:04:55.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:55.017+0000 7f4573200700 1 -- 172.21.15.28:0/991029730 wait complete. 2024-05-20T20:04:55.795 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:55 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/991029730' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-05-20T20:04:55.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:55 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/991029730' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-05-20T20:04:55.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:55 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/991029730' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-05-20T20:04:55.880 INFO:teuthology.run_tasks:Running task fs.ready... 2024-05-20T20:04:55.891 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:04:56.380 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:04:56.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:56 smithi028 ceph-mon[31178]: pgmap v426: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:56.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:56 smithi154 ceph-mon[36314]: pgmap v426: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:56.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:56 smithi033 ceph-mon[36179]: pgmap v426: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:04:58.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.626+0000 7f007f200700 1 -- 172.21.15.28:0/369172557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0078106140 msgr2=0x7f0078112650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:58.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.626+0000 7f007f200700 1 --2- 172.21.15.28:0/369172557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0078106140 0x7f0078112650 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f0070009f90 tx=0x7f007000b330 comp rx=0 tx=0).stop 2024-05-20T20:04:58.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.627+0000 7f007f200700 1 -- 172.21.15.28:0/369172557 shutdown_connections 2024-05-20T20:04:58.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.627+0000 7f007f200700 1 --2- 172.21.15.28:0/369172557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0078106140 0x7f0078112650 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.627+0000 7f007f200700 1 --2- 172.21.15.28:0/369172557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f007810b780 0x7f007810bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.627+0000 7f007f200700 1 --2- 172.21.15.28:0/369172557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0078105780 0x7f0078105c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.627+0000 7f007f200700 1 -- 172.21.15.28:0/369172557 >> 172.21.15.28:0/369172557 conn(0x7f0078101270 msgr2=0x7f0078103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:58.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.627+0000 7f007f200700 1 -- 172.21.15.28:0/369172557 shutdown_connections 2024-05-20T20:04:58.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.627+0000 7f007f200700 1 -- 172.21.15.28:0/369172557 wait complete. 2024-05-20T20:04:58.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.628+0000 7f007f200700 1 Processor -- start 2024-05-20T20:04:58.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.628+0000 7f007f200700 1 -- start start 2024-05-20T20:04:58.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f007f200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0078105780 0x7f007819be80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f007f200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0078106140 0x7f007819c3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f007f200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f007810b780 0x7f00781a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f007f200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f00781159b0 con 0x7f0078106140 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f007f200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0078115830 con 0x7f0078105780 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f007f200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0078115b30 con 0x7f007810b780 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f0077400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0078106140 0x7f007819c3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f0077e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0078105780 0x7f007819be80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f0077400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0078106140 0x7f007819c3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38494/0 (socket says 172.21.15.28:38494) 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f0077400700 1 -- 172.21.15.28:0/261872646 learned_addr learned my addr 172.21.15.28:0/261872646 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.629+0000 7f007d200700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f007810b780 0x7f00781a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:58.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0077e00700 1 -- 172.21.15.28:0/261872646 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f007810b780 msgr2=0x7f00781a0750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:58.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0077e00700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f007810b780 0x7f00781a0750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0077e00700 1 -- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0078106140 msgr2=0x7f007819c3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:58.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0077e00700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0078106140 0x7f007819c3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0077e00700 1 -- 172.21.15.28:0/261872646 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f00781a0e90 con 0x7f0078105780 2024-05-20T20:04:58.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0077e00700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0078105780 0x7f007819be80 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f006800eb90 tx=0x7f006800eea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:58.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0074c00700 1 -- 172.21.15.28:0/261872646 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f006800cc90 con 0x7f0078105780 2024-05-20T20:04:58.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f0074c00700 1 -- 172.21.15.28:0/261872646 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f006800ce30 con 0x7f0078105780 2024-05-20T20:04:58.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.630+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f00781a1180 con 0x7f0078105780 2024-05-20T20:04:58.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.631+0000 7f0074c00700 1 -- 172.21.15.28:0/261872646 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0068010430 con 0x7f0078105780 2024-05-20T20:04:58.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.631+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f00781a8a00 con 0x7f0078105780 2024-05-20T20:04:58.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.632+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f007806a230 con 0x7f0078105780 2024-05-20T20:04:58.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.634+0000 7f0074c00700 1 -- 172.21.15.28:0/261872646 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f0068003ef0 con 0x7f0078105780 2024-05-20T20:04:58.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.635+0000 7f0074c00700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f005c0b2900 0x7f005c0b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:04:58.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.636+0000 7f0074c00700 1 -- 172.21.15.28:0/261872646 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f00680e2450 con 0x7f0078105780 2024-05-20T20:04:58.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.636+0000 7f0077400700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f005c0b2900 0x7f005c0b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:04:58.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.636+0000 7f0077400700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f005c0b2900 0x7f005c0b4dc0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f007810ab50 tx=0x7f0064009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:04:58.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.640+0000 7f0074c00700 1 -- 172.21.15.28:0/261872646 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f00680aa130 con 0x7f0078105780 2024-05-20T20:04:58.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:58 smithi028 ceph-mon[31178]: pgmap v427: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:58.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:58 smithi154 ceph-mon[36314]: pgmap v427: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:58.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:58 smithi033 ceph-mon[36179]: pgmap v427: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:04:58.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.936+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f007819d1e0 con 0x7f0078105780 2024-05-20T20:04:58.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.937+0000 7f0074c00700 1 -- 172.21.15.28:0/261872646 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 28 v28) ==== 94+0+11925 (secure 0 0 0) 0x7f00680a9a30 con 0x7f0078105780 2024-05-20T20:04:58.939 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:04:58.939 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":28,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":28,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:01:50.441739+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24475,24482,24493,24500,24542]},"id":1}]} 2024-05-20T20:04:58.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f005c0b2900 msgr2=0x7f005c0b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:58.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f005c0b2900 0x7f005c0b4dc0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f007810ab50 tx=0x7f0064009260 comp rx=0 tx=0).stop 2024-05-20T20:04:58.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0078105780 msgr2=0x7f007819be80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:04:58.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0078105780 0x7f007819be80 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f006800eb90 tx=0x7f006800eea0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 shutdown_connections 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f005c0b2900 0x7f005c0b4dc0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f007810b780 0x7f00781a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0078105780 0x7f007819be80 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 --2- 172.21.15.28:0/261872646 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0078106140 0x7f007819c3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 >> 172.21.15.28:0/261872646 conn(0x7f0078101270 msgr2=0x7f0078101810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 shutdown_connections 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:04:58.941+0000 7f007f200700 1 -- 172.21.15.28:0/261872646 wait complete. 2024-05-20T20:04:58.943 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 28 2024-05-20T20:04:59.749 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:04:59.831 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:04:59 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/261872646' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:59.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:04:59 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/261872646' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:04:59.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:04:59 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/261872646' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:00.216 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:00.740 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:00 smithi028 ceph-mon[31178]: pgmap v428: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:00.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:00 smithi154 ceph-mon[36314]: pgmap v428: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:00.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:00 smithi033 ceph-mon[36179]: pgmap v428: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:02.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.447+0000 7f843be00700 1 -- 172.21.15.28:0/2296654838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 msgr2=0x7f843c10f770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.447+0000 7f843be00700 1 --2- 172.21.15.28:0/2296654838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 0x7f843c10f770 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f84240098f0 tx=0x7f8424009c00 comp rx=0 tx=0).stop 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.448+0000 7f843be00700 1 -- 172.21.15.28:0/2296654838 shutdown_connections 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.448+0000 7f843be00700 1 --2- 172.21.15.28:0/2296654838 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c10fcb0 0x7f843c112120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.448+0000 7f843be00700 1 --2- 172.21.15.28:0/2296654838 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f843c075970 0x7f843c075d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.448+0000 7f843be00700 1 --2- 172.21.15.28:0/2296654838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 0x7f843c10f770 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.448+0000 7f843be00700 1 -- 172.21.15.28:0/2296654838 >> 172.21.15.28:0/2296654838 conn(0x7f843c0fefc0 msgr2=0x7f843c1013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.448+0000 7f843be00700 1 -- 172.21.15.28:0/2296654838 shutdown_connections 2024-05-20T20:05:02.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.448+0000 7f843be00700 1 -- 172.21.15.28:0/2296654838 wait complete. 2024-05-20T20:05:02.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.449+0000 7f843be00700 1 Processor -- start 2024-05-20T20:05:02.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.449+0000 7f843be00700 1 -- start start 2024-05-20T20:05:02.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c075970 0x7f843c19be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:02.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 0x7f843c19c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:02.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f843c10fcb0 0x7f843c1a06e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f843c115880 con 0x7f843c10fcb0 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843be00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f843c115700 con 0x7f843c076290 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843be00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f843c115a00 con 0x7f843c075970 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c075970 0x7f843c19be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f843c10fcb0 0x7f843c1a06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c075970 0x7f843c19be10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:36148/0 (socket says 172.21.15.28:36148) 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f843aa00700 1 -- 172.21.15.28:0/1177969055 learned_addr learned my addr 172.21.15.28:0/1177969055 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:02.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.450+0000 7f8432000700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 0x7f843c19c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f843aa00700 1 -- 172.21.15.28:0/1177969055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 msgr2=0x7f843c19c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f843aa00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 0x7f843c19c350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f843aa00700 1 -- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f843c10fcb0 msgr2=0x7f843c1a06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f843aa00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f843c10fcb0 0x7f843c1a06e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f843aa00700 1 -- 172.21.15.28:0/1177969055 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f843c1a0e20 con 0x7f843c075970 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f843aa00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c075970 0x7f843c19be10 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f842c00e9c0 tx=0x7f842c00ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f8440a00700 1 -- 172.21.15.28:0/1177969055 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f842c004a30 con 0x7f843c075970 2024-05-20T20:05:02.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.451+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f843c1a1110 con 0x7f843c075970 2024-05-20T20:05:02.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.452+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f843c1a8990 con 0x7f843c075970 2024-05-20T20:05:02.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.452+0000 7f8440a00700 1 -- 172.21.15.28:0/1177969055 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f842c00cc70 con 0x7f843c075970 2024-05-20T20:05:02.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.452+0000 7f8440a00700 1 -- 172.21.15.28:0/1177969055 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f842c005430 con 0x7f843c075970 2024-05-20T20:05:02.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.453+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f843c06a230 con 0x7f843c075970 2024-05-20T20:05:02.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.455+0000 7f8440a00700 1 -- 172.21.15.28:0/1177969055 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f842c0055f0 con 0x7f843c075970 2024-05-20T20:05:02.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.457+0000 7f8440a00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84200b2880 0x7f84200b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:02.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.457+0000 7f8440a00700 1 -- 172.21.15.28:0/1177969055 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f842c0e2290 con 0x7f843c075970 2024-05-20T20:05:02.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.457+0000 7f8432000700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84200b2880 0x7f84200b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:02.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.458+0000 7f8432000700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84200b2880 0x7f84200b4d40 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f84240098c0 tx=0x7f8424019040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:02.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.459+0000 7f8440a00700 1 -- 172.21.15.28:0/1177969055 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f842c00ce20 con 0x7f843c075970 2024-05-20T20:05:02.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.745+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f843c19d170 con 0x7f843c075970 2024-05-20T20:05:02.746 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:02 smithi028 ceph-mon[31178]: pgmap v429: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:02.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.746+0000 7f8440a00700 1 -- 172.21.15.28:0/1177969055 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 28 v28) ==== 94+0+11925 (secure 0 0 0) 0x7f842c0a9f70 con 0x7f843c075970 2024-05-20T20:05:02.747 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:02.747 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":28,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":28,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:01:50.441739+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24475,24482,24493,24500,24542]},"id":1}]} 2024-05-20T20:05:02.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.749+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84200b2880 msgr2=0x7f84200b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:02.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.749+0000 7f843be00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84200b2880 0x7f84200b4d40 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f84240098c0 tx=0x7f8424019040 comp rx=0 tx=0).stop 2024-05-20T20:05:02.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.749+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c075970 msgr2=0x7f843c19be10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:02.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.749+0000 7f843be00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c075970 0x7f843c19be10 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f842c00e9c0 tx=0x7f842c00ecd0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 shutdown_connections 2024-05-20T20:05:02.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f84200b2880 0x7f84200b4d40 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f843c10fcb0 0x7f843c1a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f843c075970 0x7f843c19be10 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 --2- 172.21.15.28:0/1177969055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f843c076290 0x7f843c19c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:02.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 >> 172.21.15.28:0/1177969055 conn(0x7f843c0fefc0 msgr2=0x7f843c0ffdd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:02.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 shutdown_connections 2024-05-20T20:05:02.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:02.750+0000 7f843be00700 1 -- 172.21.15.28:0/1177969055 wait complete. 2024-05-20T20:05:02.753 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 28 2024-05-20T20:05:02.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:02 smithi154 ceph-mon[36314]: pgmap v429: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:02.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:02 smithi033 ceph-mon[36179]: pgmap v429: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:03.574 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:05:03.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:03 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1177969055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:03.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:03 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1177969055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:03.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:03 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1177969055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:04.050 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:04.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:04 smithi028 ceph-mon[31178]: pgmap v430: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:04.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:04 smithi154 ceph-mon[36314]: pgmap v430: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:04.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:04 smithi033 ceph-mon[36179]: pgmap v430: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:06.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 -- 172.21.15.28:0/1676997340 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 msgr2=0x7f6004112390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 --2- 172.21.15.28:0/1676997340 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 0x7f6004112390 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f5ffc00b0a0 tx=0x7f5ffc00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 -- 172.21.15.28:0/1676997340 shutdown_connections 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 --2- 172.21.15.28:0/1676997340 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 0x7f6004112390 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 --2- 172.21.15.28:0/1676997340 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 0x7f6004075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 --2- 172.21.15.28:0/1676997340 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6004076090 0x7f600410fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 -- 172.21.15.28:0/1676997340 >> 172.21.15.28:0/1676997340 conn(0x7f600407a620 msgr2=0x7f600407aa30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 -- 172.21.15.28:0/1676997340 shutdown_connections 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.302+0000 7f6009a00700 1 -- 172.21.15.28:0/1676997340 wait complete. 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.303+0000 7f6009a00700 1 Processor -- start 2024-05-20T20:05:06.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6009a00700 1 -- start start 2024-05-20T20:05:06.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6009a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 0x7f600419be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:06.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6009a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6004076090 0x7f600419c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:06.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6009a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 0x7f60041a06b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:06.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6009a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6004115920 con 0x7f600410ffa0 2024-05-20T20:05:06.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6009a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f60041157a0 con 0x7f6004075770 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6009a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6004115aa0 con 0x7f6004076090 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6002a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 0x7f600419be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6003400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 0x7f60041a06b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6002a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 0x7f600419be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:50374/0 (socket says 172.21.15.28:50374) 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.304+0000 7f6002a00700 1 -- 172.21.15.28:0/1883948218 learned_addr learned my addr 172.21.15.28:0/1883948218 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f5ffa000700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6004076090 0x7f600419c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f6002a00700 1 -- 172.21.15.28:0/1883948218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6004076090 msgr2=0x7f600419c370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f6002a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6004076090 0x7f600419c370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f6002a00700 1 -- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 msgr2=0x7f60041a06b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f6002a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 0x7f60041a06b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f6002a00700 1 -- 172.21.15.28:0/1883948218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f60041a0df0 con 0x7f6004075770 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f6002a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 0x7f600419be30 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5ff400d980 tx=0x7f5ff400dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:06.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f5ffbe00700 1 -- 172.21.15.28:0/1883948218 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ff4016070 con 0x7f6004075770 2024-05-20T20:05:06.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.305+0000 7f5ffbe00700 1 -- 172.21.15.28:0/1883948218 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5ff4004830 con 0x7f6004075770 2024-05-20T20:05:06.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.306+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f60041a10e0 con 0x7f6004075770 2024-05-20T20:05:06.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.306+0000 7f5ffbe00700 1 -- 172.21.15.28:0/1883948218 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ff4005540 con 0x7f6004075770 2024-05-20T20:05:06.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.306+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f60041a8960 con 0x7f6004075770 2024-05-20T20:05:06.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.307+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f600406a230 con 0x7f6004075770 2024-05-20T20:05:06.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.309+0000 7f5ffbe00700 1 -- 172.21.15.28:0/1883948218 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f5ff4004220 con 0x7f6004075770 2024-05-20T20:05:06.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.310+0000 7f5ffbe00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5fe80b2880 0x7f5fe80b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:06.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.311+0000 7f5ffbe00700 1 -- 172.21.15.28:0/1883948218 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f5ff40e25d0 con 0x7f6004075770 2024-05-20T20:05:06.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.311+0000 7f5ffa000700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5fe80b2880 0x7f5fe80b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:06.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.312+0000 7f5ffa000700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5fe80b2880 0x7f5fe80b4d40 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f6004111c30 tx=0x7f5fec009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:06.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.313+0000 7f5ffbe00700 1 -- 172.21.15.28:0/1883948218 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f5ff401f020 con 0x7f6004075770 2024-05-20T20:05:06.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.559+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f600402e560 con 0x7f6004075770 2024-05-20T20:05:06.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.561+0000 7f5ffbe00700 1 -- 172.21.15.28:0/1883948218 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) ==== 74+0+22571 (secure 0 0 0) 0x7f5ff40aa2b0 con 0x7f6004075770 2024-05-20T20:05:06.562 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:06.562 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":80,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:02:04.742448+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.564+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5fe80b2880 msgr2=0x7f5fe80b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.564+0000 7f6009a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5fe80b2880 0x7f5fe80b4d40 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f6004111c30 tx=0x7f5fec009260 comp rx=0 tx=0).stop 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.564+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 msgr2=0x7f600419be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.564+0000 7f6009a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 0x7f600419be30 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f5ff400d980 tx=0x7f5ff400dc90 comp rx=0 tx=0).stop 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 shutdown_connections 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f5fe80b2880 0x7f5fe80b4d40 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f600410ffa0 0x7f60041a06b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6004075770 0x7f600419be30 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 --2- 172.21.15.28:0/1883948218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6004076090 0x7f600419c370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:06.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 >> 172.21.15.28:0/1883948218 conn(0x7f600407a620 msgr2=0x7f60041007b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:06.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 shutdown_connections 2024-05-20T20:05:06.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:06.565+0000 7f6009a00700 1 -- 172.21.15.28:0/1883948218 wait complete. 2024-05-20T20:05:06.722 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:06 smithi028 ceph-mon[31178]: pgmap v431: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:06.723 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:06 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1883948218' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:06.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:06 smithi154 ceph-mon[36314]: pgmap v431: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:06.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:06 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1883948218' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:06.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:06 smithi033 ceph-mon[36179]: pgmap v431: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:06.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:06 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1883948218' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:07.514 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 28, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-05-20T20:01:26.231967+0000', 'modified': '2024-05-20T20:01:50.441739+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 5, 'in': [0, 1, 2, 3, 4], 'up': {'mds_0': 24542, 'mds_1': 24500, 'mds_2': 24493, 'mds_3': 24482, 'mds_4': 24475}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_14400': {'gid': 14400, 'name': 'a', 'rank': 4, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.28:6841/1736910968', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.28:6840', 'nonce': 1736910968}, {'type': 'v1', 'addr': '172.21.15.28:6841', 'nonce': 1736910968}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24427': {'gid': 24427, 'name': 'l', 'rank': 3, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.154:6833/2654302434', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6832', 'nonce': 2654302434}, {'type': 'v1', 'addr': '172.21.15.154:6833', 'nonce': 2654302434}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24442': {'gid': 24442, 'name': 'j', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.28:6835/34026174', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.28:6834', 'nonce': 34026174}, {'type': 'v1', 'addr': '172.21.15.28:6835', 'nonce': 34026174}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24455': {'gid': 24455, 'name': 'k', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.33:6833/260147664', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6832', 'nonce': 260147664}, {'type': 'v1', 'addr': '172.21.15.33:6833', 'nonce': 260147664}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24463': {'gid': 24463, 'name': 'f', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.154:6835/3965936826', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6834', 'nonce': 3965936826}, {'type': 'v1', 'addr': '172.21.15.154:6835', 'nonce': 3965936826}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24475': {'gid': 24475, 'name': 'c', 'rank': 4, 'incarnation': 25, 'state': 'up:active', 'state_seq': 17, 'addr': '172.21.15.154:6837/2808233682', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6836', 'nonce': 2808233682}, {'type': 'v1', 'addr': '172.21.15.154:6837', 'nonce': 2808233682}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24482': {'gid': 24482, 'name': 'e', 'rank': 3, 'incarnation': 23, 'state': 'up:active', 'state_seq': 18, 'addr': '172.21.15.33:6835/4076526106', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6834', 'nonce': 4076526106}, {'type': 'v1', 'addr': '172.21.15.33:6835', 'nonce': 4076526106}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24493': {'gid': 24493, 'name': 'i', 'rank': 2, 'incarnation': 21, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.154:6839/1291909351', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6838', 'nonce': 1291909351}, {'type': 'v1', 'addr': '172.21.15.154:6839', 'nonce': 1291909351}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24500': {'gid': 24500, 'name': 'h', 'rank': 1, 'incarnation': 19, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.33:6837/4183908883', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6836', 'nonce': 4183908883}, {'type': 'v1', 'addr': '172.21.15.33:6837', 'nonce': 4183908883}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24542': {'gid': 24542, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 7, 'addr': '172.21.15.33:6839/2646117588', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6838', 'nonce': 2646117588}, {'type': 'v1', 'addr': '172.21.15.33:6839', 'nonce': 2646117588}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 24542, 'qdb_cluster': [24475, 24482, 24493, 24500, 24542]} 2024-05-20T20:05:07.514 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 5/5 2024-05-20T20:05:07.514 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.c status 2024-05-20T20:05:07.985 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:08.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:08 smithi028 ceph-mon[31178]: pgmap v432: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:05:08.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:08 smithi154 ceph-mon[36314]: pgmap v432: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:05:08.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:08 smithi033 ceph-mon[36179]: pgmap v432: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:05:10.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 -- 172.21.15.28:0/3646871671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 msgr2=0x7fc98c1141d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 --2- 172.21.15.28:0/3646871671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 0x7fc98c1141d0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fc978009990 tx=0x7fc978009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 -- 172.21.15.28:0/3646871671 shutdown_connections 2024-05-20T20:05:10.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 --2- 172.21.15.28:0/3646871671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c114710 0x7fc98c116b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 --2- 172.21.15.28:0/3646871671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc98c075770 0x7fc98c075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 --2- 172.21.15.28:0/3646871671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 0x7fc98c1141d0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 -- 172.21.15.28:0/3646871671 >> 172.21.15.28:0/3646871671 conn(0x7fc98c07a6a0 msgr2=0x7fc98c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:10.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 -- 172.21.15.28:0/3646871671 shutdown_connections 2024-05-20T20:05:10.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.335+0000 7fc994000700 1 -- 172.21.15.28:0/3646871671 wait complete. 2024-05-20T20:05:10.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.336+0000 7fc994000700 1 Processor -- start 2024-05-20T20:05:10.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.336+0000 7fc994000700 1 -- start start 2024-05-20T20:05:10.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc994000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c075770 0x7fc98c1a0280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc994000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 0x7fc98c1a07c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc994000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc98c114710 0x7fc98c1a4b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc994000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc98c119d50 con 0x7fc98c076090 2024-05-20T20:05:10.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc994000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc98c119bd0 con 0x7fc98c114710 2024-05-20T20:05:10.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc994000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc98c119ed0 con 0x7fc98c075770 2024-05-20T20:05:10.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc991600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c075770 0x7fc98c1a0280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc992000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc98c114710 0x7fc98c1a4b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc990c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 0x7fc98c1a07c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc991600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c075770 0x7fc98c1a0280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58688/0 (socket says 172.21.15.28:58688) 2024-05-20T20:05:10.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.337+0000 7fc991600700 1 -- 172.21.15.28:0/1413550341 learned_addr learned my addr 172.21.15.28:0/1413550341 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:10.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc991600700 1 -- 172.21.15.28:0/1413550341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc98c114710 msgr2=0x7fc98c1a4b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc991600700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc98c114710 0x7fc98c1a4b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc991600700 1 -- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 msgr2=0x7fc98c1a07c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc991600700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 0x7fc98c1a07c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc991600700 1 -- 172.21.15.28:0/1413550341 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc98c1a5290 con 0x7fc98c075770 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc990c00700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 0x7fc98c1a07c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc991600700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c075770 0x7fc98c1a0280 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fc980007f10 tx=0x7fc98000d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc98a000700 1 -- 172.21.15.28:0/1413550341 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc98000ddd0 con 0x7fc98c075770 2024-05-20T20:05:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc98c1a5580 con 0x7fc98c075770 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.338+0000 7fc98a000700 1 -- 172.21.15.28:0/1413550341 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc980004020 con 0x7fc98c075770 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.339+0000 7fc98a000700 1 -- 172.21.15.28:0/1413550341 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc980015620 con 0x7fc98c075770 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.339+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc98c1ace00 con 0x7fc98c075770 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.342+0000 7fc98a000700 1 -- 172.21.15.28:0/1413550341 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fc980003a00 con 0x7fc98c075770 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.343+0000 7fc99e963b80 1 Processor -- start 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.343+0000 7fc99e963b80 1 -- start start 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6be60 0x55813ac6c240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6c780 0x55813ac66210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac66830 0x55813ac66c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc99e963b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55813abc4df0 con 0x55813ac6be60 2024-05-20T20:05:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc99e963b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55813abb4990 con 0x55813ac66830 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc99e963b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55813abb89d0 con 0x55813ac6c780 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc98a000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9740b2880 0x7fc9740b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc990c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac66830 0x55813ac66c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc991600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6c780 0x55813ac66210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc990c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac66830 0x55813ac66c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53988/0 (socket says 172.21.15.28:53988) 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc992000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6be60 0x55813ac6c240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.344+0000 7fc990c00700 1 -- 172.21.15.28:0/1433525320 learned_addr learned my addr 172.21.15.28:0/1433525320 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc990c00700 1 -- 172.21.15.28:0/1433525320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6c780 msgr2=0x55813ac66210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc990c00700 1 --2- 172.21.15.28:0/1433525320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6c780 0x55813ac66210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc990c00700 1 -- 172.21.15.28:0/1433525320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6be60 msgr2=0x55813ac6c240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc990c00700 1 --2- 172.21.15.28:0/1433525320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6be60 0x55813ac6c240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc990c00700 1 -- 172.21.15.28:0/1433525320 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55813ab7d3d0 con 0x55813ac66830 2024-05-20T20:05:10.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc98a000700 1 -- 172.21.15.28:0/1413550341 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fc9800e1dc0 con 0x7fc98c075770 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc992000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9740b2880 0x7fc9740b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc990c00700 1 --2- 172.21.15.28:0/1433525320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac66830 0x55813ac66c90 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fc978015000 tx=0x7fc978005bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9b58ced85986fef9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.345+0000 7fc97ea00700 1 -- 172.21.15.28:0/1433525320 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc97800bd80 con 0x55813ac66830 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.346+0000 7fc97ea00700 1 -- 172.21.15.28:0/1433525320 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc978026070 con 0x55813ac66830 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.346+0000 7fc97ea00700 1 -- 172.21.15.28:0/1433525320 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc978005660 con 0x55813ac66830 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.346+0000 7fc99e963b80 1 -- 172.21.15.28:0/1433525320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac66830 msgr2=0x55813ac66c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.346+0000 7fc992000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9740b2880 0x7fc9740b4d40 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fc98c06caa0 tx=0x7fc984004750 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.346+0000 7fc99e963b80 1 --2- 172.21.15.28:0/1433525320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac66830 0x55813ac66c90 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7fc978015000 tx=0x7fc978005bc0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.346+0000 7fc99e963b80 1 -- 172.21.15.28:0/1433525320 shutdown_connections 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.346+0000 7fc99e963b80 1 --2- 172.21.15.28:0/1433525320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac66830 0x55813ac66c90 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.347+0000 7fc99e963b80 1 --2- 172.21.15.28:0/1433525320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6be60 0x55813ac6c240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.347+0000 7fc99e963b80 1 --2- 172.21.15.28:0/1433525320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6c780 0x55813ac66210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.347+0000 7fc99e963b80 1 -- 172.21.15.28:0/1433525320 >> 172.21.15.28:0/1433525320 conn(0x55813ab94b90 msgr2=0x55813ac60fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.347+0000 7fc99e963b80 1 -- 172.21.15.28:0/1433525320 shutdown_connections 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.347+0000 7fc99e963b80 1 -- 172.21.15.28:0/1433525320 wait complete. 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.347+0000 7fc99e963b80 1 -- start start 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.347+0000 7fc99e963b80 1 Processor -- start 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac66830 0x55813ac723d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6be60 0x55813ac6f7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6c780 0x55813ac6fd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc99e963b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55813aabaf50 con 0x55813ac66830 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc99e963b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55813ab55710 con 0x55813ac6c780 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc99e963b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55813ac73190 con 0x55813ac6be60 2024-05-20T20:05:10.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc99e963b80 10 client.0 _ll_register_callbacks cb 0x55813ab5ec20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc991600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac66830 0x55813ac723d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc991600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac66830 0x55813ac723d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43666/0 (socket says 172.21.15.28:43666) 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc991600700 1 -- 172.21.15.28:0/8806527 learned_addr learned my addr 172.21.15.28:0/8806527 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.348+0000 7fc990c00700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6be60 0x55813ac6f7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc992000700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6c780 0x55813ac6fd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc991600700 1 -- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6be60 msgr2=0x55813ac6f7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc991600700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6be60 0x55813ac6f7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc991600700 1 -- 172.21.15.28:0/8806527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6c780 msgr2=0x55813ac6fd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc991600700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6c780 0x55813ac6fd00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc991600700 1 -- 172.21.15.28:0/8806527 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55813ab04880 con 0x55813ac66830 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc992000700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6c780 0x55813ac6fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc991600700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac66830 0x55813ac723d0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fc9800df880 tx=0x7fc9800a8f60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d3bd47c3907332b1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:10.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc973400700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc973400700 1 -- 172.21.15.28:0/8806527 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9800a93a0 con 0x55813ac66830 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc973400700 1 -- 172.21.15.28:0/8806527 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fc9800b1f80 con 0x55813ac66830 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.349+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc978026070 con 0x55813ac66830 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc973400700 1 -- 172.21.15.28:0/8806527 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fc9800ad750 con 0x55813ac66830 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 10 client.14898 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55813ab85010 con 0x55813ac66830 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc973400700 1 -- 172.21.15.28:0/8806527 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc978026070 con 0x55813ac66830 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc973400700 1 -- 172.21.15.28:0/8806527 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fc9800a4e30 con 0x55813ac66830 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 10 client.14898 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 10 client.14898 resolve_mds: resolved name 'c' to daemon mds.c 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 20 client.14898 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813ac7fcc0 0x55813ac81130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 4 client.14898 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.350+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55813abc6d40 con 0x55813ac7fcc0 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.351+0000 7fc990c00700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813ac7fcc0 0x55813ac81130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.351+0000 7fc990c00700 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813ac7fcc0 0x55813ac81130 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=ade8ae5ad67db6e2 server_cookie=cedf7bd228f9a564 in_seq=0 out_seq=0 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.351+0000 7fc973400700 10 client.14898 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.352+0000 7fc973400700 1 -- 172.21.15.28:0/8806527 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55813abc6d40 con 0x55813ac7fcc0 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.352+0000 7fc973400700 10 client.14898 handle_command_reply: tid=0 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.352+0000 7fc99e963b80 1 client.14898 shutdown 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac66830 msgr2=0x55813ac723d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac66830 0x55813ac723d0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fc9800df880 tx=0x7fc9800a8f60 comp rx=0 tx=0).stop 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 shutdown_connections 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6c780 0x55813ac6fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac66830 0x55813ac723d0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813ac7fcc0 0x55813ac81130 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 --2- 172.21.15.28:0/8806527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac6be60 0x55813ac6f7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 >> 172.21.15.28:0/8806527 conn(0x55813ab94b90 msgr2=0x55813ac65200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:10.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc973400700 0 client.14898 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:05:10.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 shutdown_connections 2024-05-20T20:05:10.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 1 -- 172.21.15.28:0/8806527 wait complete. 2024-05-20T20:05:10.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.353+0000 7fc99e963b80 20 client.14898 trim_cache size 0 max 16384 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.376+0000 7fc99e963b80 1 Processor -- start 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc99e963b80 1 -- start start 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac66210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac71c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac6e0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc99e963b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55813ac7f470 con 0x55813ac6c780 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc99e963b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55813ac7fb40 con 0x55813ac6be60 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc99e963b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55813ac80210 con 0x55813ac66830 2024-05-20T20:05:10.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc992000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac6e0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc990c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac71c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc991600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac66210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc991600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac66210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58718/0 (socket says 172.21.15.28:58718) 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc990c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac71c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54004/0 (socket says 172.21.15.28:54004) 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc991600700 1 -- 172.21.15.28:0/3722261984 learned_addr learned my addr 172.21.15.28:0/3722261984 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.377+0000 7fc992000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac6e0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43672/0 (socket says 172.21.15.28:43672) 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc991600700 1 -- 172.21.15.28:0/3722261984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 msgr2=0x55813ac71c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc991600700 1 --2- 172.21.15.28:0/3722261984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac71c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc991600700 1 -- 172.21.15.28:0/3722261984 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 msgr2=0x55813ac6e0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc991600700 1 --2- 172.21.15.28:0/3722261984 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac6e0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc991600700 1 -- 172.21.15.28:0/3722261984 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55813ac6dd30 con 0x55813ac66830 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc991600700 1 --2- 172.21.15.28:0/3722261984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac66210 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fc980004900 tx=0x7fc9800a8df0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2610901f7a8a71f3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:10.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.378+0000 7fc97e000700 1 -- 172.21.15.28:0/3722261984 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9800b2200 con 0x55813ac66830 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc97e000700 1 -- 172.21.15.28:0/3722261984 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc9800b1f80 con 0x55813ac66830 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc97e000700 1 -- 172.21.15.28:0/3722261984 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9800a4f50 con 0x55813ac66830 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 -- 172.21.15.28:0/3722261984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 msgr2=0x55813ac66210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 --2- 172.21.15.28:0/3722261984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac66210 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fc980004900 tx=0x7fc9800a8df0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 -- 172.21.15.28:0/3722261984 shutdown_connections 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 --2- 172.21.15.28:0/3722261984 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac6e0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 --2- 172.21.15.28:0/3722261984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac66210 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 --2- 172.21.15.28:0/3722261984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac71c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 -- 172.21.15.28:0/3722261984 >> 172.21.15.28:0/3722261984 conn(0x55813ab94b90 msgr2=0x55813ac69970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.379+0000 7fc99e963b80 1 -- 172.21.15.28:0/3722261984 shutdown_connections 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.380+0000 7fc99e963b80 1 -- 172.21.15.28:0/3722261984 wait complete. 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.380+0000 7fc99e963b80 1 -- start start 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.380+0000 7fc99e963b80 1 Processor -- start 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac8a950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac85560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc99e963b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac85aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc99e963b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55813ac80fb0 con 0x55813ac6c780 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc99e963b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55813ac808e0 con 0x55813ac6be60 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc99e963b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55813ac81680 con 0x55813ac66830 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc99e963b80 10 client.0 _ll_register_callbacks cb 0x55813ab5ec20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc992000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac85aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc991600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac8a950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc992000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac85aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43684/0 (socket says 172.21.15.28:43684) 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc992000700 1 -- 172.21.15.28:0/2385198855 learned_addr learned my addr 172.21.15.28:0/2385198855 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc990c00700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac85560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.381+0000 7fc992000700 1 -- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 msgr2=0x55813ac8a950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc992000700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac8a950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc992000700 1 -- 172.21.15.28:0/2385198855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 msgr2=0x55813ac85560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc992000700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac85560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc992000700 1 -- 172.21.15.28:0/2385198855 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55813ac85fe0 con 0x55813ac6c780 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc991600700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac8a950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc990c00700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac85560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc992000700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac85aa0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fc984012ee0 tx=0x7fc9840047c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dfbbcce1c6245550 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc973400700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:05:10.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc973400700 1 -- 172.21.15.28:0/2385198855 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc984004830 con 0x55813ac6c780 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc973400700 1 -- 172.21.15.28:0/2385198855 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fc9840049d0 con 0x55813ac6c780 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.382+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc9800b1f80 con 0x55813ac6c780 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc973400700 1 -- 172.21.15.28:0/2385198855 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fc984004bd0 con 0x55813ac6c780 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 10 client.14904 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55813acec220 con 0x55813ac6c780 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc973400700 1 -- 172.21.15.28:0/2385198855 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc9800b1f80 con 0x55813ac6c780 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc973400700 1 -- 172.21.15.28:0/2385198855 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fc984031020 con 0x55813ac6c780 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 10 client.14904 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 10 client.14904 resolve_mds: resolved name 'c' to daemon mds.c 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 20 client.14904 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813acf0e80 0x55813acefb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 4 client.14904 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:05:10.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.383+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "status"}) -- 0x55813abc0520 con 0x55813acf0e80 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.384+0000 7fc991600700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813acf0e80 0x55813acefb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.384+0000 7fc991600700 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813acf0e80 0x55813acefb90 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=8b69e22afb59bbfa server_cookie=4d61a8195740f9d8 in_seq=0 out_seq=0 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.384+0000 7fc973400700 10 client.14904 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.384+0000 7fc973400700 1 -- 172.21.15.28:0/2385198855 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x55813abc0520 con 0x55813acf0e80 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.384+0000 7fc973400700 10 client.14904 handle_command_reply: tid=0 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.384+0000 7fc99e963b80 1 client.14904 shutdown 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 msgr2=0x55813ac85aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac85aa0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fc984012ee0 tx=0x7fc9840047c0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 shutdown_connections 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55813ac6c780 0x55813ac85aa0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55813acf0e80 0x55813acefb90 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55813ac66830 0x55813ac8a950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 --2- 172.21.15.28:0/2385198855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55813ac6be60 0x55813ac85560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc973400700 0 client.14904 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 >> 172.21.15.28:0/2385198855 conn(0x55813ab94b90 msgr2=0x55813ac6dff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 shutdown_connections 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 1 -- 172.21.15.28:0/2385198855 wait complete. 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.385+0000 7fc99e963b80 20 client.14904 trim_cache size 0 max 16384 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 4, 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24475, 2024-05-20T20:05:10.785 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 204.73956789299999, 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 265.850024394 2024-05-20T20:05:10.786 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9740b2880 msgr2=0x7fc9740b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9740b2880 0x7fc9740b4d40 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fc98c06caa0 tx=0x7fc984004750 comp rx=0 tx=0).stop 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c075770 msgr2=0x7fc98c1a0280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c075770 0x7fc98c1a0280 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fc980007f10 tx=0x7fc98000d3b0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 shutdown_connections 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fc9740b2880 0x7fc9740b4d40 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc98c114710 0x7fc98c1a4b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc98c075770 0x7fc98c1a0280 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 --2- 172.21.15.28:0/1413550341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc98c076090 0x7fc98c1a07c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 >> 172.21.15.28:0/1413550341 conn(0x7fc98c07a6a0 msgr2=0x7fc98c1101b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 shutdown_connections 2024-05-20T20:05:10.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:10.386+0000 7fc994000700 1 -- 172.21.15.28:0/1413550341 wait complete. 2024-05-20T20:05:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:10 smithi154 ceph-mon[36314]: pgmap v433: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:10.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:10 smithi033 ceph-mon[36179]: pgmap v433: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:11.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:10 smithi028 ceph-mon[31178]: pgmap v433: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:11.218 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.e status 2024-05-20T20:05:11.689 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:12.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:12 smithi154 ceph-mon[36314]: pgmap v434: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:12.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:12 smithi033 ceph-mon[36179]: pgmap v434: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:12.885 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:12 smithi028 ceph-mon[31178]: pgmap v434: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 -- 172.21.15.28:0/167769408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8108500 msgr2=0x7f16d81088e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 --2- 172.21.15.28:0/167769408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8108500 0x7f16d81088e0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f16c800b0a0 tx=0x7f16c800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 -- 172.21.15.28:0/167769408 shutdown_connections 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 --2- 172.21.15.28:0/167769408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f16d8103f90 0x7f16d8110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 --2- 172.21.15.28:0/167769408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8108500 0x7f16d81088e0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 --2- 172.21.15.28:0/167769408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 0x7f16d8103a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 -- 172.21.15.28:0/167769408 >> 172.21.15.28:0/167769408 conn(0x7f16d807a6a0 msgr2=0x7f16d807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 -- 172.21.15.28:0/167769408 shutdown_connections 2024-05-20T20:05:13.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.969+0000 7f16de400700 1 -- 172.21.15.28:0/167769408 wait complete. 2024-05-20T20:05:13.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.970+0000 7f16de400700 1 Processor -- start 2024-05-20T20:05:13.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.971+0000 7f16de400700 1 -- start start 2024-05-20T20:05:13.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.971+0000 7f16de400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 0x7f16d819beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.971+0000 7f16de400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8103f90 0x7f16d819c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.971+0000 7f16de400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f16d8108500 0x7f16d81a0780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.971+0000 7f16de400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f16d8113790 con 0x7f16d8103f90 2024-05-20T20:05:13.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.971+0000 7f16de400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f16d8113610 con 0x7f16d8108500 2024-05-20T20:05:13.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.971+0000 7f16de400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f16d8113910 con 0x7f16d81035f0 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 0x7f16d819beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 0x7f16d819beb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58752/0 (socket says 172.21.15.28:58752) 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 -- 172.21.15.28:0/783502800 learned_addr learned my addr 172.21.15.28:0/783502800 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16cea00700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8103f90 0x7f16d819c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7e00700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f16d8108500 0x7f16d81a0780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 -- 172.21.15.28:0/783502800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f16d8108500 msgr2=0x7f16d81a0780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f16d8108500 0x7f16d81a0780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 -- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8103f90 msgr2=0x7f16d819c3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8103f90 0x7f16d819c3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 -- 172.21.15.28:0/783502800 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f16d81a0ec0 con 0x7f16d81035f0 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16cea00700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8103f90 0x7f16d819c3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.972+0000 7f16d7400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 0x7f16d819beb0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f16c800b780 tx=0x7f16c8004d00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.973+0000 7f16d4c00700 1 -- 172.21.15.28:0/783502800 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16c8021070 con 0x7f16d81035f0 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.973+0000 7f16d4c00700 1 -- 172.21.15.28:0/783502800 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f16c8007e30 con 0x7f16d81035f0 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.973+0000 7f16d4c00700 1 -- 172.21.15.28:0/783502800 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16c8004750 con 0x7f16d81035f0 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.973+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f16d81a1150 con 0x7f16d81035f0 2024-05-20T20:05:13.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.973+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f16d81a8a30 con 0x7f16d81035f0 2024-05-20T20:05:13.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.977+0000 7f16d4c00700 1 -- 172.21.15.28:0/783502800 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f16c8007980 con 0x7f16d81035f0 2024-05-20T20:05:13.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.978+0000 7f16e8e43b80 1 Processor -- start 2024-05-20T20:05:13.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.978+0000 7f16d4c00700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f16bc0b2880 0x7f16bc0b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.978+0000 7f16e8e43b80 1 -- start start 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.978+0000 7f16d4c00700 1 -- 172.21.15.28:0/783502800 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f16c800f020 con 0x7f16d81035f0 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.978+0000 7f16d7e00700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f16bc0b2880 0x7f16bc0b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4b8e9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4b88ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b89570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ae6df0 con 0x5635b4b8e5c0 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ad6990 con 0x5635b4b890f0 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ada9d0 con 0x5635b4b8eee0 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4b8e9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4b8e9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43710/0 (socket says 172.21.15.28:43710) 2024-05-20T20:05:13.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 -- 172.21.15.28:0/3507960835 learned_addr learned my addr 172.21.15.28:0/3507960835 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7e00700 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b89570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16cea00700 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4b88ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 -- 172.21.15.28:0/3507960835 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 msgr2=0x5635b4b88ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4b88ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 -- 172.21.15.28:0/3507960835 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 msgr2=0x5635b4b89570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b89570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.979+0000 7f16d7400700 1 -- 172.21.15.28:0/3507960835 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5635b4a9f3d0 con 0x5635b4b8e5c0 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.980+0000 7f16d7e00700 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b89570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.980+0000 7f16d7e00700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f16bc0b2880 0x7f16bc0b4d40 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f16d0011440 tx=0x7f16d0011800 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:13.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.980+0000 7f16d7400700 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4b8e9a0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f16c80a7f80 tx=0x7f16c80099e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af3e269d75bc82f9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.980+0000 7f16ccc00700 1 -- 172.21.15.28:0/3507960835 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16c80aa820 con 0x5635b4b8e5c0 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.980+0000 7f16ccc00700 1 -- 172.21.15.28:0/3507960835 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f16c80ab410 con 0x5635b4b8e5c0 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.980+0000 7f16ccc00700 1 -- 172.21.15.28:0/3507960835 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16c8011e10 con 0x5635b4b8e5c0 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 -- 172.21.15.28:0/3507960835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 msgr2=0x5635b4b8e9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4b8e9a0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f16c80a7f80 tx=0x7f16c80099e0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 -- 172.21.15.28:0/3507960835 shutdown_connections 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b89570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4b8e9a0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/3507960835 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4b88ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 -- 172.21.15.28:0/3507960835 >> 172.21.15.28:0/3507960835 conn(0x5635b4ab6b90 msgr2=0x5635b4b83640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 -- 172.21.15.28:0/3507960835 shutdown_connections 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.981+0000 7f16e8e43b80 1 -- 172.21.15.28:0/3507960835 wait complete. 2024-05-20T20:05:13.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.982+0000 7f16e8e43b80 1 -- start start 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.982+0000 7f16e8e43b80 1 Processor -- start 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b890f0 0x5635b4b94c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b925a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5635b49dcf50 con 0x5635b4b8eee0 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5635b4a77710 con 0x5635b4b8e5c0 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5635b4b95750 con 0x5635b4b890f0 2024-05-20T20:05:13.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16e8e43b80 10 client.0 _ll_register_callbacks cb 0x5635b4a80c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16cea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16cea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54032/0 (socket says 172.21.15.28:54032) 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16d7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b890f0 0x5635b4b94c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16cea00700 1 -- 172.21.15.28:0/905549983 learned_addr learned my addr 172.21.15.28:0/905549983 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16d7e00700 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b925a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.983+0000 7f16cea00700 1 -- 172.21.15.28:0/905549983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b890f0 msgr2=0x5635b4b94c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16cea00700 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b890f0 0x5635b4b94c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16cea00700 1 -- 172.21.15.28:0/905549983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 msgr2=0x5635b4b925a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:13.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16cea00700 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b925a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16cea00700 1 -- 172.21.15.28:0/905549983 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5635b4a26880 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16cea00700 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92060 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f16c0012720 tx=0x7f16c0012a30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1747b1aa6aa3f451 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16c5600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16c5600700 1 -- 172.21.15.28:0/905549983 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16c0004830 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16c5600700 1 -- 172.21.15.28:0/905549983 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f16c00049d0 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.984+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f16c80ab410 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.985+0000 7f16c5600700 1 -- 172.21.15.28:0/905549983 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f16c0004bd0 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.985+0000 7f16e8e43b80 10 client.25055 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.985+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x5635b4aa7010 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.985+0000 7f16c5600700 1 -- 172.21.15.28:0/905549983 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f16c80ab410 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.985+0000 7f16c5600700 1 -- 172.21.15.28:0/905549983 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f16c00119a0 con 0x5635b4b8e5c0 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.986+0000 7f16e8e43b80 10 client.25055 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.986+0000 7f16e8e43b80 10 client.25055 resolve_mds: resolved name 'e' to daemon mds.e 2024-05-20T20:05:13.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.986+0000 7f16e8e43b80 20 client.25055 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:13.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.986+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4ba2250 0x5635b4ba36c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:13.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.986+0000 7f16e8e43b80 4 client.25055 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:05:13.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.986+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5635b4ae8d40 con 0x5635b4ba2250 2024-05-20T20:05:13.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.986+0000 7f16d7400700 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4ba2250 0x5635b4ba36c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:13.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.987+0000 7f16d7400700 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4ba2250 0x5635b4ba36c0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=4d9bb00e9a19a5e5 server_cookie=cf8ae71e126e8e0c in_seq=0 out_seq=0 2024-05-20T20:05:13.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.987+0000 7f16c5600700 10 client.25055 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:05:13.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16c5600700 1 -- 172.21.15.28:0/905549983 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5635b4ae8d40 con 0x5635b4ba2250 2024-05-20T20:05:13.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16c5600700 10 client.25055 handle_command_reply: tid=0 2024-05-20T20:05:13.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 client.25055 shutdown 2024-05-20T20:05:13.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b8e5c0 msgr2=0x5635b4b92060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:13.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92060 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f16c0012720 tx=0x7f16c0012a30 comp rx=0 tx=0).stop 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 shutdown_connections 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4ba2250 0x5635b4ba36c0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b925a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b890f0 0x5635b4b94c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/905549983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92060 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16c5600700 0 client.25055 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 >> 172.21.15.28:0/905549983 conn(0x5635b4ab6b90 msgr2=0x5635b4b82d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 shutdown_connections 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 1 -- 172.21.15.28:0/905549983 wait complete. 2024-05-20T20:05:13.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:13.988+0000 7f16e8e43b80 20 client.25055 trim_cache size 0 max 16384 2024-05-20T20:05:14.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.012+0000 7f16e8e43b80 1 Processor -- start 2024-05-20T20:05:14.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.012+0000 7f16e8e43b80 1 -- start start 2024-05-20T20:05:14.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b9e640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:14.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:14.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b834a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:14.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ba1a00 con 0x5635b4b8eee0 2024-05-20T20:05:14.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ba20d0 con 0x5635b4b890f0 2024-05-20T20:05:14.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ba27a0 con 0x5635b4b8e5c0 2024-05-20T20:05:14.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16d7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b834a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:14.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16d7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b834a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43714/0 (socket says 172.21.15.28:43714) 2024-05-20T20:05:14.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16d7e00700 1 -- 172.21.15.28:0/1960258594 learned_addr learned my addr 172.21.15.28:0/1960258594 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:14.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16d7400700 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b9e640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:14.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.013+0000 7f16cea00700 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:14.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16d7e00700 1 -- 172.21.15.28:0/1960258594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8e5c0 msgr2=0x5635b4b92550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:14.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16d7e00700 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16d7e00700 1 -- 172.21.15.28:0/1960258594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 msgr2=0x5635b4b9e640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16d7e00700 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b9e640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16d7e00700 1 -- 172.21.15.28:0/1960258594 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5635b4b84350 con 0x5635b4b8eee0 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16cea00700 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16d7400700 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b9e640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16d7e00700 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b834a0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f16d0004460 tx=0x7f16d0009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f4a7d4f01a7e2a77 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16c7e00700 1 -- 172.21.15.28:0/1960258594 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16d001a040 con 0x5635b4b8eee0 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16c7e00700 1 -- 172.21.15.28:0/1960258594 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f16d0004490 con 0x5635b4b8eee0 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.014+0000 7f16c7e00700 1 -- 172.21.15.28:0/1960258594 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16d001d620 con 0x5635b4b8eee0 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 -- 172.21.15.28:0/1960258594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 msgr2=0x5635b4b834a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b834a0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f16d0004460 tx=0x7f16d0009260 comp rx=0 tx=0).stop 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 -- 172.21.15.28:0/1960258594 shutdown_connections 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8eee0 0x5635b4b834a0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4b9e640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/1960258594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8e5c0 0x5635b4b92550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 -- 172.21.15.28:0/1960258594 >> 172.21.15.28:0/1960258594 conn(0x5635b4ab6b90 msgr2=0x5635b4b91e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:14.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 -- 172.21.15.28:0/1960258594 shutdown_connections 2024-05-20T20:05:14.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.015+0000 7f16e8e43b80 1 -- 172.21.15.28:0/1960258594 wait complete. 2024-05-20T20:05:14.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.016+0000 7f16e8e43b80 1 -- start start 2024-05-20T20:05:14.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.016+0000 7f16e8e43b80 1 Processor -- start 2024-05-20T20:05:14.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4bacd90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4bad2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16e8e43b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4ba7ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ba3540 con 0x5635b4b8e5c0 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ba2e70 con 0x5635b4b890f0 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16e8e43b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5635b4ba3c10 con 0x5635b4b8eee0 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16e8e43b80 10 client.0 _ll_register_callbacks cb 0x5635b4a80c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16d7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4bacd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16d7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4ba7ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16cea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4bad2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16d7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4ba7ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58764/0 (socket says 172.21.15.28:58764) 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16d7e00700 1 -- 172.21.15.28:0/511395017 learned_addr learned my addr 172.21.15.28:0/511395017 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:14.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.017+0000 7f16d7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4bacd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54058/0 (socket says 172.21.15.28:54058) 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16d7e00700 1 -- 172.21.15.28:0/511395017 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 msgr2=0x5635b4bacd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16d7e00700 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4bacd90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16d7e00700 1 -- 172.21.15.28:0/511395017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 msgr2=0x5635b4bad2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16d7e00700 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4bad2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16d7e00700 1 -- 172.21.15.28:0/511395017 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5635b4ba83e0 con 0x5635b4b8eee0 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16d7e00700 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4ba7ba0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f16d0026000 tx=0x7f16d0021830 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dbca3f6586eab5c7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16c5600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16c5600700 1 -- 172.21.15.28:0/511395017 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f16d001a040 con 0x5635b4b8eee0 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f16d0004490 con 0x5635b4b8eee0 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.018+0000 7f16c5600700 1 -- 172.21.15.28:0/511395017 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f16d0005250 con 0x5635b4b8eee0 2024-05-20T20:05:14.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16c5600700 1 -- 172.21.15.28:0/511395017 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f16d0021b00 con 0x5635b4b8eee0 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 10 client.25045 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x5635b4c16dd0 con 0x5635b4b8eee0 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16c5600700 1 -- 172.21.15.28:0/511395017 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f16d0004490 con 0x5635b4b8eee0 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16c5600700 1 -- 172.21.15.28:0/511395017 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f16d0031020 con 0x5635b4b8eee0 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 10 client.25045 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 10 client.25045 resolve_mds: resolved name 'e' to daemon mds.e 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 20 client.25045 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4c1b780 0x5635b4c1a490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 4 client.25045 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:05:14.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.019+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "status"}) -- 0x5635b4ae2520 con 0x5635b4c1b780 2024-05-20T20:05:14.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.020+0000 7f16d7400700 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4c1b780 0x5635b4c1a490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:14.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.020+0000 7f16d7400700 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4c1b780 0x5635b4c1a490 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=2ca8a278b3e35170 server_cookie=b6e06ed9b59c61cb in_seq=0 out_seq=0 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.020+0000 7f16c5600700 10 client.25045 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16c5600700 1 -- 172.21.15.28:0/511395017 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x5635b4ae2520 con 0x5635b4c1b780 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16c5600700 10 client.25045 handle_command_reply: tid=0 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 client.25045 shutdown 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 msgr2=0x5635b4ba7ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4ba7ba0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f16d0026000 tx=0x7f16d0021830 comp rx=0 tx=0).stop 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 shutdown_connections 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5635b4c1b780 0x5635b4c1a490 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635b4b8eee0 0x5635b4ba7ba0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635b4b890f0 0x5635b4bacd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16c5600700 0 client.25045 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 --2- 172.21.15.28:0/511395017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635b4b8e5c0 0x5635b4bad2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 >> 172.21.15.28:0/511395017 conn(0x5635b4ab6b90 msgr2=0x5635b4b8f2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 shutdown_connections 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 1 -- 172.21.15.28:0/511395017 wait complete. 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.021+0000 7f16e8e43b80 20 client.25045 trim_cache size 0 max 16384 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:05:14.025 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 3, 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24482, 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 209.38796293499999, 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 275.43417548999997 2024-05-20T20:05:14.026 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f16bc0b2880 msgr2=0x7f16bc0b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f16bc0b2880 0x7f16bc0b4d40 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f16d0011440 tx=0x7f16d0011800 comp rx=0 tx=0).stop 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 msgr2=0x7f16d819beb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 0x7f16d819beb0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f16c800b780 tx=0x7f16c8004d00 comp rx=0 tx=0).stop 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 shutdown_connections 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f16bc0b2880 0x7f16bc0b4d40 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f16d8108500 0x7f16d81a0780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f16d81035f0 0x7f16d819beb0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 --2- 172.21.15.28:0/783502800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f16d8103f90 0x7f16d819c3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:14.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 >> 172.21.15.28:0/783502800 conn(0x7f16d807a6a0 msgr2=0x7f16d8101460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:14.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 shutdown_connections 2024-05-20T20:05:14.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:14.022+0000 7f16de400700 1 -- 172.21.15.28:0/783502800 wait complete. 2024-05-20T20:05:14.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:14 smithi028 ceph-mon[31178]: pgmap v435: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:14.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:14 smithi154 ceph-mon[36314]: pgmap v435: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:14.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:14 smithi033 ceph-mon[36179]: pgmap v435: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:15.036 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.i status 2024-05-20T20:05:15.512 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:16.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:16 smithi028 ceph-mon[31178]: pgmap v436: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:16 smithi154 ceph-mon[36314]: pgmap v436: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:16.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:16 smithi033 ceph-mon[36179]: pgmap v436: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 -- 172.21.15.28:0/2113093717 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa11010fe20 msgr2=0x7fa110112210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 --2- 172.21.15.28:0/2113093717 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa11010fe20 0x7fa110112210 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fa10800b0a0 tx=0x7fa10800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 -- 172.21.15.28:0/2113093717 shutdown_connections 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 --2- 172.21.15.28:0/2113093717 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa11010fe20 0x7fa110112210 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 --2- 172.21.15.28:0/2113093717 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1101018e0 0x7fa110101cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 --2- 172.21.15.28:0/2113093717 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa110102200 0x7fa11010f8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 -- 172.21.15.28:0/2113093717 >> 172.21.15.28:0/2113093717 conn(0x7fa1100771e0 msgr2=0x7fa1100775f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.782+0000 7fa10f200700 1 -- 172.21.15.28:0/2113093717 shutdown_connections 2024-05-20T20:05:17.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.783+0000 7fa10f200700 1 -- 172.21.15.28:0/2113093717 wait complete. 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 Processor -- start 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 -- start start 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa1101018e0 0x7fa110105560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa110102200 0x7fa110105aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa11010fe20 0x7fa110105fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa110115870 con 0x7fa11010fe20 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa1101156f0 con 0x7fa110102200 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.784+0000 7fa10f200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa1101159f0 con 0x7fa1101018e0 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa110102200 0x7fa110105aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa110102200 0x7fa110105aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60654/0 (socket says 172.21.15.28:60654) 2024-05-20T20:05:17.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 -- 172.21.15.28:0/753497731 learned_addr learned my addr 172.21.15.28:0/753497731 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:17.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10de00700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa1101018e0 0x7fa110105560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10e800700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa11010fe20 0x7fa110105fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 -- 172.21.15.28:0/753497731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa1101018e0 msgr2=0x7fa110105560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa1101018e0 0x7fa110105560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 -- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa11010fe20 msgr2=0x7fa110105fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa11010fe20 0x7fa110105fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.785+0000 7fa10d400700 1 -- 172.21.15.28:0/753497731 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa1101ac3d0 con 0x7fa110102200 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.786+0000 7fa10d400700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa110102200 0x7fa110105aa0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fa0fc00e9f0 tx=0x7fa0fc00ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.786+0000 7fa106a00700 1 -- 172.21.15.28:0/753497731 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0fc00cbd0 con 0x7fa110102200 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.786+0000 7fa106a00700 1 -- 172.21.15.28:0/753497731 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa0fc00ee90 con 0x7fa110102200 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.786+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa1101ac6c0 con 0x7fa110102200 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.786+0000 7fa106a00700 1 -- 172.21.15.28:0/753497731 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0fc005590 con 0x7fa110102200 2024-05-20T20:05:17.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.786+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa1101acc00 con 0x7fa110102200 2024-05-20T20:05:17.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.790+0000 7fa106a00700 1 -- 172.21.15.28:0/753497731 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fa0fc004750 con 0x7fa110102200 2024-05-20T20:05:17.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 Processor -- start 2024-05-20T20:05:17.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 -- start start 2024-05-20T20:05:17.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9913b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b98b460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b98bee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55973b8e9df0 con 0x55973b98ba80 2024-05-20T20:05:17.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55973b8d9990 con 0x55973b9918f0 2024-05-20T20:05:17.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.791+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55973b8dd9d0 con 0x55973b990fd0 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b98b460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9913b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b98b460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60656/0 (socket says 172.21.15.28:60656) 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10de00700 1 -- 172.21.15.28:0/1671277023 learned_addr learned my addr 172.21.15.28:0/1671277023 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9913b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56328/0 (socket says 172.21.15.28:56328) 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10d400700 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b98bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10e800700 1 -- 172.21.15.28:0/1671277023 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 msgr2=0x55973b98b460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10e800700 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b98b460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10e800700 1 -- 172.21.15.28:0/1671277023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 msgr2=0x55973b98bee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10e800700 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b98bee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10e800700 1 -- 172.21.15.28:0/1671277023 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55973b8a23d0 con 0x55973b990fd0 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa106a00700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa0f40b2900 0x7fa0f40b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa10d400700 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b98bee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.792+0000 7fa106a00700 1 -- 172.21.15.28:0/753497731 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fa0fc0e23f0 con 0x7fa110102200 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.793+0000 7fa10e800700 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9913b0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fa10800b070 tx=0x7fa1080092e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=606e81716c2fe8d2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.793+0000 7fa10de00700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa0f40b2900 0x7fa0f40b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.793+0000 7fa0fb400700 1 -- 172.21.15.28:0/1671277023 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa10800bcb0 con 0x55973b990fd0 2024-05-20T20:05:17.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.793+0000 7fa0fb400700 1 -- 172.21.15.28:0/1671277023 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa1080074e0 con 0x55973b990fd0 2024-05-20T20:05:17.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.793+0000 7fa0fb400700 1 -- 172.21.15.28:0/1671277023 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa108005280 con 0x55973b990fd0 2024-05-20T20:05:17.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.793+0000 7fa10de00700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa0f40b2900 0x7fa0f40b4dc0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fa100005c50 tx=0x7fa100009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:17.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1671277023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 msgr2=0x55973b9913b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9913b0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fa10800b070 tx=0x7fa1080092e0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1671277023 shutdown_connections 2024-05-20T20:05:17.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b98bee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9913b0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1671277023 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b98b460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1671277023 >> 172.21.15.28:0/1671277023 conn(0x55973b8b9b90 msgr2=0x55973b986100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.794+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1671277023 shutdown_connections 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.795+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1671277023 wait complete. 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.795+0000 7fa11f2cdb80 1 -- start start 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.795+0000 7fa11f2cdb80 1 Processor -- start 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b9975c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b994960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b994ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55973b7dff50 con 0x55973b98ba80 2024-05-20T20:05:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55973b87a710 con 0x55973b9918f0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55973b9983b0 con 0x55973b990fd0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa11f2cdb80 10 client.0 _ll_register_callbacks cb 0x55973b883c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa10de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b9975c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa10e800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b994ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa10de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b9975c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56314/0 (socket says 172.21.15.28:56314) 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa10de00700 1 -- 172.21.15.28:0/277911055 learned_addr learned my addr 172.21.15.28:0/277911055 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.796+0000 7fa10d400700 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b994960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa10e800700 1 -- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 msgr2=0x55973b994960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa10e800700 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b994960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa10e800700 1 -- 172.21.15.28:0/277911055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 msgr2=0x55973b9975c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa10e800700 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b9975c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa10e800700 1 -- 172.21.15.28:0/277911055 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55973b829880 con 0x55973b9918f0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa10e800700 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b994ea0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fa10800b040 tx=0x7fa108009780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b063d389121d02b9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa0efe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:05:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.797+0000 7fa0efe00700 1 -- 172.21.15.28:0/277911055 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa108003a00 con 0x55973b9918f0 2024-05-20T20:05:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.798+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa1080074e0 con 0x55973b9918f0 2024-05-20T20:05:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.798+0000 7fa0efe00700 1 -- 172.21.15.28:0/277911055 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fa108003ba0 con 0x55973b9918f0 2024-05-20T20:05:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.798+0000 7fa0efe00700 1 -- 172.21.15.28:0/277911055 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fa108037050 con 0x55973b9918f0 2024-05-20T20:05:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.798+0000 7fa11f2cdb80 10 client.25073 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.798+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55973b8aa010 con 0x55973b9918f0 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.798+0000 7fa0efe00700 1 -- 172.21.15.28:0/277911055 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa1080074e0 con 0x55973b9918f0 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa0efe00700 1 -- 172.21.15.28:0/277911055 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fa108011070 con 0x55973b9918f0 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa11f2cdb80 10 client.25073 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa11f2cdb80 10 client.25073 resolve_mds: resolved name 'i' to daemon mds.i 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa11f2cdb80 20 client.25073 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973b9a3b90 0x55973b9a5d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa11f2cdb80 4 client.25073 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55973b8ebd40 con 0x55973b9a3b90 2024-05-20T20:05:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.799+0000 7fa10de00700 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973b9a3b90 0x55973b9a5d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.800+0000 7fa10de00700 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973b9a3b90 0x55973b9a5d80 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=132800af01ea14e3 server_cookie=254198dcd62faec0 in_seq=0 out_seq=0 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.800+0000 7fa0efe00700 10 client.25073 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa0efe00700 1 -- 172.21.15.28:0/277911055 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55973b8ebd40 con 0x55973b9a3b90 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa0efe00700 10 client.25073 handle_command_reply: tid=0 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 client.25073 shutdown 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 msgr2=0x55973b994ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b994ea0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fa10800b040 tx=0x7fa108009780 comp rx=0 tx=0).stop 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 shutdown_connections 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b9918f0 0x55973b994ea0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973b9a3b90 0x55973b9a5d80 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b98ba80 0x55973b9975c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/277911055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b994960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa0efe00700 0 client.25073 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 >> 172.21.15.28:0/277911055 conn(0x55973b8b9b90 msgr2=0x55973b98a450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:17.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.801+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 shutdown_connections 2024-05-20T20:05:17.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.802+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/277911055 wait complete. 2024-05-20T20:05:17.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.802+0000 7fa11f2cdb80 20 client.25073 trim_cache size 0 max 16384 2024-05-20T20:05:17.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.825+0000 7fa11f2cdb80 1 Processor -- start 2024-05-20T20:05:17.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.825+0000 7fa11f2cdb80 1 -- start start 2024-05-20T20:05:17.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b98b460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9a56b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b9918f0 0x55973b9a4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55973b9a40c0 con 0x55973b9918f0 2024-05-20T20:05:17.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55973b9a4790 con 0x55973b98ba80 2024-05-20T20:05:17.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55973b9a4e60 con 0x55973b990fd0 2024-05-20T20:05:17.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa10e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b9918f0 0x55973b9a4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa10de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b98b460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa10d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9a56b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa10de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b98b460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60666/0 (socket says 172.21.15.28:60666) 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.826+0000 7fa10de00700 1 -- 172.21.15.28:0/1452476540 learned_addr learned my addr 172.21.15.28:0/1452476540 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa10d400700 1 -- 172.21.15.28:0/1452476540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 msgr2=0x55973b98b460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa10d400700 1 --2- 172.21.15.28:0/1452476540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b98b460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa10d400700 1 -- 172.21.15.28:0/1452476540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b9918f0 msgr2=0x55973b9a4fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa10d400700 1 --2- 172.21.15.28:0/1452476540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b9918f0 0x55973b9a4fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa10d400700 1 -- 172.21.15.28:0/1452476540 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55973b993f20 con 0x55973b990fd0 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa10d400700 1 --2- 172.21.15.28:0/1452476540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9a56b0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fa0fc009860 tx=0x7fa0fc0e59a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=24563c8821e7bfe2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa0faa00700 1 -- 172.21.15.28:0/1452476540 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0fc0a9320 con 0x55973b990fd0 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa0faa00700 1 -- 172.21.15.28:0/1452476540 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa0fc0b2170 con 0x55973b990fd0 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.827+0000 7fa0faa00700 1 -- 172.21.15.28:0/1452476540 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0fc0a5160 con 0x55973b990fd0 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1452476540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 msgr2=0x55973b9a56b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1452476540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9a56b0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fa0fc009860 tx=0x7fa0fc0e59a0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1452476540 shutdown_connections 2024-05-20T20:05:17.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1452476540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b9918f0 0x55973b9a4fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1452476540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b98b460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/1452476540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b990fd0 0x55973b9a56b0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1452476540 >> 172.21.15.28:0/1452476540 conn(0x55973b8b9b90 msgr2=0x55973b98ea50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:17.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1452476540 shutdown_connections 2024-05-20T20:05:17.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.828+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/1452476540 wait complete. 2024-05-20T20:05:17.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 -- start start 2024-05-20T20:05:17.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 Processor -- start 2024-05-20T20:05:17.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b9962b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b990fd0 0x55973b9967f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b9918f0 0x55973b996d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55973b9a5c00 con 0x55973b990fd0 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55973b9a5530 con 0x55973b98ba80 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.829+0000 7fa11f2cdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55973b9a62d0 con 0x55973b9918f0 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa11f2cdb80 10 client.0 _ll_register_callbacks cb 0x55973b883c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b9962b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b990fd0 0x55973b9967f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b9918f0 0x55973b996d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b9962b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60676/0 (socket says 172.21.15.28:60676) 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 -- 172.21.15.28:0/878218940 learned_addr learned my addr 172.21.15.28:0/878218940 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 -- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b9918f0 msgr2=0x55973b996d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b9918f0 0x55973b996d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 -- 172.21.15.28:0/878218940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b990fd0 msgr2=0x55973b9967f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b990fd0 0x55973b9967f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 -- 172.21.15.28:0/878218940 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55973b9ad8b0 con 0x55973b98ba80 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10e800700 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b9918f0 0x55973b996d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa10de00700 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b9962b0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fa1000095f0 tx=0x7fa100004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4322be862f8af04 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.830+0000 7fa0efe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.831+0000 7fa0efe00700 1 -- 172.21.15.28:0/878218940 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa10001de60 con 0x55973b98ba80 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.831+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55973b8e5520 con 0x55973b98ba80 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.831+0000 7fa0efe00700 1 -- 172.21.15.28:0/878218940 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fa100034410 con 0x55973b98ba80 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.831+0000 7fa0efe00700 1 -- 172.21.15.28:0/878218940 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fa100034610 con 0x55973b98ba80 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.831+0000 7fa11f2cdb80 10 client.25079 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.831+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55973ba104d0 con 0x55973b98ba80 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.831+0000 7fa0efe00700 1 -- 172.21.15.28:0/878218940 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55973b8e5520 con 0x55973b98ba80 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa0efe00700 1 -- 172.21.15.28:0/878218940 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fa10003d9d0 con 0x55973b98ba80 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa11f2cdb80 10 client.25079 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa11f2cdb80 10 client.25079 resolve_mds: resolved name 'i' to daemon mds.i 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa11f2cdb80 20 client.25079 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973ba15150 0x55973ba13e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa11f2cdb80 4 client.25079 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "status"}) -- 0x55973b75c200 con 0x55973ba15150 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.832+0000 7fa10d400700 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973ba15150 0x55973ba13e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa10d400700 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973ba15150 0x55973ba13e60 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=c216c46641184f0f server_cookie=59cc8e7ca20c89d8 in_seq=0 out_seq=0 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa0efe00700 10 client.25079 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa0efe00700 1 -- 172.21.15.28:0/878218940 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55973b75c200 con 0x55973ba15150 2024-05-20T20:05:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa0efe00700 10 client.25079 handle_command_reply: tid=0 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 client.25079 shutdown 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 msgr2=0x55973b9962b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b9962b0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fa1000095f0 tx=0x7fa100004750 comp rx=0 tx=0).stop 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 shutdown_connections 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55973b9918f0 0x55973b996d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55973b98ba80 0x55973b9962b0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55973ba15150 0x55973ba13e60 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 --2- 172.21.15.28:0/878218940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55973b990fd0 0x55973b9967f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 >> 172.21.15.28:0/878218940 conn(0x55973b8b9b90 msgr2=0x55973b994740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa0efe00700 0 client.25079 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 shutdown_connections 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.833+0000 7fa11f2cdb80 1 -- 172.21.15.28:0/878218940 wait complete. 2024-05-20T20:05:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa11f2cdb80 20 client.25079 trim_cache size 0 max 16384 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 2, 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24493, 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:05:17.838 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 214.21212556899999, 2024-05-20T20:05:17.839 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:05:17.839 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:05:17.839 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:05:17.839 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 267.52294178099999 2024-05-20T20:05:17.839 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:05:17.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa0f40b2900 msgr2=0x7fa0f40b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa0f40b2900 0x7fa0f40b4dc0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fa100005c50 tx=0x7fa100009260 comp rx=0 tx=0).stop 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa110102200 msgr2=0x7fa110105aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa110102200 0x7fa110105aa0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fa0fc00e9f0 tx=0x7fa0fc00ed00 comp rx=0 tx=0).stop 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 shutdown_connections 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fa0f40b2900 0x7fa0f40b4dc0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa11010fe20 0x7fa110105fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa1101018e0 0x7fa110105560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 --2- 172.21.15.28:0/753497731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa110102200 0x7fa110105aa0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.834+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 >> 172.21.15.28:0/753497731 conn(0x7fa1100771e0 msgr2=0x7fa110110360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.835+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 shutdown_connections 2024-05-20T20:05:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:17.835+0000 7fa10f200700 1 -- 172.21.15.28:0/753497731 wait complete. 2024-05-20T20:05:18.629 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.h status 2024-05-20T20:05:19.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:18 smithi028 ceph-mon[31178]: pgmap v437: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:05:19.092 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:18 smithi154 ceph-mon[36314]: pgmap v437: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:05:19.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:18 smithi033 ceph-mon[36179]: pgmap v437: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 10 op/s 2024-05-20T20:05:20.979 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:20 smithi028 ceph-mon[31178]: pgmap v438: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:20 smithi154 ceph-mon[36314]: pgmap v438: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:20 smithi033 ceph-mon[36179]: pgmap v438: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:21.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 -- 172.21.15.28:0/533441274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1037f0 msgr2=0x7fcf8c103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 --2- 172.21.15.28:0/533441274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1037f0 0x7fcf8c103bd0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fcf80009fc0 tx=0x7fcf8000b330 comp rx=0 tx=0).stop 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 -- 172.21.15.28:0/533441274 shutdown_connections 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 --2- 172.21.15.28:0/533441274 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf8c1088f0 0x7fcf8c110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 --2- 172.21.15.28:0/533441274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1037f0 0x7fcf8c103bd0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 --2- 172.21.15.28:0/533441274 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf8c1041a0 0x7fcf8c1081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 -- 172.21.15.28:0/533441274 >> 172.21.15.28:0/533441274 conn(0x7fcf8c07a6a0 msgr2=0x7fcf8c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.566+0000 7fcf94200700 1 -- 172.21.15.28:0/533441274 shutdown_connections 2024-05-20T20:05:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.567+0000 7fcf94200700 1 -- 172.21.15.28:0/533441274 wait complete. 2024-05-20T20:05:21.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 Processor -- start 2024-05-20T20:05:21.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 -- start start 2024-05-20T20:05:21.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf8c1037f0 0x7fcf8c19bd70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf8c1041a0 0x7fcf8c19c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1088f0 0x7fcf8c1a0640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fcf8c1136e0 con 0x7fcf8c1088f0 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fcf8c113560 con 0x7fcf8c1037f0 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.568+0000 7fcf94200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fcf8c113860 con 0x7fcf8c1041a0 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf92200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1088f0 0x7fcf8c1a0640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf91800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf8c1037f0 0x7fcf8c19bd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf90e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf8c1041a0 0x7fcf8c19c2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf90e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf8c1041a0 0x7fcf8c19c2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56374/0 (socket says 172.21.15.28:56374) 2024-05-20T20:05:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf90e00700 1 -- 172.21.15.28:0/2484307408 learned_addr learned my addr 172.21.15.28:0/2484307408 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:21.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf91800700 1 -- 172.21.15.28:0/2484307408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf8c1041a0 msgr2=0x7fcf8c19c2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf91800700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf8c1041a0 0x7fcf8c19c2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf91800700 1 -- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1088f0 msgr2=0x7fcf8c1a0640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf91800700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1088f0 0x7fcf8c1a0640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf91800700 1 -- 172.21.15.28:0/2484307408 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcf8c1a0d80 con 0x7fcf8c1037f0 2024-05-20T20:05:21.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.569+0000 7fcf91800700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf8c1037f0 0x7fcf8c19bd70 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fcf80000f80 tx=0x7fcf80009510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:21.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.570+0000 7fcf8a000700 1 -- 172.21.15.28:0/2484307408 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf80021070 con 0x7fcf8c1037f0 2024-05-20T20:05:21.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.570+0000 7fcf8a000700 1 -- 172.21.15.28:0/2484307408 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcf80007ac0 con 0x7fcf8c1037f0 2024-05-20T20:05:21.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.570+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcf8c1a1010 con 0x7fcf8c1037f0 2024-05-20T20:05:21.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.570+0000 7fcf8a000700 1 -- 172.21.15.28:0/2484307408 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf80005040 con 0x7fcf8c1037f0 2024-05-20T20:05:21.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.570+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcf8c1a88f0 con 0x7fcf8c1037f0 2024-05-20T20:05:21.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.573+0000 7fcf8a000700 1 -- 172.21.15.28:0/2484307408 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7fcf80024070 con 0x7fcf8c1037f0 2024-05-20T20:05:21.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.574+0000 7fcf9ebaab80 1 Processor -- start 2024-05-20T20:05:21.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf9ebaab80 1 -- start start 2024-05-20T20:05:21.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba3e60 0x557547ba4240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547b9e210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547b9e830 0x557547b9ec90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557547afcdf0 con 0x557547b9e830 2024-05-20T20:05:21.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557547aec990 con 0x557547ba4780 2024-05-20T20:05:21.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557547af09d0 con 0x557547ba3e60 2024-05-20T20:05:21.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf8a000700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf740b2900 0x7fcf740b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf91800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547b9e210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf91800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547b9e210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60684/0 (socket says 172.21.15.28:60684) 2024-05-20T20:05:21.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf91800700 1 -- 172.21.15.28:0/2949565015 learned_addr learned my addr 172.21.15.28:0/2949565015 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:21.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.575+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547b9e830 0x557547b9ec90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf92200700 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba3e60 0x557547ba4240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf92200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf740b2900 0x7fcf740b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf8a000700 1 -- 172.21.15.28:0/2484307408 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fcf8000f020 con 0x7fcf8c1037f0 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf91800700 1 -- 172.21.15.28:0/2949565015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba3e60 msgr2=0x557547ba4240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf91800700 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba3e60 0x557547ba4240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf91800700 1 -- 172.21.15.28:0/2949565015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547b9e830 msgr2=0x557547b9ec90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf91800700 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547b9e830 0x557547b9ec90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf91800700 1 -- 172.21.15.28:0/2949565015 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557547ab53d0 con 0x557547ba4780 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.576+0000 7fcf91800700 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547b9e210 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fcf800a6c30 tx=0x7fcf80009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=de7cd3ea3089f7b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:21.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.577+0000 7fcf7ea00700 1 -- 172.21.15.28:0/2949565015 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf800aa320 con 0x557547ba4780 2024-05-20T20:05:21.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.577+0000 7fcf7ea00700 1 -- 172.21.15.28:0/2949565015 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcf800b32c0 con 0x557547ba4780 2024-05-20T20:05:21.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.577+0000 7fcf7ea00700 1 -- 172.21.15.28:0/2949565015 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf800aae10 con 0x557547ba4780 2024-05-20T20:05:21.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.577+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2949565015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 msgr2=0x557547b9e210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.577+0000 7fcf92200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf740b2900 0x7fcf740b4dc0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fcf84009a40 tx=0x7fcf84009310 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:21.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.577+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547b9e210 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fcf800a6c30 tx=0x7fcf80009920 comp rx=0 tx=0).stop 2024-05-20T20:05:21.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2949565015 shutdown_connections 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547b9e830 0x557547b9ec90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba3e60 0x557547ba4240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2949565015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547b9e210 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2949565015 >> 172.21.15.28:0/2949565015 conn(0x557547accb90 msgr2=0x557547b98fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2949565015 shutdown_connections 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2949565015 wait complete. 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 -- start start 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.578+0000 7fcf9ebaab80 1 Processor -- start 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547b9e830 0x557547baa450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba77f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba4780 0x557547ba7d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5575479f2f50 con 0x557547ba3e60 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557547a8d710 con 0x557547b9e830 2024-05-20T20:05:21.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557547baafb0 con 0x557547ba4780 2024-05-20T20:05:21.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf9ebaab80 10 client.0 _ll_register_callbacks cb 0x557547a96c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:21.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf91800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547b9e830 0x557547baa450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf90e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba77f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf91800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547b9e830 0x557547baa450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60698/0 (socket says 172.21.15.28:60698) 2024-05-20T20:05:21.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf91800700 1 -- 172.21.15.28:0/2539645248 learned_addr learned my addr 172.21.15.28:0/2539645248 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:21.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.579+0000 7fcf92200700 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba4780 0x557547ba7d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf91800700 1 -- 172.21.15.28:0/2539645248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba4780 msgr2=0x557547ba7d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf91800700 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba4780 0x557547ba7d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf91800700 1 -- 172.21.15.28:0/2539645248 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 msgr2=0x557547ba77f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf91800700 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba77f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf91800700 1 -- 172.21.15.28:0/2539645248 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557547a3c880 con 0x557547b9e830 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba77f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf91800700 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547b9e830 0x557547baa450 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fcf800a7e20 tx=0x7fcf800a63c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=786572bc370b63bb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf73400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf73400700 1 -- 172.21.15.28:0/2539645248 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf800aa320 con 0x557547b9e830 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf73400700 1 -- 172.21.15.28:0/2539645248 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fcf800b36f0 con 0x557547b9e830 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x557547afed40 con 0x557547b9e830 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.580+0000 7fcf73400700 1 -- 172.21.15.28:0/2539645248 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fcf800aae10 con 0x557547b9e830 2024-05-20T20:05:21.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 10 client.25091 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x557547abd010 con 0x557547b9e830 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf73400700 1 -- 172.21.15.28:0/2539645248 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x557547afed40 con 0x557547b9e830 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf73400700 1 -- 172.21.15.28:0/2539645248 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fcf80011070 con 0x557547b9e830 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 10 client.25091 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 10 client.25091 resolve_mds: resolved name 'h' to daemon mds.h 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 20 client.25091 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547bb77a0 0x557547bb8c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 4 client.25091 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557547af8520 con 0x557547bb77a0 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.581+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547bb77a0 0x557547bb8c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.582+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547bb77a0 0x557547bb8c10 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=216fa3a1cb0196c7 server_cookie=61326387e180c79c in_seq=0 out_seq=0 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.582+0000 7fcf73400700 10 client.25091 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:05:21.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf73400700 1 -- 172.21.15.28:0/2539645248 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x557547af8520 con 0x557547bb77a0 2024-05-20T20:05:21.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf73400700 10 client.25091 handle_command_reply: tid=0 2024-05-20T20:05:21.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 client.25091 shutdown 2024-05-20T20:05:21.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547b9e830 msgr2=0x557547baa450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547b9e830 0x557547baa450 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fcf800a7e20 tx=0x7fcf800a63c0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 shutdown_connections 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547bb77a0 0x557547bb8c10 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547ba4780 0x557547ba7d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547b9e830 0x557547baa450 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2539645248 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba77f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf73400700 0 client.25091 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 >> 172.21.15.28:0/2539645248 conn(0x557547accb90 msgr2=0x557547b9d200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.583+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 shutdown_connections 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.584+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2539645248 wait complete. 2024-05-20T20:05:21.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.584+0000 7fcf9ebaab80 20 client.25091 trim_cache size 0 max 16384 2024-05-20T20:05:21.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.607+0000 7fcf9ebaab80 1 Processor -- start 2024-05-20T20:05:21.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.607+0000 7fcf9ebaab80 1 -- start start 2024-05-20T20:05:21.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547b9e210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba83c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547ba1970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557547bb6f50 con 0x557547ba3e60 2024-05-20T20:05:21.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557547bb7620 con 0x557547ba4780 2024-05-20T20:05:21.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557547bb7cf0 con 0x557547b9e830 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf90e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba83c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf90e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba83c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56370/0 (socket says 172.21.15.28:56370) 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf90e00700 1 -- 172.21.15.28:0/2786683358 learned_addr learned my addr 172.21.15.28:0/2786683358 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf92200700 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547ba1970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.608+0000 7fcf91800700 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547b9e210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf90e00700 1 -- 172.21.15.28:0/2786683358 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 msgr2=0x557547b9e210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547b9e210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf90e00700 1 -- 172.21.15.28:0/2786683358 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 msgr2=0x557547ba1970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547ba1970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf90e00700 1 -- 172.21.15.28:0/2786683358 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557547b99ce0 con 0x557547ba3e60 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba83c0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x557547ba8dc0 tx=0x7fcf78009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a79467fde21fc282 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf7e000700 1 -- 172.21.15.28:0/2786683358 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf78004830 con 0x557547ba3e60 2024-05-20T20:05:21.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf7e000700 1 -- 172.21.15.28:0/2786683358 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcf78005020 con 0x557547ba3e60 2024-05-20T20:05:21.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.609+0000 7fcf7e000700 1 -- 172.21.15.28:0/2786683358 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf78010670 con 0x557547ba3e60 2024-05-20T20:05:21.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2786683358 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 msgr2=0x557547ba83c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba83c0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x557547ba8dc0 tx=0x7fcf78009260 comp rx=0 tx=0).stop 2024-05-20T20:05:21.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2786683358 shutdown_connections 2024-05-20T20:05:21.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547ba1970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547b9e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2786683358 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547ba83c0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2786683358 >> 172.21.15.28:0/2786683358 conn(0x557547accb90 msgr2=0x557547bb6770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2786683358 shutdown_connections 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.610+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2786683358 wait complete. 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.611+0000 7fcf9ebaab80 1 -- start start 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.611+0000 7fcf9ebaab80 1 Processor -- start 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547bc2900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547bb3580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf9ebaab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547bb3cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557547bb8a90 con 0x557547ba3e60 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557547bb83c0 con 0x557547ba4780 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf9ebaab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557547bb9160 con 0x557547b9e830 2024-05-20T20:05:21.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf9ebaab80 10 client.0 _ll_register_callbacks cb 0x557547a96c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf90e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547bb3580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf90e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547bb3580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56372/0 (socket says 172.21.15.28:56372) 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf90e00700 1 -- 172.21.15.28:0/2144590167 learned_addr learned my addr 172.21.15.28:0/2144590167 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf91800700 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547bc2900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf92200700 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547bb3cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf91800700 1 -- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 msgr2=0x557547bb3cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf91800700 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547bb3cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf91800700 1 -- 172.21.15.28:0/2144590167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 msgr2=0x557547bb3580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf91800700 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547bb3580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.612+0000 7fcf91800700 1 -- 172.21.15.28:0/2144590167 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557547bc0420 con 0x557547b9e830 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf91800700 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547bc2900 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fcf80007f20 tx=0x7fcf800aa9e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=11f3a04bdb738e09 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf73400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf73400700 1 -- 172.21.15.28:0/2144590167 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf800aa320 con 0x557547b9e830 2024-05-20T20:05:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf73400700 1 -- 172.21.15.28:0/2144590167 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fcf800e30b0 con 0x557547b9e830 2024-05-20T20:05:21.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fcf78005020 con 0x557547b9e830 2024-05-20T20:05:21.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf73400700 1 -- 172.21.15.28:0/2144590167 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7fcf800ab2b0 con 0x557547b9e830 2024-05-20T20:05:21.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf9ebaab80 10 client.25081 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:21.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.613+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x557547bc0420 con 0x557547b9e830 2024-05-20T20:05:21.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf73400700 1 -- 172.21.15.28:0/2144590167 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcf78005020 con 0x557547b9e830 2024-05-20T20:05:21.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf73400700 1 -- 172.21.15.28:0/2144590167 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7fcf80011040 con 0x557547b9e830 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf9ebaab80 10 client.25081 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf9ebaab80 10 client.25081 resolve_mds: resolved name 'h' to daemon mds.h 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf9ebaab80 20 client.25081 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547c300a0 0x557547c2edb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf9ebaab80 4 client.25081 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "status"}) -- 0x55754796f200 con 0x557547c300a0 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.614+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547c300a0 0x557547c2edb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.615+0000 7fcf90e00700 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547c300a0 0x557547c2edb0 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=6474d6725384023d server_cookie=6fae5be98170c8cc in_seq=0 out_seq=0 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.615+0000 7fcf73400700 10 client.25081 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.615+0000 7fcf73400700 1 -- 172.21.15.28:0/2144590167 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x55754796f200 con 0x557547c300a0 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.615+0000 7fcf73400700 10 client.25081 handle_command_reply: tid=0 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.615+0000 7fcf9ebaab80 1 client.25081 shutdown 2024-05-20T20:05:21.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.615+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 msgr2=0x557547bc2900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.615+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547bc2900 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fcf80007f20 tx=0x7fcf800aa9e0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 shutdown_connections 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x557547c300a0 0x557547c2edb0 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557547ba4780 0x557547bb3cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557547b9e830 0x557547bc2900 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 --2- 172.21.15.28:0/2144590167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557547ba3e60 0x557547bb3580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf73400700 0 client.25081 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 >> 172.21.15.28:0/2144590167 conn(0x557547accb90 msgr2=0x557547ba1970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 shutdown_connections 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 1 -- 172.21.15.28:0/2144590167 wait complete. 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.616+0000 7fcf9ebaab80 20 client.25081 trim_cache size 0 max 16384 2024-05-20T20:05:21.619 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 1, 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24500, 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 219.007879339, 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 274.20593461999999 2024-05-20T20:05:21.620 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:05:21.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf740b2900 msgr2=0x7fcf740b4dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf740b2900 0x7fcf740b4dc0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fcf84009a40 tx=0x7fcf84009310 comp rx=0 tx=0).stop 2024-05-20T20:05:21.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf8c1037f0 msgr2=0x7fcf8c19bd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:21.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf8c1037f0 0x7fcf8c19bd70 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fcf80000f80 tx=0x7fcf80009510 comp rx=0 tx=0).stop 2024-05-20T20:05:21.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 shutdown_connections 2024-05-20T20:05:21.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fcf740b2900 0x7fcf740b4dc0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf8c1088f0 0x7fcf8c1a0640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf8c1037f0 0x7fcf8c19bd70 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 --2- 172.21.15.28:0/2484307408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf8c1041a0 0x7fcf8c19c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:21.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 >> 172.21.15.28:0/2484307408 conn(0x7fcf8c07a6a0 msgr2=0x7fcf8c101920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:21.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 shutdown_connections 2024-05-20T20:05:21.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:21.617+0000 7fcf94200700 1 -- 172.21.15.28:0/2484307408 wait complete. 2024-05-20T20:05:22.531 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.b status 2024-05-20T20:05:22.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:22 smithi028 ceph-mon[31178]: pgmap v439: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:23.000 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:23.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:22 smithi154 ceph-mon[36314]: pgmap v439: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:22 smithi033 ceph-mon[36179]: pgmap v439: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:05:24.739 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:24 smithi028 ceph-mon[31178]: pgmap v440: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:25.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:24 smithi154 ceph-mon[36314]: pgmap v440: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:25.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:24 smithi033 ceph-mon[36179]: pgmap v440: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:05:25.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 -- 172.21.15.28:0/1830561070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c076090 msgr2=0x7f4a6c10fa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1830561070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c076090 0x7f4a6c10fa00 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f4a58009990 tx=0x7f4a58009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 -- 172.21.15.28:0/1830561070 shutdown_connections 2024-05-20T20:05:25.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1830561070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a6c10ff40 0x7f4a6c112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1830561070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 0x7f4a6c075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1830561070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c076090 0x7f4a6c10fa00 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 -- 172.21.15.28:0/1830561070 >> 172.21.15.28:0/1830561070 conn(0x7f4a6c07a6a0 msgr2=0x7f4a6c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:25.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 -- 172.21.15.28:0/1830561070 shutdown_connections 2024-05-20T20:05:25.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.301+0000 7f4a73c00700 1 -- 172.21.15.28:0/1830561070 wait complete. 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.302+0000 7f4a73c00700 1 Processor -- start 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a73c00700 1 -- start start 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a73c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 0x7f4a6c19be90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a73c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a6c076090 0x7f4a6c19c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a73c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c10ff40 0x7f4a6c1a0760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a73c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4a6c115920 con 0x7f4a6c10ff40 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a73c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4a6c1157a0 con 0x7f4a6c075770 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a73c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4a6c115aa0 con 0x7f4a6c076090 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a71200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 0x7f4a6c19be90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a71200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 0x7f4a6c19be90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60748/0 (socket says 172.21.15.28:60748) 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a71200700 1 -- 172.21.15.28:0/1682287647 learned_addr learned my addr 172.21.15.28:0/1682287647 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.303+0000 7f4a71c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c10ff40 0x7f4a6c1a0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.304+0000 7f4a6be00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a6c076090 0x7f4a6c19c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.304+0000 7f4a71200700 1 -- 172.21.15.28:0/1682287647 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a6c076090 msgr2=0x7f4a6c19c3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.304+0000 7f4a71200700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a6c076090 0x7f4a6c19c3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.304+0000 7f4a71200700 1 -- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c10ff40 msgr2=0x7f4a6c1a0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.304+0000 7f4a71200700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c10ff40 0x7f4a6c1a0760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.304+0000 7f4a71200700 1 -- 172.21.15.28:0/1682287647 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4a6c1a0ea0 con 0x7f4a6c075770 2024-05-20T20:05:25.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.304+0000 7f4a71200700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 0x7f4a6c19be90 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f4a6000eb30 tx=0x7f4a6000ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:25.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.305+0000 7f4a69600700 1 -- 172.21.15.28:0/1682287647 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a6000cbd0 con 0x7f4a6c075770 2024-05-20T20:05:25.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.305+0000 7f4a69600700 1 -- 172.21.15.28:0/1682287647 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a600044d0 con 0x7f4a6c075770 2024-05-20T20:05:25.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.305+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4a6c1a1190 con 0x7f4a6c075770 2024-05-20T20:05:25.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.305+0000 7f4a69600700 1 -- 172.21.15.28:0/1682287647 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a60003e30 con 0x7f4a6c075770 2024-05-20T20:05:25.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.305+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4a6c1a8a10 con 0x7f4a6c075770 2024-05-20T20:05:25.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.308+0000 7f4a69600700 1 -- 172.21.15.28:0/1682287647 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f4a60004000 con 0x7f4a6c075770 2024-05-20T20:05:25.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.309+0000 7f4a7e5a0b80 1 Processor -- start 2024-05-20T20:05:25.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a7e5a0b80 1 -- start start 2024-05-20T20:05:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c979a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a69600700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a540b2880 0x7f4a540b4d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c97ee0 0x55ac21c91ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c92570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ac21befdf0 con 0x55ac21c975c0 2024-05-20T20:05:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ac21bdf990 con 0x55ac21c97ee0 2024-05-20T20:05:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ac21be39d0 con 0x55ac21c920f0 2024-05-20T20:05:25.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a69600700 1 -- 172.21.15.28:0/1682287647 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f4a600e25e0 con 0x7f4a6c075770 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a71200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c979a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a71200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c979a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56404/0 (socket says 172.21.15.28:56404) 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a71200700 1 -- 172.21.15.28:0/1072721892 learned_addr learned my addr 172.21.15.28:0/1072721892 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a71c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a540b2880 0x7f4a540b4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.310+0000 7f4a6be00700 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c97ee0 0x55ac21c91ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71c00700 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c92570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71200700 1 -- 172.21.15.28:0/1072721892 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 msgr2=0x55ac21c92570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71200700 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c92570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71200700 1 -- 172.21.15.28:0/1072721892 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c97ee0 msgr2=0x55ac21c91ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71200700 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c97ee0 0x55ac21c91ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71200700 1 -- 172.21.15.28:0/1072721892 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ac21ba83d0 con 0x55ac21c975c0 2024-05-20T20:05:25.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a6be00700 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c97ee0 0x55ac21c91ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71200700 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c979a0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f4a600a5c40 tx=0x7f4a600a5b00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c13785b7437eafd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.311+0000 7f4a71c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a540b2880 0x7f4a540b4d40 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x55ac21c92ab0 tx=0x7f4a64009920 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.312+0000 7f4a5e000700 1 -- 172.21.15.28:0/1072721892 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a600a9330 con 0x55ac21c975c0 2024-05-20T20:05:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.312+0000 7f4a5e000700 1 -- 172.21.15.28:0/1072721892 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a600a96d0 con 0x55ac21c975c0 2024-05-20T20:05:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.312+0000 7f4a5e000700 1 -- 172.21.15.28:0/1072721892 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a600a5350 con 0x55ac21c975c0 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.312+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/1072721892 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 msgr2=0x55ac21c979a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.312+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c979a0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f4a600a5c40 tx=0x7f4a600a5b00 comp rx=0 tx=0).stop 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/1072721892 shutdown_connections 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c92570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c979a0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/1072721892 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c97ee0 0x55ac21c91ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/1072721892 >> 172.21.15.28:0/1072721892 conn(0x55ac21bbfb90 msgr2=0x55ac21c8c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/1072721892 shutdown_connections 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/1072721892 wait complete. 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.313+0000 7f4a7e5a0b80 1 -- start start 2024-05-20T20:05:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 1 Processor -- start 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21c9dc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c9b000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c97ee0 0x55ac21c9b540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ac21ae5f50 con 0x55ac21c975c0 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ac21b80710 con 0x55ac21c920f0 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ac21c9e990 con 0x55ac21c97ee0 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.314+0000 7f4a7e5a0b80 10 client.0 _ll_register_callbacks cb 0x55ac21b89c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c9b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c9b000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56418/0 (socket says 172.21.15.28:56418) 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 -- 172.21.15.28:0/3261312757 learned_addr learned my addr 172.21.15.28:0/3261312757 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a71200700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21c9dc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a71c00700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c97ee0 0x55ac21c9b540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 -- 172.21.15.28:0/3261312757 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c97ee0 msgr2=0x55ac21c9b540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c97ee0 0x55ac21c9b540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 -- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 msgr2=0x55ac21c9dc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21c9dc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a6be00700 1 -- 172.21.15.28:0/3261312757 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ac21b2f880 con 0x55ac21c975c0 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.315+0000 7f4a71200700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21c9dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.316+0000 7f4a6be00700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c9b000 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f4a58009990 tx=0x7f4a58005800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a4b4fa10b8497068 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:25.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.316+0000 7f4a52a00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:05:25.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.316+0000 7f4a52a00700 1 -- 172.21.15.28:0/3261312757 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a58029060 con 0x55ac21c975c0 2024-05-20T20:05:25.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.316+0000 7f4a52a00700 1 -- 172.21.15.28:0/3261312757 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f4a5800bd00 con 0x55ac21c975c0 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.316+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4a600a96d0 con 0x55ac21c975c0 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a52a00700 1 -- 172.21.15.28:0/3261312757 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f4a58003680 con 0x55ac21c975c0 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 10 client.14970 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55ac21bb0010 con 0x55ac21c975c0 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a52a00700 1 -- 172.21.15.28:0/3261312757 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a600a96d0 con 0x55ac21c975c0 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a52a00700 1 -- 172.21.15.28:0/3261312757 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f4a5802e070 con 0x55ac21c975c0 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 10 client.14970 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 10 client.14970 resolve_mds: resolved name 'b' to daemon mds.b 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 20 client.14970 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21cab500 0x55ac21cac970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 4 client.14970 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.317+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ac21bf1d40 con 0x55ac21cab500 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.318+0000 7f4a71200700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21cab500 0x55ac21cac970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.318+0000 7f4a71200700 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21cab500 0x55ac21cac970 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=688147bedd16ec01 server_cookie=f6629c63ae8c2fc6 in_seq=0 out_seq=0 2024-05-20T20:05:25.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.318+0000 7f4a52a00700 10 client.14970 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a52a00700 1 -- 172.21.15.28:0/3261312757 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55ac21bf1d40 con 0x55ac21cab500 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a52a00700 10 client.14970 handle_command_reply: tid=0 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 client.14970 shutdown 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 msgr2=0x55ac21c9b000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c9b000 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f4a58009990 tx=0x7f4a58005800 comp rx=0 tx=0).stop 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 shutdown_connections 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21cab500 0x55ac21cac970 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c97ee0 0x55ac21c9b540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21c9dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/3261312757 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c975c0 0x55ac21c9b000 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 >> 172.21.15.28:0/3261312757 conn(0x55ac21bbfb90 msgr2=0x55ac21c8bd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:25.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.319+0000 7f4a52a00700 0 client.14970 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:25.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.320+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 shutdown_connections 2024-05-20T20:05:25.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.320+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/3261312757 wait complete. 2024-05-20T20:05:25.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.320+0000 7f4a7e5a0b80 20 client.14970 trim_cache size 0 max 16384 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.343+0000 7f4a7e5a0b80 1 Processor -- start 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a7e5a0b80 1 -- start start 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c91ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c975c0 0x55ac21c8c4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c9ade0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ac21caacb0 con 0x55ac21c97ee0 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ac21cab380 con 0x55ac21c975c0 2024-05-20T20:05:25.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ac21caba50 con 0x55ac21c920f0 2024-05-20T20:05:25.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a6be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c975c0 0x55ac21c8c4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a71200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c91ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a6be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c975c0 0x55ac21c8c4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60768/0 (socket says 172.21.15.28:60768) 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a6be00700 1 -- 172.21.15.28:0/472658076 learned_addr learned my addr 172.21.15.28:0/472658076 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.344+0000 7f4a71c00700 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c9ade0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.345+0000 7f4a6be00700 1 -- 172.21.15.28:0/472658076 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 msgr2=0x55ac21c91ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.345+0000 7f4a6be00700 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c91ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.345+0000 7f4a6be00700 1 -- 172.21.15.28:0/472658076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 msgr2=0x55ac21c9ade0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.345+0000 7f4a6be00700 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c9ade0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.345+0000 7f4a6be00700 1 -- 172.21.15.28:0/472658076 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ac21c99570 con 0x55ac21c975c0 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.345+0000 7f4a6be00700 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c975c0 0x55ac21c8c4a0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f4a58024ac0 tx=0x7f4a58009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f81a97fd537d08f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a5d600700 1 -- 172.21.15.28:0/472658076 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a58034070 con 0x55ac21c975c0 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a5d600700 1 -- 172.21.15.28:0/472658076 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a58024b40 con 0x55ac21c975c0 2024-05-20T20:05:25.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a5d600700 1 -- 172.21.15.28:0/472658076 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a58029030 con 0x55ac21c975c0 2024-05-20T20:05:25.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/472658076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c975c0 msgr2=0x55ac21c8c4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c975c0 0x55ac21c8c4a0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f4a58024ac0 tx=0x7f4a58009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/472658076 shutdown_connections 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c9ade0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c920f0 0x55ac21c91ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/472658076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c975c0 0x55ac21c8c4a0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.346+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/472658076 >> 172.21.15.28:0/472658076 conn(0x55ac21bbfb90 msgr2=0x55ac21c95060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.347+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/472658076 shutdown_connections 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.347+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/472658076 wait complete. 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.347+0000 7f4a7e5a0b80 1 -- start start 2024-05-20T20:05:25.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.347+0000 7f4a7e5a0b80 1 Processor -- start 2024-05-20T20:05:25.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21cb6450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c975c0 0x55ac21c8b6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a7e5a0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c8bbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ac21cac7f0 con 0x55ac21c97ee0 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ac21cac120 con 0x55ac21c920f0 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a7e5a0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ac21cacec0 con 0x55ac21c975c0 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a7e5a0b80 10 client.0 _ll_register_callbacks cb 0x55ac21b89c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a71200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21cb6450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a71200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21cb6450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60778/0 (socket says 172.21.15.28:60778) 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a71200700 1 -- 172.21.15.28:0/2631934479 learned_addr learned my addr 172.21.15.28:0/2631934479 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a6be00700 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c975c0 0x55ac21c8b6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.348+0000 7f4a71c00700 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c8bbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a71200700 1 -- 172.21.15.28:0/2631934479 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c975c0 msgr2=0x55ac21c8b6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a71200700 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c975c0 0x55ac21c8b6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a71200700 1 -- 172.21.15.28:0/2631934479 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 msgr2=0x55ac21c8bbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a71200700 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c8bbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a71200700 1 -- 172.21.15.28:0/2631934479 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ac21c8c400 con 0x55ac21c920f0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a71200700 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21cb6450 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f4a600b24f0 tx=0x7f4a60003680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=130722dda6e7c1b2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a52a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a52a00700 1 -- 172.21.15.28:0/2631934479 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a600af700 con 0x55ac21c920f0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a52a00700 1 -- 172.21.15.28:0/2631934479 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f4a600c4170 con 0x55ac21c920f0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4a58024b40 con 0x55ac21c920f0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a52a00700 1 -- 172.21.15.28:0/2631934479 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=28) ==== 24+0+0 (secure 0 0 0) 0x7f4a600a96d0 con 0x55ac21c920f0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a7e5a0b80 10 client.25115 fetch_fsmap learned FSMap version 28 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.349+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=28,monmap=4+}) -- 0x55ac21d16c90 con 0x55ac21c920f0 2024-05-20T20:05:25.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a52a00700 1 -- 172.21.15.28:0/2631934479 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a58024b40 con 0x55ac21c920f0 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a52a00700 1 -- 172.21.15.28:0/2631934479 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 28) ==== 8354+0+0 (secure 0 0 0) 0x7f4a600e0120 con 0x55ac21c920f0 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a7e5a0b80 10 client.25115 fetch_fsmap finished waiting for FSMap version 28 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a7e5a0b80 10 client.25115 resolve_mds: resolved name 'b' to daemon mds.b 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a7e5a0b80 20 client.25115 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21d1b290 0x55ac21d19fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a7e5a0b80 4 client.25115 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "status"}) -- 0x55ac21beb520 con 0x55ac21d1b290 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.350+0000 7f4a6be00700 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21d1b290 0x55ac21d19fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a6be00700 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21d1b290 0x55ac21d19fa0 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=47150f0e52cbd38e server_cookie=34437a6ea328a859 in_seq=0 out_seq=0 2024-05-20T20:05:25.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a52a00700 10 client.25115 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:25.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a52a00700 1 -- 172.21.15.28:0/2631934479 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55ac21beb520 con 0x55ac21d1b290 2024-05-20T20:05:25.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a52a00700 10 client.25115 handle_command_reply: tid=0 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 client.25115 shutdown 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 msgr2=0x55ac21cb6450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21cb6450 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f4a600b24f0 tx=0x7f4a60003680 comp rx=0 tx=0).stop 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 shutdown_connections 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ac21d1b290 0x55ac21d19fa0 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ac21c97ee0 0x55ac21c8bbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ac21c920f0 0x55ac21cb6450 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 --2- 172.21.15.28:0/2631934479 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ac21c975c0 0x55ac21c8b6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a52a00700 0 client.25115 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.351+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 >> 172.21.15.28:0/2631934479 conn(0x55ac21bbfb90 msgr2=0x55ac21c9ade0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 shutdown_connections 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a7e5a0b80 1 -- 172.21.15.28:0/2631934479 wait complete. 2024-05-20T20:05:25.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a7e5a0b80 20 client.25115 trim_cache size 0 max 16384 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 0, 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24542, 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 238.83468166099999, 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 28, 2024-05-20T20:05:25.356 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 80, 2024-05-20T20:05:25.357 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 78, 2024-05-20T20:05:25.357 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 262.32726036100001 2024-05-20T20:05:25.357 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:05:25.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a540b2880 msgr2=0x7f4a540b4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a540b2880 0x7f4a540b4d40 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x55ac21c92ab0 tx=0x7f4a64009920 comp rx=0 tx=0).stop 2024-05-20T20:05:25.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 msgr2=0x7f4a6c19be90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 0x7f4a6c19be90 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f4a6000eb30 tx=0x7f4a6000ee40 comp rx=0 tx=0).stop 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 shutdown_connections 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4a540b2880 0x7f4a540b4d40 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a6c10ff40 0x7f4a6c1a0760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a6c075770 0x7f4a6c19be90 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 --2- 172.21.15.28:0/1682287647 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a6c076090 0x7f4a6c19c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.352+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 >> 172.21.15.28:0/1682287647 conn(0x7f4a6c07a6a0 msgr2=0x7f4a6c101bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.353+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 shutdown_connections 2024-05-20T20:05:25.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:25.353+0000 7f4a73c00700 1 -- 172.21.15.28:0/1682287647 wait complete. 2024-05-20T20:05:26.202 INFO:teuthology.run_tasks:Running task sequential... 2024-05-20T20:05:26.212 INFO:teuthology.task.sequential:In sequential, running task sequential... 2024-05-20T20:05:26.213 INFO:teuthology.task.sequential:In sequential, running task print... 2024-05-20T20:05:26.213 INFO:teuthology.task.print:Enabling mgr modules 2024-05-20T20:05:26.213 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-05-20T20:05:26.214 INFO:teuthology.task.exec:Executing custom commands... 2024-05-20T20:05:26.214 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:05:26.214 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph mgr module enable snap_schedule' 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.430+0000 7f7bb3e00640 1 -- 172.21.15.28:0/262930478 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac10a960 msgr2=0x7f7bac10cdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.430+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/262930478 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac10a960 0x7f7bac10cdf0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f7ba4009ca0 tx=0x7f7ba402a950 comp rx=0 tx=0).stop 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.431+0000 7f7bb3e00640 1 -- 172.21.15.28:0/262930478 shutdown_connections 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.431+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/262930478 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac10a960 0x7f7bac10cdf0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.431+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/262930478 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac075430 0x7f7bac075890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.431+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/262930478 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7bac076a10 0x7f7bac074e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.431+0000 7f7bb3e00640 1 -- 172.21.15.28:0/262930478 >> 172.21.15.28:0/262930478 conn(0x7f7bac0fe040 msgr2=0x7f7bac100460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.431+0000 7f7bb3e00640 1 -- 172.21.15.28:0/262930478 shutdown_connections 2024-05-20T20:05:26.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.431+0000 7f7bb3e00640 1 -- 172.21.15.28:0/262930478 wait complete. 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.432+0000 7f7bb3e00640 1 Processor -- start 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.432+0000 7f7bb3e00640 1 -- start start 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.432+0000 7f7bb3e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7bac075430 0x7f7bac19add0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb3e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac076a10 0x7f7bac19b310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac10a960 0x7f7bac1a2380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb3e00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7bac110820 con 0x7f7bac10a960 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb3e00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7bac1106a0 con 0x7f7bac076a10 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb3e00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7bac1109a0 con 0x7f7bac075430 2024-05-20T20:05:26.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac10a960 0x7f7bac1a2380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:26.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac10a960 0x7f7bac1a2380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56442/0 (socket says 172.21.15.28:56442) 2024-05-20T20:05:26.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1e00640 1 -- 172.21.15.28:0/1596946577 learned_addr learned my addr 172.21.15.28:0/1596946577 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:26.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1400640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7bac075430 0x7f7bac19add0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:26.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb0a00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac076a10 0x7f7bac19b310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1400640 1 -- 172.21.15.28:0/1596946577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac076a10 msgr2=0x7f7bac19b310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1400640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac076a10 0x7f7bac19b310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1400640 1 -- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac10a960 msgr2=0x7f7bac1a2380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1400640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac10a960 0x7f7bac1a2380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1400640 1 -- 172.21.15.28:0/1596946577 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7bac1a29f0 con 0x7f7bac075430 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.433+0000 7f7bb1e00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac10a960 0x7f7bac1a2380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.434+0000 7f7bb1400640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7bac075430 0x7f7bac19add0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f7ba000e9f0 tx=0x7f7ba000eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.434+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ba000ccc0 con 0x7f7bac075430 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.434+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7bac1a2ce0 con 0x7f7bac075430 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.434+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7ba00108e0 con 0x7f7bac075430 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.434+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ba0018880 con 0x7f7bac075430 2024-05-20T20:05:26.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.434+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7bac1a31d0 con 0x7f7bac075430 2024-05-20T20:05:26.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.435+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7bac109720 con 0x7f7bac075430 2024-05-20T20:05:26.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.438+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) ==== 149652+0+0 (secure 0 0 0) 0x7f7ba0010430 con 0x7f7bac075430 2024-05-20T20:05:26.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.439+0000 7f7baa000640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7b940b2920 0x7f7b940b4de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:26.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.439+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f7ba00e2510 con 0x7f7bac075430 2024-05-20T20:05:26.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.439+0000 7f7bb0a00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7b940b2920 0x7f7b940b4de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:26.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.440+0000 7f7bb0a00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7b940b2920 0x7f7b940b4de0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f7bac19c2e0 tx=0x7f7b98005f90 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:26.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.444+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7ba00a9ba0 con 0x7f7bac075430 2024-05-20T20:05:26.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:26.718+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "snap_schedule"} v 0) -- 0x7f7bac02dba0 con 0x7f7bac075430 2024-05-20T20:05:26.929 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:26 smithi033 ceph-mon[36179]: pgmap v441: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:27.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:26 smithi028 ceph-mon[31178]: pgmap v441: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:27.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:26 smithi154 ceph-mon[36314]: pgmap v441: 97 pgs: 97 active+clean; 645 KiB data, 339 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2024-05-20T20:05:27.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.686+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mgrmap(e 19) ==== 149669+0+0 (secure 0 0 0) 0x7f7ba00a5030 con 0x7f7bac075430 2024-05-20T20:05:27.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.687+0000 7f7baa000640 1 -- 172.21.15.28:0/1596946577 <== mon.1 v2:172.21.15.154:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "snap_schedule"}]=0 v19) ==== 92+0+0 (secure 0 0 0) 0x7f7bac02dba0 con 0x7f7bac075430 2024-05-20T20:05:27.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7b940b2920 msgr2=0x7f7b940b4de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:27.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7b940b2920 0x7f7b940b4de0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f7bac19c2e0 tx=0x7f7b98005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:27.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7bac075430 msgr2=0x7f7bac19add0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:27.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7bac075430 0x7f7bac19add0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f7ba000e9f0 tx=0x7f7ba000eeb0 comp rx=0 tx=0).stop 2024-05-20T20:05:27.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 shutdown_connections 2024-05-20T20:05:27.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f7b940b2920 0x7f7b940b4de0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:27.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7bac076a10 0x7f7bac19b310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:27.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7bac10a960 0x7f7bac1a2380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:27.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 --2- 172.21.15.28:0/1596946577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7bac075430 0x7f7bac19add0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:27.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.691+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 >> 172.21.15.28:0/1596946577 conn(0x7f7bac0fe040 msgr2=0x7f7bac1068f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:27.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.692+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 shutdown_connections 2024-05-20T20:05:27.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:27.692+0000 7f7bb3e00640 1 -- 172.21.15.28:0/1596946577 wait complete. 2024-05-20T20:05:27.714 INFO:teuthology.run_tasks:Running task kclient... 2024-05-20T20:05:27.741 INFO:tasks.kclient:Mounting kernel clients... 2024-05-20T20:05:27.742 INFO:tasks.kclient:config is {'client.0': {}} 2024-05-20T20:05:27.742 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:27.742 DEBUG:teuthology.orchestra.run.smithi028:> ip netns list 2024-05-20T20:05:27.793 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:27.793 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip link delete ceph-brx 2024-05-20T20:05:27.868 INFO:teuthology.orchestra.run.smithi028.stderr:Cannot find device "ceph-brx" 2024-05-20T20:05:27.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T20:05:27.871 INFO:tasks.kclient:client.0 config is {'mntopts': ['ms_mode=secure', 'nowsync'], 'syntax': 'v1', 'mount_subvol_num': 1} 2024-05-20T20:05:27.871 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-05-20T20:05:27.871 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-05-20T20:05:27.871 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs ls 2024-05-20T20:05:27.927 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:27 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: ignoring --setuser ceph since I am not root 2024-05-20T20:05:27.927 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:27 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: ignoring --setgroup ceph since I am not root 2024-05-20T20:05:27.927 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:27 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1596946577' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-05-20T20:05:27.927 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:27 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-05-20T20:05:28.035 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:27 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1596946577' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-05-20T20:05:28.035 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:27 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-05-20T20:05:28.035 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: ignoring --setuser ceph since I am not root 2024-05-20T20:05:28.035 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: ignoring --setgroup ceph since I am not root 2024-05-20T20:05:28.035 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:27 smithi154 ceph-mgr[38168]: -- 172.21.15.154:0/129187942 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564394e0e5a0 con 0x564394de3000 2024-05-20T20:05:28.035 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:27.850+0000 7f5a06e6b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T20:05:28.036 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:28.035+0000 7f5a06e6b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T20:05:28.054 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:27 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1596946577' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-05-20T20:05:28.054 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:27 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-05-20T20:05:28.054 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: ignoring --setuser ceph since I am not root 2024-05-20T20:05:28.054 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: ignoring --setgroup ceph since I am not root 2024-05-20T20:05:28.054 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:27 smithi033 ceph-mgr[37560]: -- 172.21.15.33:0/3216620709 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56323b7105a0 con 0x56323b6e4800 2024-05-20T20:05:28.054 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:27.860+0000 7f8ec1515200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T20:05:28.054 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:28.053+0000 7f8ec1515200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T20:05:28.275 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:28.009+0000 7f7df29cb200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-05-20T20:05:28.275 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:28.273+0000 7f7df29cb200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-05-20T20:05:28.298 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:28.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:28.137+0000 7f5a06e6b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T20:05:28.377 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:28.160+0000 7f8ec1515200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T20:05:28.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:28.417+0000 7f7df29cb200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-05-20T20:05:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:28 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-05-20T20:05:28.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:28 smithi154 ceph-mon[36314]: mgrmap e19: x(active, since 12m), standbys: z, y 2024-05-20T20:05:28.875 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:28.446+0000 7f5a06e6b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T20:05:28.875 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:28.532+0000 7f5a06e6b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T20:05:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:28 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-05-20T20:05:28.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:28 smithi033 ceph-mon[36179]: mgrmap e19: x(active, since 12m), standbys: z, y 2024-05-20T20:05:28.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:28.483+0000 7f8ec1515200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T20:05:28.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:28.573+0000 7f8ec1515200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T20:05:28.931 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:28.841+0000 7f7df29cb200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-05-20T20:05:28.931 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:28 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-05-20T20:05:28.931 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:28 smithi028 ceph-mon[31178]: mgrmap e19: x(active, since 12m), standbys: z, y 2024-05-20T20:05:29.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:28.961+0000 7f7df29cb200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-05-20T20:05:30.059 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:29.716+0000 7f5a06e6b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T20:05:30.059 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:29.887+0000 7f5a06e6b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T20:05:30.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:29.822+0000 7f8ec1515200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T20:05:30.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:30.000+0000 7f8ec1515200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T20:05:30.350 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:30.059+0000 7f5a06e6b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T20:05:30.479 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:30.176+0000 7f8ec1515200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T20:05:30.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:30.350+0000 7f5a06e6b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T20:05:30.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:30.444+0000 7f5a06e6b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T20:05:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.724+0000 7f8293800700 1 -- 172.21.15.28:0/4260846127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 msgr2=0x7f828c1099b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.724+0000 7f8293800700 1 --2- 172.21.15.28:0/4260846127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 0x7f828c1099b0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f827c009990 tx=0x7f827c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.725+0000 7f8293800700 1 -- 172.21.15.28:0/4260846127 shutdown_connections 2024-05-20T20:05:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.725+0000 7f8293800700 1 --2- 172.21.15.28:0/4260846127 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f828c103f80 0x7f828c110410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:30.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.725+0000 7f8293800700 1 --2- 172.21.15.28:0/4260846127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 0x7f828c1099b0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.725+0000 7f8293800700 1 --2- 172.21.15.28:0/4260846127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f828c1035c0 0x7f828c103a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.725+0000 7f8293800700 1 -- 172.21.15.28:0/4260846127 >> 172.21.15.28:0/4260846127 conn(0x7f828c07a6f0 msgr2=0x7f828c07ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.725+0000 7f8293800700 1 -- 172.21.15.28:0/4260846127 shutdown_connections 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.725+0000 7f8293800700 1 -- 172.21.15.28:0/4260846127 wait complete. 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 Processor -- start 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 -- start start 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f828c1035c0 0x7f828c19be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f828c103f80 0x7f828c19c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 0x7f828c1a0700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f828c113790 con 0x7f828c1035c0 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f828c113610 con 0x7f828c1095d0 2024-05-20T20:05:30.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.726+0000 7f8293800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f828c113910 con 0x7f828c103f80 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8290e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f828c1035c0 0x7f828c19be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8290e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f828c1035c0 0x7f828c19be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40178/0 (socket says 172.21.15.28:40178) 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8283e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f828c103f80 0x7f828c19c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8290e00700 1 -- 172.21.15.28:0/3427072363 learned_addr learned my addr 172.21.15.28:0/3427072363 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8291800700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 0x7f828c1a0700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8283e00700 1 -- 172.21.15.28:0/3427072363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 msgr2=0x7f828c1a0700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8283e00700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 0x7f828c1a0700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8283e00700 1 -- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f828c1035c0 msgr2=0x7f828c19be30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8283e00700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f828c1035c0 0x7f828c19be30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8283e00700 1 -- 172.21.15.28:0/3427072363 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f828c1a0e40 con 0x7f828c103f80 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.727+0000 7f8283e00700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f828c103f80 0x7f828c19c370 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f827400e9f0 tx=0x7f827400ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.728+0000 7f828a000700 1 -- 172.21.15.28:0/3427072363 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8274004220 con 0x7f828c103f80 2024-05-20T20:05:30.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.728+0000 7f828a000700 1 -- 172.21.15.28:0/3427072363 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f82740043c0 con 0x7f828c103f80 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.728+0000 7f828a000700 1 -- 172.21.15.28:0/3427072363 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8274005020 con 0x7f828c103f80 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.728+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f828c1a1130 con 0x7f828c103f80 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.728+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f828c1a89b0 con 0x7f828c103f80 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.729+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f828c06a230 con 0x7f828c103f80 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.733+0000 7f828a000700 1 -- 172.21.15.28:0/3427072363 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) ==== 149669+0+0 (secure 0 0 0) 0x7f8274004750 con 0x7f828c103f80 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.734+0000 7f828a000700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 0x7f82700b4d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.734+0000 7f8290e00700 1 -- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 msgr2=0x7f82700b4d90 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/2019900521 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.734+0000 7f828a000700 1 -- 172.21.15.28:0/3427072363 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f82740e2230 con 0x7f828c103f80 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.735+0000 7f8290e00700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 0x7f82700b4d90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T20:05:30.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.738+0000 7f828a000700 1 -- 172.21.15.28:0/3427072363 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f82740a9f10 con 0x7f828c103f80 2024-05-20T20:05:30.786 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:30.478+0000 7f8ec1515200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T20:05:30.786 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:30.575+0000 7f8ec1515200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T20:05:30.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.936+0000 7f8290e00700 1 -- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 msgr2=0x7f82700b4d90 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/2019900521 2024-05-20T20:05:30.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:30.936+0000 7f8290e00700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 0x7f82700b4d90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-05-20T20:05:30.938 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:30 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:30.580+0000 7f7df29cb200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-05-20T20:05:30.938 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:30 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:30.823+0000 7f7df29cb200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-05-20T20:05:31.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.024+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs ls"} v 0) -- 0x7f828c19d190 con 0x7f828c103f80 2024-05-20T20:05:31.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.025+0000 7f828a000700 1 -- 172.21.15.28:0/3427072363 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v28) ==== 53+0+73 (secure 0 0 0) 0x7f82740a9810 con 0x7f828c103f80 2024-05-20T20:05:31.030 INFO:teuthology.orchestra.run.smithi028.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-05-20T20:05:31.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 msgr2=0x7f82700b4d90 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:05:31.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 0x7f82700b4d90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:31.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f828c103f80 msgr2=0x7f828c19c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:31.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f828c103f80 0x7f828c19c370 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f827400e9f0 tx=0x7f827400ed00 comp rx=0 tx=0).stop 2024-05-20T20:05:31.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 shutdown_connections 2024-05-20T20:05:31.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f82700b28d0 0x7f82700b4d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:31.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f828c1095d0 0x7f828c1a0700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:31.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f828c1035c0 0x7f828c19be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:31.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 --2- 172.21.15.28:0/3427072363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f828c103f80 0x7f828c19c370 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:31.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 >> 172.21.15.28:0/3427072363 conn(0x7f828c07a6f0 msgr2=0x7f828c075a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:31.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.028+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 shutdown_connections 2024-05-20T20:05:31.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:31.029+0000 7f8293800700 1 -- 172.21.15.28:0/3427072363 wait complete. 2024-05-20T20:05:31.078 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:30.649+0000 7f5a06e6b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T20:05:31.079 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:30.844+0000 7f5a06e6b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T20:05:31.079 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:30.785+0000 7f8ec1515200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T20:05:31.079 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:30.987+0000 7f8ec1515200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T20:05:31.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:31 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3427072363' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-05-20T20:05:31.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:31 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:31.047+0000 7f7df29cb200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-05-20T20:05:31.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:31 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3427072363' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-05-20T20:05:31.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:31 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3427072363' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-05-20T20:05:31.796 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:31 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:31.454+0000 7f7df29cb200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-05-20T20:05:31.796 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:31 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:31.587+0000 7f7df29cb200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-05-20T20:05:31.856 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-05-20T20:05:31.856 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-05-20T20:05:31.856 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi028.front.sepia.ceph.com 2024-05-20T20:05:31.856 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-05-20T20:05:31.856 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-05-20T20:05:31.856 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-05-20T20:05:31.856 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-05-20T20:05:31.857 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-05-20T20:05:31.857 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.0' 2024-05-20T20:05:31.857 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:31.857 DEBUG:teuthology.orchestra.run.smithi028:> ip addr 2024-05-20T20:05:31.875 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:31.450+0000 7f5a06e6b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T20:05:31.875 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:31.673+0000 7f5a06e6b200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T20:05:31.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:31.621+0000 7f8ec1515200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T20:05:31.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:31.850+0000 7f8ec1515200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T20:05:31.909 INFO:teuthology.orchestra.run.smithi028.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-05-20T20:05:31.909 INFO:teuthology.orchestra.run.smithi028.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-05-20T20:05:31.909 INFO:teuthology.orchestra.run.smithi028.stdout: inet 127.0.0.1/8 scope host lo 2024-05-20T20:05:31.910 INFO:teuthology.orchestra.run.smithi028.stdout: valid_lft forever preferred_lft forever 2024-05-20T20:05:31.910 INFO:teuthology.orchestra.run.smithi028.stdout: inet6 ::1/128 scope host 2024-05-20T20:05:31.910 INFO:teuthology.orchestra.run.smithi028.stdout: valid_lft forever preferred_lft forever 2024-05-20T20:05:31.910 INFO:teuthology.orchestra.run.smithi028.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-05-20T20:05:31.910 INFO:teuthology.orchestra.run.smithi028.stdout: link/ether 0c:c4:7a:6b:ee:16 brd ff:ff:ff:ff:ff:ff 2024-05-20T20:05:31.911 INFO:teuthology.orchestra.run.smithi028.stdout: altname enp7s0f0 2024-05-20T20:05:31.911 INFO:teuthology.orchestra.run.smithi028.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-05-20T20:05:31.911 INFO:teuthology.orchestra.run.smithi028.stdout: link/ether 0c:c4:7a:6b:ee:17 brd ff:ff:ff:ff:ff:ff 2024-05-20T20:05:31.911 INFO:teuthology.orchestra.run.smithi028.stdout: altname enp7s0f1 2024-05-20T20:05:31.911 INFO:teuthology.orchestra.run.smithi028.stdout:4: ens1f0: mtu 1500 qdisc mq state UP group default qlen 1000 2024-05-20T20:05:31.912 INFO:teuthology.orchestra.run.smithi028.stdout: link/ether ac:1f:6b:a5:ca:3c brd ff:ff:ff:ff:ff:ff 2024-05-20T20:05:31.912 INFO:teuthology.orchestra.run.smithi028.stdout: altname enp1s0f0 2024-05-20T20:05:31.912 INFO:teuthology.orchestra.run.smithi028.stdout: inet 172.21.15.28/20 brd 172.21.15.255 scope global dynamic ens1f0 2024-05-20T20:05:31.912 INFO:teuthology.orchestra.run.smithi028.stdout: valid_lft 41773sec preferred_lft 41773sec 2024-05-20T20:05:31.912 INFO:teuthology.orchestra.run.smithi028.stdout: inet6 fe80::ae1f:6bff:fea5:ca3c/64 scope link noprefixroute 2024-05-20T20:05:31.912 INFO:teuthology.orchestra.run.smithi028.stdout: valid_lft forever preferred_lft forever 2024-05-20T20:05:31.913 INFO:teuthology.orchestra.run.smithi028.stdout:5: ens1f1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-05-20T20:05:31.913 INFO:teuthology.orchestra.run.smithi028.stdout: link/ether ac:1f:6b:a5:ca:3d brd ff:ff:ff:ff:ff:ff 2024-05-20T20:05:31.913 INFO:teuthology.orchestra.run.smithi028.stdout: altname enp1s0f1 2024-05-20T20:05:31.914 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-05-20T20:05:31.914 DEBUG:teuthology.orchestra.run.smithi028:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-05-20T20:05:31.914 DEBUG:teuthology.orchestra.run.smithi028:> set -e 2024-05-20T20:05:31.914 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip link add name ceph-brx type bridge 2024-05-20T20:05:31.914 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip addr flush dev ceph-brx 2024-05-20T20:05:31.915 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip link set ceph-brx up 2024-05-20T20:05:31.915 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-05-20T20:05:31.915 DEBUG:teuthology.orchestra.run.smithi028:> ') 2024-05-20T20:05:32.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:32 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-05-20T20:05:32.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:31 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:31.868+0000 7f7df29cb200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-05-20T20:05:32.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:32 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:32.370 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:32.370 DEBUG:teuthology.orchestra.run.smithi028:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-05-20T20:05:32.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:32.095+0000 7f5a06e6b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T20:05:32.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:32.191+0000 7f5a06e6b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T20:05:32.409 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:32 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:32.145+0000 7f7df29cb200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-05-20T20:05:32.437 INFO:teuthology.orchestra.run.smithi028.stdout:1 2024-05-20T20:05:32.440 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:32.441 DEBUG:teuthology.orchestra.run.smithi028:> route 2024-05-20T20:05:32.559 INFO:teuthology.orchestra.run.smithi028.stdout:Kernel IP routing table 2024-05-20T20:05:32.559 INFO:teuthology.orchestra.run.smithi028.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-05-20T20:05:32.559 INFO:teuthology.orchestra.run.smithi028.stdout:default _gateway 0.0.0.0 UG 100 0 0 ens1f0 2024-05-20T20:05:32.559 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 ens1f0 2024-05-20T20:05:32.560 INFO:teuthology.orchestra.run.smithi028.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-05-20T20:05:32.561 DEBUG:teuthology.orchestra.run.smithi028:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-05-20T20:05:32.561 DEBUG:teuthology.orchestra.run.smithi028:> set -e 2024-05-20T20:05:32.561 DEBUG:teuthology.orchestra.run.smithi028:> sudo iptables -A FORWARD -o ens1f0 -i ceph-brx -j ACCEPT 2024-05-20T20:05:32.561 DEBUG:teuthology.orchestra.run.smithi028:> sudo iptables -A FORWARD -i ens1f0 -o ceph-brx -j ACCEPT 2024-05-20T20:05:32.561 DEBUG:teuthology.orchestra.run.smithi028:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o ens1f0 -j MASQUERADE 2024-05-20T20:05:32.561 DEBUG:teuthology.orchestra.run.smithi028:> ') 2024-05-20T20:05:32.627 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:32.290+0000 7f8ec1515200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T20:05:32.627 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:32.391+0000 7f8ec1515200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T20:05:32.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:32 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-05-20T20:05:32.875 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:32.564+0000 7f5a06e6b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T20:05:33.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:32.775+0000 7f8ec1515200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T20:05:33.263 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:32.924+0000 7f5a06e6b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T20:05:33.263 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:33.017+0000 7f5a06e6b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T20:05:33.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:32.998+0000 7f7df29cb200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-05-20T20:05:33.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:33 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:33.327 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:33.327 DEBUG:teuthology.orchestra.run.smithi028:> ip netns list 2024-05-20T20:05:33.376 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:33.377 DEBUG:teuthology.orchestra.run.smithi028:> ip netns list-id 2024-05-20T20:05:33.429 DEBUG:teuthology.orchestra.run.smithi028:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-05-20T20:05:33.429 DEBUG:teuthology.orchestra.run.smithi028:> set -e 2024-05-20T20:05:33.429 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.0 2024-05-20T20:05:33.429 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.0 0 2024-05-20T20:05:33.429 DEBUG:teuthology.orchestra.run.smithi028:> ') 2024-05-20T20:05:33.501 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:33.148+0000 7f8ec1515200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T20:05:33.501 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:33.245+0000 7f8ec1515200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T20:05:33.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:33 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-05-20T20:05:33.556 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:33.302+0000 7f7df29cb200 -1 mgr[py] Module smb has missing NOTIFY_TYPES member 2024-05-20T20:05:33.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:33 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:33.603 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.0' with 192.168.0.1/16 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> set -e 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.0 type veth peer name brx.0 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set veth0 up 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set lo up 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip route add default via 192.168.255.254 2024-05-20T20:05:33.603 DEBUG:teuthology.orchestra.run.smithi028:> ') 2024-05-20T20:05:33.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:33.262+0000 7f5a06e6b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T20:05:33.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:33.397+0000 7f5a06e6b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T20:05:33.625 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:33.493+0000 7f5a06e6b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T20:05:33.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:33 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-05-20T20:05:33.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:33.499+0000 7f8ec1515200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T20:05:33.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:33.638+0000 7f8ec1515200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T20:05:33.877 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:33.739+0000 7f8ec1515200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T20:05:33.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:33 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:33.890 DEBUG:teuthology.orchestra.run.smithi028:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-05-20T20:05:33.891 DEBUG:teuthology.orchestra.run.smithi028:> set -e 2024-05-20T20:05:33.891 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip link set brx.0 up 2024-05-20T20:05:33.893 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip link set dev brx.0 master ceph-brx 2024-05-20T20:05:33.893 DEBUG:teuthology.orchestra.run.smithi028:> ') 2024-05-20T20:05:33.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:33 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-05-20T20:05:34.067 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:33.908+0000 7f7df29cb200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-05-20T20:05:34.068 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:34 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:34.053+0000 7f7df29cb200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-05-20T20:05:34.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:34 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:34.102 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-05-20T20:05:34.102 DEBUG:teuthology.orchestra.run.smithi028:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-05-20T20:05:34.125 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:33.824+0000 7f5a06e6b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T20:05:34.125 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:33.916+0000 7f5a06e6b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T20:05:34.148 INFO:teuthology.orchestra.run.smithi028.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-05-20T20:05:34.148 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-05-20T20:05:34.148 DEBUG:teuthology.orchestra.run.smithi028:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-05-20T20:05:34.241 WARNING:tasks.cephfs.mount:You cannot specify both: cephfs_mntpt and mount_subvol_num 2024-05-20T20:05:34.241 INFO:tasks.cephfs.mount:Mounting subvol 1 for now 2024-05-20T20:05:34.241 INFO:tasks.cephfs.kernel_mount:mounting using device: :/volumes/qa/sv_1/9e9b2651-9ad5-4ff0-99f5-cbf91c762d93 2024-05-20T20:05:34.241 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:34.241 DEBUG:teuthology.orchestra.run.smithi028:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph :/volumes/qa/sv_1/9e9b2651-9ad5-4ff0-99f5-cbf91c762d93 /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,nowsync 2024-05-20T20:05:34.370 INFO:teuthology.orchestra.run.smithi028.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,nowsync 2024-05-20T20:05:34.370 INFO:teuthology.orchestra.run.smithi028.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,nowsync". 2024-05-20T20:05:34.370 INFO:teuthology.orchestra.run.smithi028.stdout:invalid new device string format 2024-05-20T20:05:34.377 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:34.077+0000 7f8ec1515200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T20:05:34.377 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:34.173+0000 7f8ec1515200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T20:05:34.806 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:34 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:34.560+0000 7f7df29cb200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-05-20T20:05:34.817 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:34.518+0000 7f5a06e6b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T20:05:34.817 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:05:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z[38164]: 2024-05-20T20:05:34.735+0000 7f5a06e6b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T20:05:35.095 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:34 smithi028 ceph-mon[31178]: Standby manager daemon z restarted 2024-05-20T20:05:35.095 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:34 smithi028 ceph-mon[31178]: Standby manager daemon z started 2024-05-20T20:05:35.096 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-05-20T20:05:35.096 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T20:05:35.096 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-05-20T20:05:35.096 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:34 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T20:05:35.096 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:35.048+0000 7f7df29cb200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-05-20T20:05:35.122 INFO:teuthology.orchestra.run.smithi028.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,nowsync 2024-05-20T20:05:35.123 INFO:teuthology.orchestra.run.smithi028.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,nowsync". 2024-05-20T20:05:35.123 INFO:teuthology.orchestra.run.smithi028.stdout:invalid new device string format 2024-05-20T20:05:35.123 INFO:teuthology.orchestra.run.smithi028.stdout:mount.ceph: resolved to: "172.21.15.28:3300,172.21.15.33:3300,172.21.15.154:3300" 2024-05-20T20:05:35.123 INFO:teuthology.orchestra.run.smithi028.stdout:mount.ceph: trying mount with old device syntax: 172.21.15.28:3300,172.21.15.33:3300,172.21.15.154:3300:/volumes/qa/sv_1/9e9b2651-9ad5-4ff0-99f5-cbf91c762d93 2024-05-20T20:05:35.123 INFO:teuthology.orchestra.run.smithi028.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,nowsync,key=0,fsid=3fbec9b2-16e2-11ef-bc9a-c7b262605968" will pass to kernel 2024-05-20T20:05:35.124 INFO:teuthology.orchestra.run.smithi028.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-05-20T20:05:35.124 INFO:teuthology.orchestra.run.smithi028.stdout: You just mounted a file system that supports labels which does not 2024-05-20T20:05:35.124 INFO:teuthology.orchestra.run.smithi028.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-05-20T20:05:35.124 INFO:teuthology.orchestra.run.smithi028.stdout: applications will generate AVC messages and not be allowed access to 2024-05-20T20:05:35.124 INFO:teuthology.orchestra.run.smithi028.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-05-20T20:05:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:34 smithi154 ceph-mon[36314]: Standby manager daemon z restarted 2024-05-20T20:05:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:34 smithi154 ceph-mon[36314]: Standby manager daemon z started 2024-05-20T20:05:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-05-20T20:05:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T20:05:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-05-20T20:05:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:34 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T20:05:35.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:34.810+0000 7f8ec1515200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T20:05:35.127 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:05:35 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y[37556]: 2024-05-20T20:05:35.038+0000 7f8ec1515200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T20:05:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:34 smithi033 ceph-mon[36179]: Standby manager daemon z restarted 2024-05-20T20:05:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:34 smithi033 ceph-mon[36179]: Standby manager daemon z started 2024-05-20T20:05:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-05-20T20:05:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T20:05:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-05-20T20:05:35.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:34 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.154:0/3518933656' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T20:05:35.128 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-05-20T20:05:35.128 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:05:35.129 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-05-20T20:05:35.208 DEBUG:teuthology.orchestra.run.smithi028:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-05-20T20:05:35.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-05-20T20:05:35.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:35.327 INFO:teuthology.orchestra.run.smithi028.stdout:client15006 2024-05-20T20:05:35.328 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T20:05:35.328 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/proc/self/mounts of=/dev/stdout 2024-05-20T20:05:35.374 DEBUG:teuthology.orchestra.run.smithi028:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-05-20T20:05:35.486 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:35.185+0000 7f7df29cb200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-05-20T20:05:35.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-05-20T20:05:35.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:35.541 INFO:teuthology.orchestra.run.smithi028.stdout:client15006 2024-05-20T20:05:35.542 DEBUG:teuthology.orchestra.run.smithi028:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-05-20T20:05:35.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-05-20T20:05:35.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:35.653 INFO:teuthology.orchestra.run.smithi028.stdout:client15006 2024-05-20T20:05:35.653 DEBUG:teuthology.orchestra.run.smithi028:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=300 -- bash -c 'sudo dd if=/sys/kernel/debug/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968.client15006/status') 2024-05-20T20:05:35.745 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:35.549+0000 7f7df29cb200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-05-20T20:05:35.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-05-20T20:05:35.780 INFO:teuthology.orchestra.run.smithi028.stdout:instance: client.15006 (3)192.168.0.1:0/1116450425 2024-05-20T20:05:35.781 INFO:teuthology.orchestra.run.smithi028.stdout:blocklisted: false 2024-05-20T20:05:35.781 INFO:teuthology.orchestra.run.smithi028.stderr:0+1 records in 2024-05-20T20:05:35.781 INFO:teuthology.orchestra.run.smithi028.stderr:0+1 records out 2024-05-20T20:05:35.781 INFO:teuthology.orchestra.run.smithi028.stderr:70 bytes copied, 9.3615e-05 s, 748 kB/s 2024-05-20T20:05:35.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-05-20T20:05:35.792 INFO:teuthology.run_tasks:Running task exec... 2024-05-20T20:05:35.801 INFO:teuthology.task.exec:Executing custom commands... 2024-05-20T20:05:35.802 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:05:35.802 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs set cephfs balance_automate false' 2024-05-20T20:05:35.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 -- 172.21.15.28:0/748494486 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c105e60 msgr2=0x7fde6c108250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:35.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 --2- 172.21.15.28:0/748494486 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c105e60 0x7fde6c108250 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fde58009900 tx=0x7fde5802a3f0 comp rx=0 tx=0).stop 2024-05-20T20:05:35.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 -- 172.21.15.28:0/748494486 shutdown_connections 2024-05-20T20:05:35.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 --2- 172.21.15.28:0/748494486 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fde6c108790 0x7fde6c10abe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:35.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 --2- 172.21.15.28:0/748494486 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c105e60 0x7fde6c108250 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:35.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 --2- 172.21.15.28:0/748494486 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fde6c103530 0x7fde6c105920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:35.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 -- 172.21.15.28:0/748494486 >> 172.21.15.28:0/748494486 conn(0x7fde6c0fcf40 msgr2=0x7fde6c0ff360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:35.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 -- 172.21.15.28:0/748494486 shutdown_connections 2024-05-20T20:05:35.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.983+0000 7fde73e00640 1 -- 172.21.15.28:0/748494486 wait complete. 2024-05-20T20:05:35.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 Processor -- start 2024-05-20T20:05:35.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 -- start start 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fde6c103530 0x7fde6c197670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fde6c105e60 0x7fde6c197bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c108790 0x7fde6c1941e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fde6c10e620 con 0x7fde6c108790 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fde6c10e4a0 con 0x7fde6c103530 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.985+0000 7fde73e00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fde6c10e7a0 con 0x7fde6c105e60 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c108790 0x7fde6c1941e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:35.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c108790 0x7fde6c1941e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40196/0 (socket says 172.21.15.28:40196) 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 -- 172.21.15.28:0/3172282555 learned_addr learned my addr 172.21.15.28:0/3172282555 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71400640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fde6c103530 0x7fde6c197670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde70a00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fde6c105e60 0x7fde6c197bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 -- 172.21.15.28:0/3172282555 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fde6c105e60 msgr2=0x7fde6c197bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fde6c105e60 0x7fde6c197bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 -- 172.21.15.28:0/3172282555 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fde6c103530 msgr2=0x7fde6c197670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fde6c103530 0x7fde6c197670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:35.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.986+0000 7fde71e00640 1 -- 172.21.15.28:0/3172282555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fde6c194aa0 con 0x7fde6c108790 2024-05-20T20:05:35.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.987+0000 7fde71e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c108790 0x7fde6c1941e0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fde6400eb10 tx=0x7fde6400c2d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:35.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.987+0000 7fde6a000640 1 -- 172.21.15.28:0/3172282555 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde64004b20 con 0x7fde6c108790 2024-05-20T20:05:35.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.987+0000 7fde6a000640 1 -- 172.21.15.28:0/3172282555 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fde64004cc0 con 0x7fde6c108790 2024-05-20T20:05:35.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.987+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fde6c1a72a0 con 0x7fde6c108790 2024-05-20T20:05:35.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.987+0000 7fde6a000640 1 -- 172.21.15.28:0/3172282555 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fde64010670 con 0x7fde6c108790 2024-05-20T20:05:35.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.987+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fde6c1a76a0 con 0x7fde6c108790 2024-05-20T20:05:35.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.988+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fde6c0507e0 con 0x7fde6c108790 2024-05-20T20:05:35.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.992+0000 7fde6a000640 1 -- 172.21.15.28:0/3172282555 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 21) ==== 149669+0+0 (secure 0 0 0) 0x7fde64005000 con 0x7fde6c108790 2024-05-20T20:05:35.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.995+0000 7fde6a000640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 0x7fde540b4e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:35.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.995+0000 7fde71400640 1 -- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 msgr2=0x7fde540b4e30 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/2019900521 2024-05-20T20:05:35.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.995+0000 7fde6a000640 1 -- 172.21.15.28:0/3172282555 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7fde640e29d0 con 0x7fde6c108790 2024-05-20T20:05:35.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.995+0000 7fde71400640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 0x7fde540b4e30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T20:05:35.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:35.996+0000 7fde6a000640 1 -- 172.21.15.28:0/3172282555 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fde640a6240 con 0x7fde6c108790 2024-05-20T20:05:36.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:35.744+0000 7f7df29cb200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:35.892+0000 7f7df29cb200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:35 smithi028 ceph-mon[31178]: mgrmap e20: x(active, since 12m), standbys: y, z 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:35 smithi028 ceph-mon[31178]: Standby manager daemon y restarted 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:35 smithi028 ceph-mon[31178]: Standby manager daemon y started 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:35 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:35 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:35 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T20:05:36.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:35 smithi028 ceph-mon[31178]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T20:05:36.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:35 smithi154 ceph-mon[36314]: mgrmap e20: x(active, since 12m), standbys: y, z 2024-05-20T20:05:36.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:35 smithi154 ceph-mon[36314]: Standby manager daemon y restarted 2024-05-20T20:05:36.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:35 smithi154 ceph-mon[36314]: Standby manager daemon y started 2024-05-20T20:05:36.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:35 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T20:05:36.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:35 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T20:05:36.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:35 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T20:05:36.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:35 smithi154 ceph-mon[36314]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T20:05:36.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:35 smithi033 ceph-mon[36179]: mgrmap e20: x(active, since 12m), standbys: y, z 2024-05-20T20:05:36.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:35 smithi033 ceph-mon[36179]: Standby manager daemon y restarted 2024-05-20T20:05:36.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:35 smithi033 ceph-mon[36179]: Standby manager daemon y started 2024-05-20T20:05:36.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:35 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-05-20T20:05:36.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:35 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T20:05:36.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:35 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-05-20T20:05:36.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:35 smithi033 ceph-mon[36179]: from='mgr.? 172.21.15.33:0/3585286838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T20:05:36.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.196+0000 7fde71400640 1 -- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 msgr2=0x7fde540b4e30 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/2019900521 2024-05-20T20:05:36.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.196+0000 7fde71400640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 0x7fde540b4e30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-05-20T20:05:36.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.253+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"} v 0) -- 0x7fde6c10a2e0 con 0x7fde6c108790 2024-05-20T20:05:36.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.597+0000 7fde71400640 1 -- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 msgr2=0x7fde540b4e30 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/2019900521 2024-05-20T20:05:36.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.597+0000 7fde71400640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 0x7fde540b4e30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-05-20T20:05:36.722 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:36 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:36.318+0000 7f7df29cb200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-05-20T20:05:36.722 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:36 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:36.451+0000 7f7df29cb200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-05-20T20:05:36.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.830+0000 7fde6a000640 1 -- 172.21.15.28:0/3172282555 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]=0 v29) ==== 118+0+0 (secure 0 0 0) 0x7fde6c10a2e0 con 0x7fde6c108790 2024-05-20T20:05:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 msgr2=0x7fde540b4e30 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:05:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 0x7fde540b4e30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c108790 msgr2=0x7fde6c1941e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c108790 0x7fde6c1941e0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fde6400eb10 tx=0x7fde6400c2d0 comp rx=0 tx=0).stop 2024-05-20T20:05:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 shutdown_connections 2024-05-20T20:05:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7fde540b2970 0x7fde540b4e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fde6c108790 0x7fde6c1941e0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fde6c105e60 0x7fde6c197bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 --2- 172.21.15.28:0/3172282555 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fde6c103530 0x7fde6c197670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 >> 172.21.15.28:0/3172282555 conn(0x7fde6c0fcf40 msgr2=0x7fde6c102c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 shutdown_connections 2024-05-20T20:05:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:36.833+0000 7fde73e00640 1 -- 172.21.15.28:0/3172282555 wait complete. 2024-05-20T20:05:36.854 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs subvolumegroup pin cephfs qa random 0.10' 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.044+0000 7f4dbda00640 1 -- 172.21.15.28:0/211824837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db8103ad0 msgr2=0x7f4db8103f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.044+0000 7f4dbda00640 1 --2- 172.21.15.28:0/211824837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db8103ad0 0x7f4db8103f50 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f4da4009900 tx=0x7f4da402a410 comp rx=0 tx=0).stop 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.044+0000 7f4dbda00640 1 -- 172.21.15.28:0/211824837 shutdown_connections 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.044+0000 7f4dbda00640 1 --2- 172.21.15.28:0/211824837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db8103ad0 0x7f4db8103f50 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.044+0000 7f4dbda00640 1 --2- 172.21.15.28:0/211824837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4db8104490 0x7f4db810ad20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.044+0000 7f4dbda00640 1 --2- 172.21.15.28:0/211824837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4db81028e0 0x7f4db8102d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.044+0000 7f4dbda00640 1 -- 172.21.15.28:0/211824837 >> 172.21.15.28:0/211824837 conn(0x7f4db80fde50 msgr2=0x7f4db8100290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.045+0000 7f4dbda00640 1 -- 172.21.15.28:0/211824837 shutdown_connections 2024-05-20T20:05:37.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.045+0000 7f4dbda00640 1 -- 172.21.15.28:0/211824837 wait complete. 2024-05-20T20:05:37.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.045+0000 7f4dbda00640 1 Processor -- start 2024-05-20T20:05:37.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbda00640 1 -- start start 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbda00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db81028e0 0x7f4db8198b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbda00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4db8103ad0 0x7f4db8199040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbda00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4db8104490 0x7f4db81a00b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbda00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4db810e440 con 0x7f4db8103ad0 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbda00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4db810e2c0 con 0x7f4db81028e0 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbda00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4db810e5c0 con 0x7f4db8104490 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4db7400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4db8103ad0 0x7f4db8199040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4db7e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db81028e0 0x7f4db8198b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4db7400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4db8103ad0 0x7f4db8199040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57300/0 (socket says 172.21.15.28:57300) 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4db7e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db81028e0 0x7f4db8198b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37698/0 (socket says 172.21.15.28:37698) 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4db7400640 1 -- 172.21.15.28:0/339814748 learned_addr learned my addr 172.21.15.28:0/339814748 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.046+0000 7f4dbd000640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4db8104490 0x7f4db81a00b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:37.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db7e00640 1 -- 172.21.15.28:0/339814748 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4db8104490 msgr2=0x7f4db81a00b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db7e00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4db8104490 0x7f4db81a00b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db7e00640 1 -- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4db8103ad0 msgr2=0x7f4db8199040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db7e00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4db8103ad0 0x7f4db8199040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db7e00640 1 -- 172.21.15.28:0/339814748 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4db81a0720 con 0x7f4db81028e0 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db7e00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db81028e0 0x7f4db8198b00 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4da800e9c0 tx=0x7f4da800ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4da800ccc0 con 0x7f4db81028e0 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4da80035f0 con 0x7f4db81028e0 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4db81a0a10 con 0x7f4db81028e0 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.047+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4da800ce60 con 0x7f4db81028e0 2024-05-20T20:05:37.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.048+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4db81a0fb0 con 0x7f4db81028e0 2024-05-20T20:05:37.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.049+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4db8109cd0 con 0x7f4db81028e0 2024-05-20T20:05:37.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.051+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 21) ==== 149669+0+0 (secure 0 0 0) 0x7f4da8004000 con 0x7f4db81028e0 2024-05-20T20:05:37.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.052+0000 7f4db4c00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4d9c0b28b0 0x7f4d9c0b4d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:37.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.052+0000 7f4db7400640 1 -- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4d9c0b28b0 msgr2=0x7f4d9c0b4d70 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/2019900521 2024-05-20T20:05:37.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.052+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(80..80 src has 1..80) ==== 8240+0+0 (secure 0 0 0) 0x7f4da80e2730 con 0x7f4db81028e0 2024-05-20T20:05:37.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.052+0000 7f4db7400640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4d9c0b28b0 0x7f4d9c0b4d70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-05-20T20:05:37.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:36 smithi028 ceph-mon[31178]: mgrmap e21: x(active, since 12m), standbys: z, y 2024-05-20T20:05:37.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:36 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3172282555' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-05-20T20:05:37.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.057+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f4da80aa410 con 0x7f4db81028e0 2024-05-20T20:05:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:36 smithi154 ceph-mon[36314]: mgrmap e21: x(active, since 12m), standbys: z, y 2024-05-20T20:05:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:36 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3172282555' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-05-20T20:05:37.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:36 smithi033 ceph-mon[36179]: mgrmap e21: x(active, since 12m), standbys: z, y 2024-05-20T20:05:37.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:36 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3172282555' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-05-20T20:05:37.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.253+0000 7f4db7400640 1 -- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4d9c0b28b0 msgr2=0x7f4d9c0b4d70 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.28:6800/2019900521 2024-05-20T20:05:37.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.253+0000 7f4db7400640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4d9c0b28b0 0x7f4d9c0b4d70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-05-20T20:05:37.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.298+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 --> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}) -- 0x7f4db8062f30 con 0x7f4d9c0b28b0 2024-05-20T20:05:37.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.277+0000 7f7df29cb200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-05-20T20:05:37.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.611+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mgrmap(e 22) ==== 149337+0+0 (secure 0 0 0) 0x7f4da80a5fa0 con 0x7f4db81028e0 2024-05-20T20:05:37.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.611+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4d9c0b28b0 msgr2=0x7f4d9c0b4d70 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:05:37.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:37.611+0000 7f4db4c00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2019900521,v1:172.21.15.28:6801/2019900521] conn(0x7f4d9c0b28b0 0x7f4d9c0b4d70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:37.836 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.583+0000 7f7df29cb200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3172282555' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: Active manager daemon x restarted 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: Activating manager daemon x 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: osdmap e81: 12 total, 12 up, 12 in 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: mgrmap e22: x(active, starting, since 0.0233675s), standbys: z, y 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T20:05:38.047 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:05:38.048 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T20:05:38.049 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: Manager daemon x is now available 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T20:05:38.050 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:37 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3172282555' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: Active manager daemon x restarted 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: Activating manager daemon x 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: osdmap e81: 12 total, 12 up, 12 in 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: mgrmap e22: x(active, starting, since 0.0233675s), standbys: z, y 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T20:05:38.051 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:05:38.052 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T20:05:38.053 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:05:38.054 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T20:05:38.054 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T20:05:38.054 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T20:05:38.054 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: Manager daemon x is now available 2024-05-20T20:05:38.054 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:38.054 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T20:05:38.054 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:37 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T20:05:38.133 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d42a00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.133 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d42a00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d42a00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d42a00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d42a00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d45c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d45c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d45c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d45c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.134 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:37.932+0000 7f7d45c00700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3172282555' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: Active manager daemon x restarted 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: Activating manager daemon x 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: osdmap e81: 12 total, 12 up, 12 in 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: mgrmap e22: x(active, starting, since 0.0233675s), standbys: z, y 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-05-20T20:05:38.135 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-05-20T20:05:38.136 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T20:05:38.137 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T20:05:38.138 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T20:05:38.138 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: Manager daemon x is now available 2024-05-20T20:05:38.138 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:38.138 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-05-20T20:05:38.138 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:37 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-05-20T20:05:38.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.624+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mon.2 v2:172.21.15.33:3300/0 8 ==== mgrmap(e 23) ==== 149464+0+0 (secure 0 0 0) 0x7f4da80e0270 con 0x7f4db81028e0 2024-05-20T20:05:38.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.625+0000 7f4db4c00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d9c074d60 0x7f4d9c075140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:38.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.625+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}) -- 0x7f4d9c042310 con 0x7f4d9c074d60 2024-05-20T20:05:38.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.630+0000 7f4db7400640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d9c074d60 0x7f4d9c075140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:38.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.640+0000 7f4db7400640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d9c074d60 0x7f4d9c075140 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f4db819a010 tx=0x7f4da4005a20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:38.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.661+0000 7f4db4c00640 1 -- 172.21.15.28:0/339814748 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+3 (secure 0 0 0) 0x7f4d9c042310 con 0x7f4d9c074d60 2024-05-20T20:05:38.662 INFO:teuthology.orchestra.run.smithi028.stdout:{} 2024-05-20T20:05:38.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.664+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d9c074d60 msgr2=0x7f4d9c075140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.664+0000 7f4dbda00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d9c074d60 0x7f4d9c075140 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f4db819a010 tx=0x7f4da4005a20 comp rx=0 tx=0).stop 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.664+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db81028e0 msgr2=0x7f4db8198b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.664+0000 7f4dbda00640 1 --2- 172.21.15.28:0/339814748 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4db81028e0 0x7f4db8198b00 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4da800e9c0 tx=0x7f4da800ee80 comp rx=0 tx=0).stop 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.664+0000 7f4dbd000640 1 -- 172.21.15.28:0/339814748 reap_dead start 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.664+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 shutdown_connections 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.664+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 >> 172.21.15.28:0/339814748 conn(0x7f4db80fde50 msgr2=0x7f4db8106ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.665+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 shutdown_connections 2024-05-20T20:05:38.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.665+0000 7f4dbda00640 1 -- 172.21.15.28:0/339814748 wait complete. 2024-05-20T20:05:38.690 INFO:teuthology.run_tasks:Running task check-counter... 2024-05-20T20:05:38.701 INFO:teuthology.run_tasks:Running task fwd_scrub... 2024-05-20T20:05:38.721 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-05-20T20:05:38.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 -- 172.21.15.28:0/96710267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 msgr2=0x7fcf50102d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:38.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 --2- 172.21.15.28:0/96710267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf50102d10 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fcf44009900 tx=0x7fcf4402a410 comp rx=0 tx=0).stop 2024-05-20T20:05:38.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 -- 172.21.15.28:0/96710267 shutdown_connections 2024-05-20T20:05:38.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 --2- 172.21.15.28:0/96710267 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf501044a0 0x7fcf5010ad30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:38.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 --2- 172.21.15.28:0/96710267 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf50103ae0 0x7fcf50103f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:38.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 --2- 172.21.15.28:0/96710267 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf50102d10 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:38.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 -- 172.21.15.28:0/96710267 >> 172.21.15.28:0/96710267 conn(0x7fcf500fde60 msgr2=0x7fcf501002a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:38.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 -- 172.21.15.28:0/96710267 shutdown_connections 2024-05-20T20:05:38.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.898+0000 7fcf57000640 1 -- 172.21.15.28:0/96710267 wait complete. 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.899+0000 7fcf57000640 1 Processor -- start 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.899+0000 7fcf57000640 1 -- start start 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf57000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf5019ad40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf57000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf50103ae0 0x7fcf5019b280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf57000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf501044a0 0x7fcf501a22b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf57000640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fcf5010e460 con 0x7fcf501028f0 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf57000640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fcf5010e2e0 con 0x7fcf501044a0 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf57000640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fcf5010e5e0 con 0x7fcf50103ae0 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf5019ad40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf5019ad40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57468/0 (socket says 172.21.15.28:57468) 2024-05-20T20:05:38.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 -- 172.21.15.28:0/969182784 learned_addr learned my addr 172.21.15.28:0/969182784 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55200640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf50103ae0 0x7fcf5019b280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf56600640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf501044a0 0x7fcf501a22b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 -- 172.21.15.28:0/969182784 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf50103ae0 msgr2=0x7fcf5019b280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf50103ae0 0x7fcf5019b280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 -- 172.21.15.28:0/969182784 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf501044a0 msgr2=0x7fcf501a22b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf501044a0 0x7fcf501a22b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.900+0000 7fcf55c00640 1 -- 172.21.15.28:0/969182784 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcf501a2950 con 0x7fcf501028f0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.901+0000 7fcf55c00640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf5019ad40 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7fcf44031500 tx=0x7fcf44003c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.901+0000 7fcf4e000640 1 -- 172.21.15.28:0/969182784 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf4403f070 con 0x7fcf501028f0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.901+0000 7fcf4e000640 1 -- 172.21.15.28:0/969182784 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcf44031c50 con 0x7fcf501028f0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.901+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcf501a2be0 con 0x7fcf501028f0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.901+0000 7fcf4e000640 1 -- 172.21.15.28:0/969182784 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcf44035030 con 0x7fcf501028f0 2024-05-20T20:05:38.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.902+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcf501a3120 con 0x7fcf501028f0 2024-05-20T20:05:38.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.903+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcf501a3350 con 0x7fcf501028f0 2024-05-20T20:05:38.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.905+0000 7fcf4e000640 1 -- 172.21.15.28:0/969182784 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 23) ==== 149464+0+0 (secure 0 0 0) 0x7fcf44042070 con 0x7fcf501028f0 2024-05-20T20:05:38.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.907+0000 7fcf4e000640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcf380b2660 0x7fcf380b4b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:38.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.907+0000 7fcf4e000640 1 -- 172.21.15.28:0/969182784 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fcf44102d30 con 0x7fcf501028f0 2024-05-20T20:05:38.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.907+0000 7fcf55200640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcf380b2660 0x7fcf380b4b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:38.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.908+0000 7fcf55200640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcf380b2660 0x7fcf380b4b20 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fcf3c005fd0 tx=0x7fcf3c00b390 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:38.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:38.914+0000 7fcf4e000640 1 -- 172.21.15.28:0/969182784 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fcf440ca5a0 con 0x7fcf501028f0 2024-05-20T20:05:39.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:38.648+0000 7f7d40200700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:39.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:38.648+0000 7f7d40200700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:39.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:38.648+0000 7f7d40200700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:39.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:38.648+0000 7f7d40200700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:39.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:05:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:05:38.648+0000 7f7d40200700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:05:39.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.143+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fcf501a3520 con 0x7fcf501028f0 2024-05-20T20:05:39.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.145+0000 7fcf4e000640 1 -- 172.21.15.28:0/969182784 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) ==== 74+0+22947 (secure 0 0 0) 0x7fcf440ce8a0 con 0x7fcf501028f0 2024-05-20T20:05:39.146 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:39.146 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":81,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:05:37.589166+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6800/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/2759010565":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/1496067961":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1975849601":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/19609679":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcf380b2660 msgr2=0x7fcf380b4b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcf380b2660 0x7fcf380b4b20 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fcf3c005fd0 tx=0x7fcf3c00b390 comp rx=0 tx=0).stop 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 msgr2=0x7fcf5019ad40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf5019ad40 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7fcf44031500 tx=0x7fcf44003c10 comp rx=0 tx=0).stop 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 shutdown_connections 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcf501044a0 0x7fcf501a22b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcf380b2660 0x7fcf380b4b20 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcf50103ae0 0x7fcf5019b280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 --2- 172.21.15.28:0/969182784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcf501028f0 0x7fcf5019ad40 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 >> 172.21.15.28:0/969182784 conn(0x7fcf500fde60 msgr2=0x7fcf50109280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:39.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 shutdown_connections 2024-05-20T20:05:39.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.147+0000 7fcf57000640 1 -- 172.21.15.28:0/969182784 wait complete. 2024-05-20T20:05:39.169 INFO:tasks.fwd_scrub.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-20T19:55:47.376785+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-05-20T20:01:17.448722+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '78', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-05-20T20:01:22.420379+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '78', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-05-20T20:05:39.169 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-05-20T20:05:39.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 -- 172.21.15.28:0/411687903 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 msgr2=0x7fb9fc10cd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 --2- 172.21.15.28:0/411687903 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc10cd30 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7fb9f4009ca0 tx=0x7fb9f402a8c0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 -- 172.21.15.28:0/411687903 shutdown_connections 2024-05-20T20:05:39.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 --2- 172.21.15.28:0/411687903 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9fc0754b0 0x7fb9fc075930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 --2- 172.21.15.28:0/411687903 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc10cd30 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 --2- 172.21.15.28:0/411687903 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9fc076a90 0x7fb9fc074ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 -- 172.21.15.28:0/411687903 >> 172.21.15.28:0/411687903 conn(0x7fb9fc0fdf50 msgr2=0x7fb9fc100390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:39.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 -- 172.21.15.28:0/411687903 shutdown_connections 2024-05-20T20:05:39.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.349+0000 7fba03000640 1 -- 172.21.15.28:0/411687903 wait complete. 2024-05-20T20:05:39.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 Processor -- start 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 -- start start 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9fc0754b0 0x7fb9fc19ad40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9fc076a90 0x7fb9fc19b280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc1a22b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb9fc1106d0 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb9fc110550 con 0x7fb9fc0754b0 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.350+0000 7fba03000640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb9fc110850 con 0x7fb9fc076a90 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc1a22b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc1a22b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57478/0 (socket says 172.21.15.28:57478) 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 -- 172.21.15.28:0/2172476231 learned_addr learned my addr 172.21.15.28:0/2172476231 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba01c00640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9fc0754b0 0x7fb9fc19ad40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba01200640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9fc076a90 0x7fb9fc19b280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 -- 172.21.15.28:0/2172476231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9fc076a90 msgr2=0x7fb9fc19b280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9fc076a90 0x7fb9fc19b280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 -- 172.21.15.28:0/2172476231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9fc0754b0 msgr2=0x7fb9fc19ad40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9fc0754b0 0x7fb9fc19ad40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba02600640 1 -- 172.21.15.28:0/2172476231 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb9fc1a2950 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba01200640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9fc076a90 0x7fb9fc19b280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:39.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.351+0000 7fba01c00640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9fc0754b0 0x7fb9fc19ad40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:39.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.352+0000 7fba02600640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc1a22b0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fb9f4033040 tx=0x7fb9f40316f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:39.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.352+0000 7fb9fa000640 1 -- 172.21.15.28:0/2172476231 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9f4003b70 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.352+0000 7fb9fa000640 1 -- 172.21.15.28:0/2172476231 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb9f4003d10 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.352+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb9fc1a2be0 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.352+0000 7fb9fa000640 1 -- 172.21.15.28:0/2172476231 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9f402fb40 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.352+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb9fc1a3180 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.353+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb9fc1095d0 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.355+0000 7fb9fa000640 1 -- 172.21.15.28:0/2172476231 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 23) ==== 149464+0+0 (secure 0 0 0) 0x7fb9f4004610 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.357+0000 7fb9fa000640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9e40b2660 0x7fb9e40b4b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.357+0000 7fb9fa000640 1 -- 172.21.15.28:0/2172476231 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb9f4103360 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.357+0000 7fba01c00640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9e40b2660 0x7fb9e40b4b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.358+0000 7fba01c00640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9e40b2660 0x7fb9e40b4b20 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb9fc19c210 tx=0x7fb9f0005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:39.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.362+0000 7fb9fa000640 1 -- 172.21.15.28:0/2172476231 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb9f40cb220 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.603+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7fb9fc063650 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.603+0000 7fb9fa000640 1 -- 172.21.15.28:0/2172476231 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) ==== 93+0+10 (secure 0 0 0) 0x7fb9f40cab20 con 0x7fb9fc10a8a0 2024-05-20T20:05:39.604 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 1 2024-05-20T20:05:39.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9e40b2660 msgr2=0x7fb9e40b4b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9e40b2660 0x7fb9e40b4b20 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb9fc19c210 tx=0x7fb9f0005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 msgr2=0x7fb9fc1a22b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc1a22b0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fb9f4033040 tx=0x7fb9f40316f0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 shutdown_connections 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9e40b2660 0x7fb9e40b4b20 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9fc10a8a0 0x7fb9fc1a22b0 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9fc076a90 0x7fb9fc19b280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 --2- 172.21.15.28:0/2172476231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9fc0754b0 0x7fb9fc19ad40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.607+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 >> 172.21.15.28:0/2172476231 conn(0x7fb9fc0fdf50 msgr2=0x7fb9fc1067a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.608+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 shutdown_connections 2024-05-20T20:05:39.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.608+0000 7fba03000640 1 -- 172.21.15.28:0/2172476231 wait complete. 2024-05-20T20:05:39.637 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-05-20T20:05:39.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:39 smithi028 ceph-mon[31178]: mgrmap e23: x(active, since 1.04006s), standbys: z, y 2024-05-20T20:05:39.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:39 smithi028 ceph-mon[31178]: from='client.25154 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:39.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:39 smithi028 ceph-mon[31178]: pgmap v3: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:39.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:39 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/969182784' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:39.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:39 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2172476231' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:05:39.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 -- 172.21.15.28:0/3097226847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 msgr2=0x7f3a981113a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 --2- 172.21.15.28:0/3097226847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981113a0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f3a90009c10 tx=0x7f3a9002a6e0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 -- 172.21.15.28:0/3097226847 shutdown_connections 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 --2- 172.21.15.28:0/3097226847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981113a0 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 --2- 172.21.15.28:0/3097226847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a980754b0 0x7f3a98075930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 --2- 172.21.15.28:0/3097226847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a98076a90 0x7f3a98074ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 -- 172.21.15.28:0/3097226847 >> 172.21.15.28:0/3097226847 conn(0x7f3a980fdf50 msgr2=0x7f3a98100390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 -- 172.21.15.28:0/3097226847 shutdown_connections 2024-05-20T20:05:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.825+0000 7f3a97200640 1 -- 172.21.15.28:0/3097226847 wait complete. 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 Processor -- start 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 -- start start 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a980754b0 0x7f3a9819f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a98076a90 0x7f3a9819f6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981a66d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3a98114a70 con 0x7f3a98076a90 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3a981148f0 con 0x7f3a9810efb0 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a97200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3a98114bf0 con 0x7f3a980754b0 2024-05-20T20:05:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a8fe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a98076a90 0x7f3a9819f6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a96800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981a66d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a95e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a980754b0 0x7f3a9819f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a96800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981a66d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37882/0 (socket says 172.21.15.28:37882) 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.827+0000 7f3a96800640 1 -- 172.21.15.28:0/2420925412 learned_addr learned my addr 172.21.15.28:0/2420925412 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a96800640 1 -- 172.21.15.28:0/2420925412 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a980754b0 msgr2=0x7f3a9819f170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a96800640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a980754b0 0x7f3a9819f170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a96800640 1 -- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a98076a90 msgr2=0x7f3a9819f6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a96800640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a98076a90 0x7f3a9819f6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a96800640 1 -- 172.21.15.28:0/2420925412 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3a981a6d40 con 0x7f3a9810efb0 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a8fe00640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a98076a90 0x7f3a9819f6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a96800640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981a66d0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f3a90004af0 tx=0x7f3a90030d50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a8ea00640 1 -- 172.21.15.28:0/2420925412 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a90041020 con 0x7f3a9810efb0 2024-05-20T20:05:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a8ea00640 1 -- 172.21.15.28:0/2420925412 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3a90030ec0 con 0x7f3a9810efb0 2024-05-20T20:05:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.828+0000 7f3a8ea00640 1 -- 172.21.15.28:0/2420925412 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a90036040 con 0x7f3a9810efb0 2024-05-20T20:05:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.829+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3a981a6fd0 con 0x7f3a9810efb0 2024-05-20T20:05:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.829+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3a981a73e0 con 0x7f3a9810efb0 2024-05-20T20:05:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.830+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3a98105180 con 0x7f3a9810efb0 2024-05-20T20:05:39.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.832+0000 7f3a8ea00640 1 -- 172.21.15.28:0/2420925412 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 24) ==== 149823+0+0 (secure 0 0 0) 0x7f3a90044020 con 0x7f3a9810efb0 2024-05-20T20:05:39.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.833+0000 7f3a8ea00640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a7c0b29f0 0x7f3a7c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:39.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.834+0000 7f3a8ea00640 1 -- 172.21.15.28:0/2420925412 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f3a90103530 con 0x7f3a9810efb0 2024-05-20T20:05:39.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.834+0000 7f3a95e00640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a7c0b29f0 0x7f3a7c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:39.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.834+0000 7f3a95e00640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a7c0b29f0 0x7f3a7c0b4eb0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f3a880095f0 tx=0x7f3a88005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:39.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:39.838+0000 7f3a8ea00640 1 -- 172.21.15.28:0/2420925412 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f3a9004b020 con 0x7f3a9810efb0 2024-05-20T20:05:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:39 smithi154 ceph-mon[36314]: mgrmap e23: x(active, since 1.04006s), standbys: z, y 2024-05-20T20:05:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:39 smithi154 ceph-mon[36314]: from='client.25154 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:39 smithi154 ceph-mon[36314]: pgmap v3: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:39 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/969182784' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:39 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2172476231' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:05:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:39 smithi033 ceph-mon[36179]: mgrmap e23: x(active, since 1.04006s), standbys: z, y 2024-05-20T20:05:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:39 smithi033 ceph-mon[36179]: from='client.25154 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:39 smithi033 ceph-mon[36179]: pgmap v3: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:39 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/969182784' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:39 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2172476231' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:05:40.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.074+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f3a9802ddc0 con 0x7f3a9810efb0 2024-05-20T20:05:40.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.075+0000 7f3a8ea00640 1 -- 172.21.15.28:0/2420925412 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v81) ==== 104+0+11 (secure 0 0 0) 0x7f3a900cad50 con 0x7f3a9810efb0 2024-05-20T20:05:40.076 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 32 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.077+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a7c0b29f0 msgr2=0x7f3a7c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.077+0000 7f3a97200640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a7c0b29f0 0x7f3a7c0b4eb0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f3a880095f0 tx=0x7f3a88005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.077+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 msgr2=0x7f3a981a66d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.077+0000 7f3a97200640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981a66d0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f3a90004af0 tx=0x7f3a90030d50 comp rx=0 tx=0).stop 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 shutdown_connections 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a9810efb0 0x7f3a981a66d0 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a7c0b29f0 0x7f3a7c0b4eb0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a98076a90 0x7f3a9819f6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 --2- 172.21.15.28:0/2420925412 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a980754b0 0x7f3a9819f170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 >> 172.21.15.28:0/2420925412 conn(0x7f3a980fdf50 msgr2=0x7f3a98102350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 shutdown_connections 2024-05-20T20:05:40.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.078+0000 7f3a97200640 1 -- 172.21.15.28:0/2420925412 wait complete. 2024-05-20T20:05:40.097 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-05-20T20:05:40.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 -- 172.21.15.28:0/4193652278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 msgr2=0x7f25b41082f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:40.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 --2- 172.21.15.28:0/4193652278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 0x7f25b41082f0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f25a0009900 tx=0x7f25a002a3f0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 -- 172.21.15.28:0/4193652278 shutdown_connections 2024-05-20T20:05:40.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 --2- 172.21.15.28:0/4193652278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b4108830 0x7f25b410ac20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 --2- 172.21.15.28:0/4193652278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 0x7f25b41082f0 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 --2- 172.21.15.28:0/4193652278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f25b41035d0 0x7f25b41059c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 -- 172.21.15.28:0/4193652278 >> 172.21.15.28:0/4193652278 conn(0x7f25b40fcf40 msgr2=0x7f25b40ff360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:40.288 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 -- 172.21.15.28:0/4193652278 shutdown_connections 2024-05-20T20:05:40.288 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.285+0000 7f25ba800640 1 -- 172.21.15.28:0/4193652278 wait complete. 2024-05-20T20:05:40.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.286+0000 7f25ba800640 1 Processor -- start 2024-05-20T20:05:40.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.286+0000 7f25ba800640 1 -- start start 2024-05-20T20:05:40.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25ba800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b41035d0 0x7f25b419af20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:40.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25ba800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 0x7f25b419b460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:40.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25ba800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f25b4108830 0x7f25b41a24d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:40.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25ba800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f25b410e630 con 0x7f25b4108830 2024-05-20T20:05:40.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25ba800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f25b410e4b0 con 0x7f25b41035d0 2024-05-20T20:05:40.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25ba800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f25b410e7b0 con 0x7f25b4105f00 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b41035d0 0x7f25b419af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b41035d0 0x7f25b419af20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37906/0 (socket says 172.21.15.28:37906) 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3400640 1 -- 172.21.15.28:0/768896262 learned_addr learned my addr 172.21.15.28:0/768896262 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b2a00640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 0x7f25b419b460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3e00640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f25b4108830 0x7f25b41a24d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3400640 1 -- 172.21.15.28:0/768896262 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 msgr2=0x7f25b419b460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3400640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 0x7f25b419b460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3400640 1 -- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f25b4108830 msgr2=0x7f25b41a24d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:40.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.287+0000 7f25b3400640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f25b4108830 0x7f25b41a24d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.288+0000 7f25b3400640 1 -- 172.21.15.28:0/768896262 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f25b41a2b40 con 0x7f25b41035d0 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.288+0000 7f25b3400640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b41035d0 0x7f25b419af20 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f25a400d950 tx=0x7f25a400de10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.288+0000 7f25abe00640 1 -- 172.21.15.28:0/768896262 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f25a4017030 con 0x7f25b41035d0 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.288+0000 7f25abe00640 1 -- 172.21.15.28:0/768896262 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f25a4004020 con 0x7f25b41035d0 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.288+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f25b41a2e30 con 0x7f25b41035d0 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.288+0000 7f25abe00640 1 -- 172.21.15.28:0/768896262 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f25a40047a0 con 0x7f25b41035d0 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.288+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f25b41a3290 con 0x7f25b41035d0 2024-05-20T20:05:40.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.289+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f25b410a320 con 0x7f25b41035d0 2024-05-20T20:05:40.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.293+0000 7f25abe00640 1 -- 172.21.15.28:0/768896262 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 24) ==== 149823+0+0 (secure 0 0 0) 0x7f25a4009a30 con 0x7f25b41035d0 2024-05-20T20:05:40.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.295+0000 7f25abe00640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f25980b29f0 0x7f25980b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:40.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.295+0000 7f25abe00640 1 -- 172.21.15.28:0/768896262 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f25a40e25d0 con 0x7f25b41035d0 2024-05-20T20:05:40.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.295+0000 7f25b2a00640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f25980b29f0 0x7f25980b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:40.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.296+0000 7f25abe00640 1 -- 172.21.15.28:0/768896262 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f25a40a9df0 con 0x7f25b41035d0 2024-05-20T20:05:40.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.296+0000 7f25b2a00640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f25980b29f0 0x7f25980b4eb0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f25a0005ab0 tx=0x7f25a0005a20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:40.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.545+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f25b4063680 con 0x7f25b41035d0 2024-05-20T20:05:40.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.545+0000 7f25abe00640 1 -- 172.21.15.28:0/768896262 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v81) ==== 100+0+11 (secure 0 0 0) 0x7f25a40b0e20 con 0x7f25b41035d0 2024-05-20T20:05:40.547 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 64 2024-05-20T20:05:40.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f25980b29f0 msgr2=0x7f25980b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:40.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f25980b29f0 0x7f25980b4eb0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f25a0005ab0 tx=0x7f25a0005a20 comp rx=0 tx=0).stop 2024-05-20T20:05:40.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b41035d0 msgr2=0x7f25b419af20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:40.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b41035d0 0x7f25b419af20 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f25a400d950 tx=0x7f25a400de10 comp rx=0 tx=0).stop 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 shutdown_connections 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f25980b29f0 0x7f25980b4eb0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f25b4108830 0x7f25b41a24d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f25b4105f00 0x7f25b419b460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 --2- 172.21.15.28:0/768896262 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f25b41035d0 0x7f25b419af20 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 >> 172.21.15.28:0/768896262 conn(0x7f25b40fcf40 msgr2=0x7f25b4102cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 shutdown_connections 2024-05-20T20:05:40.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:40.548+0000 7f25ba800640 1 -- 172.21.15.28:0/768896262 wait complete. 2024-05-20T20:05:40.566 INFO:tasks.fwd_scrub:Wait for all MDSs to reach steady state... 2024-05-20T20:05:40.566 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:05:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: [20/May/2024:20:05:39] ENGINE Bus STARTING 2024-05-20T20:05:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: pgmap v4: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:40.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: mgrmap e24: x(active, since 2s), standbys: z, y 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: [20/May/2024:20:05:39] ENGINE Serving on https://172.21.15.28:7150 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: [20/May/2024:20:05:39] ENGINE Serving on http://172.21.15.28:8765 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: [20/May/2024:20:05:39] ENGINE Bus STARTED 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2420925412' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-05-20T20:05:40.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:40 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/768896262' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-05-20T20:05:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: [20/May/2024:20:05:39] ENGINE Bus STARTING 2024-05-20T20:05:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: pgmap v4: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: mgrmap e24: x(active, since 2s), standbys: z, y 2024-05-20T20:05:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: [20/May/2024:20:05:39] ENGINE Serving on https://172.21.15.28:7150 2024-05-20T20:05:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: [20/May/2024:20:05:39] ENGINE Serving on http://172.21.15.28:8765 2024-05-20T20:05:40.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: [20/May/2024:20:05:39] ENGINE Bus STARTED 2024-05-20T20:05:40.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:40.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2420925412' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-05-20T20:05:40.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:40 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/768896262' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-05-20T20:05:40.985 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:41.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: [20/May/2024:20:05:39] ENGINE Bus STARTING 2024-05-20T20:05:41.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: pgmap v4: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:41.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: mgrmap e24: x(active, since 2s), standbys: z, y 2024-05-20T20:05:41.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: [20/May/2024:20:05:39] ENGINE Serving on https://172.21.15.28:7150 2024-05-20T20:05:41.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:41.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: [20/May/2024:20:05:39] ENGINE Serving on http://172.21.15.28:8765 2024-05-20T20:05:41.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: [20/May/2024:20:05:39] ENGINE Bus STARTED 2024-05-20T20:05:41.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:41.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:41.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:41.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2420925412' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-05-20T20:05:41.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:40 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/768896262' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-05-20T20:05:42.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi154 to 176.2M 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi033 to 177.2M 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: Unable to set osd_memory_target on smithi154 to 184788172: error parsing value: Value '184788172' is below minimum 939524096 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: Unable to set osd_memory_target on smithi033 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-05-20T20:05:42.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:42 smithi028 ceph-mon[31178]: pgmap v5: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:42.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi154 to 176.2M 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi033 to 177.2M 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: Unable to set osd_memory_target on smithi154 to 184788172: error parsing value: Value '184788172' is below minimum 939524096 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: Unable to set osd_memory_target on smithi033 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:42 smithi154 ceph-mon[36314]: pgmap v5: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi154 to 176.2M 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi033 to 177.2M 2024-05-20T20:05:42.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: Unable to set osd_memory_target on smithi154 to 184788172: error parsing value: Value '184788172' is below minimum 939524096 2024-05-20T20:05:42.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: Unable to set osd_memory_target on smithi033 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-05-20T20:05:42.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:42 smithi033 ceph-mon[36179]: pgmap v5: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:05:43.141 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:43 smithi028 ceph-mon[31178]: mgrmap e25: x(active, since 4s), standbys: z, y 2024-05-20T20:05:43.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:43 smithi154 ceph-mon[36314]: mgrmap e25: x(active, since 4s), standbys: z, y 2024-05-20T20:05:43.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:43 smithi033 ceph-mon[36179]: mgrmap e25: x(active, since 4s), standbys: z, y 2024-05-20T20:05:43.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.432+0000 7f90cac00700 1 -- 172.21.15.28:0/3614680467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c41095a0 msgr2=0x7f90c4109980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.432+0000 7f90cac00700 1 --2- 172.21.15.28:0/3614680467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c41095a0 0x7f90c4109980 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f90b4009990 tx=0x7f90b4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.432+0000 7f90cac00700 1 -- 172.21.15.28:0/3614680467 shutdown_connections 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.432+0000 7f90cac00700 1 --2- 172.21.15.28:0/3614680467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406c470 0x7f90c41103c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.432+0000 7f90cac00700 1 --2- 172.21.15.28:0/3614680467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c41095a0 0x7f90c4109980 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.432+0000 7f90cac00700 1 --2- 172.21.15.28:0/3614680467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90c406bab0 0x7f90c406bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.432+0000 7f90cac00700 1 -- 172.21.15.28:0/3614680467 >> 172.21.15.28:0/3614680467 conn(0x7f90c40793b0 msgr2=0x7f90c40797c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.433+0000 7f90cac00700 1 -- 172.21.15.28:0/3614680467 shutdown_connections 2024-05-20T20:05:43.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.433+0000 7f90cac00700 1 -- 172.21.15.28:0/3614680467 wait complete. 2024-05-20T20:05:43.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.434+0000 7f90cac00700 1 Processor -- start 2024-05-20T20:05:43.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.434+0000 7f90cac00700 1 -- start start 2024-05-20T20:05:43.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90cac00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406bab0 0x7f90c4102500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:43.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90cac00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c406c470 0x7f90c4102a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:43.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90cac00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90c41095a0 0x7f90c4102f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:43.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90cac00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f90c4113730 con 0x7f90c406c470 2024-05-20T20:05:43.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90cac00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f90c41135b0 con 0x7f90c406bab0 2024-05-20T20:05:43.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90cac00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f90c41138b0 con 0x7f90c41095a0 2024-05-20T20:05:43.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90c3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406bab0 0x7f90c4102500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:43.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90c3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406bab0 0x7f90c4102500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37928/0 (socket says 172.21.15.28:37928) 2024-05-20T20:05:43.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90c3e00700 1 -- 172.21.15.28:0/859531976 learned_addr learned my addr 172.21.15.28:0/859531976 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:43.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90c8c00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90c41095a0 0x7f90c4102f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:43.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.435+0000 7f90c3400700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c406c470 0x7f90c4102a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:43.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.436+0000 7f90c3e00700 1 -- 172.21.15.28:0/859531976 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90c41095a0 msgr2=0x7f90c4102f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:43.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.436+0000 7f90c3e00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90c41095a0 0x7f90c4102f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.436+0000 7f90c3e00700 1 -- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c406c470 msgr2=0x7f90c4102a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:43.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.436+0000 7f90c3e00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c406c470 0x7f90c4102a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.436+0000 7f90c3e00700 1 -- 172.21.15.28:0/859531976 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f90c41a81d0 con 0x7f90c406bab0 2024-05-20T20:05:43.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.436+0000 7f90c3e00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406bab0 0x7f90c4102500 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f90b4005910 tx=0x7f90b400b670 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:43.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.436+0000 7f90c0c00700 1 -- 172.21.15.28:0/859531976 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f90b4027070 con 0x7f90c406bab0 2024-05-20T20:05:43.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.437+0000 7f90c0c00700 1 -- 172.21.15.28:0/859531976 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f90b4022470 con 0x7f90c406bab0 2024-05-20T20:05:43.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.437+0000 7f90c0c00700 1 -- 172.21.15.28:0/859531976 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f90b4017630 con 0x7f90c406bab0 2024-05-20T20:05:43.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.437+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f90c41a8460 con 0x7f90c406bab0 2024-05-20T20:05:43.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.437+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f90c41a8860 con 0x7f90c406bab0 2024-05-20T20:05:43.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.438+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f90c410ba80 con 0x7f90c406bab0 2024-05-20T20:05:43.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.440+0000 7f90c0c00700 1 -- 172.21.15.28:0/859531976 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f90b402a070 con 0x7f90c406bab0 2024-05-20T20:05:43.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.442+0000 7f90c0c00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90a80b2b90 0x7f90a80b5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:43.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.442+0000 7f90c0c00700 1 -- 172.21.15.28:0/859531976 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f90b40e34c0 con 0x7f90c406bab0 2024-05-20T20:05:43.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.442+0000 7f90c3400700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90a80b2b90 0x7f90a80b5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:43.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.443+0000 7f90c3400700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90a80b2b90 0x7f90a80b5050 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f90c4079a90 tx=0x7f90b0009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:43.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.444+0000 7f90c0c00700 1 -- 172.21.15.28:0/859531976 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f90b4013070 con 0x7f90c406bab0 2024-05-20T20:05:43.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.793+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f90c402e560 con 0x7f90c406bab0 2024-05-20T20:05:43.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.794+0000 7f90c0c00700 1 -- 172.21.15.28:0/859531976 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f90b40aac90 con 0x7f90c406bab0 2024-05-20T20:05:43.795 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:43.795 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:05:43.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90a80b2b90 msgr2=0x7f90a80b5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:43.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90a80b2b90 0x7f90a80b5050 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f90c4079a90 tx=0x7f90b0009260 comp rx=0 tx=0).stop 2024-05-20T20:05:43.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406bab0 msgr2=0x7f90c4102500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:43.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406bab0 0x7f90c4102500 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f90b4005910 tx=0x7f90b400b670 comp rx=0 tx=0).stop 2024-05-20T20:05:43.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 shutdown_connections 2024-05-20T20:05:43.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90c41095a0 0x7f90c4102f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90a80b2b90 0x7f90a80b5050 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90c406bab0 0x7f90c4102500 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 --2- 172.21.15.28:0/859531976 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90c406c470 0x7f90c4102a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:43.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 >> 172.21.15.28:0/859531976 conn(0x7f90c40793b0 msgr2=0x7f90c4101290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:43.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 shutdown_connections 2024-05-20T20:05:43.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:43.797+0000 7f90cac00700 1 -- 172.21.15.28:0/859531976 wait complete. 2024-05-20T20:05:43.800 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:05:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi028 to 177.5M 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: Unable to set osd_memory_target on smithi028 to 186173030: error parsing value: Value '186173030' is below minimum 939524096 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: Updating smithi028:/etc/ceph/ceph.conf 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: pgmap v6: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 1.0 KiB/s wr, 7 op/s 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:44.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:44 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/859531976' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi028 to 177.5M 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: Unable to set osd_memory_target on smithi028 to 186173030: error parsing value: Value '186173030' is below minimum 939524096 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: Updating smithi028:/etc/ceph/ceph.conf 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: pgmap v6: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 1.0 KiB/s wr, 7 op/s 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:44.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:44 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/859531976' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi028 to 177.5M 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: Unable to set osd_memory_target on smithi028 to 186173030: error parsing value: Value '186173030' is below minimum 939524096 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:44.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: Updating smithi028:/etc/ceph/ceph.conf 2024-05-20T20:05:44.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: Updating smithi033:/etc/ceph/ceph.conf 2024-05-20T20:05:44.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: Updating smithi154:/etc/ceph/ceph.conf 2024-05-20T20:05:44.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: pgmap v6: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 1.0 KiB/s wr, 7 op/s 2024-05-20T20:05:44.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:44.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:44.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:44 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/859531976' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:44.694 INFO:tasks.fwd_scrub:Ready to start scrub thrashing 2024-05-20T20:05:44.694 INFO:tasks.fwd_scrub.ceph_manager:waiting for clean 2024-05-20T20:05:44.694 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-05-20T20:05:44.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 -- 172.21.15.28:0/1715022893 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 msgr2=0x7f339c106340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 --2- 172.21.15.28:0/1715022893 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c106340 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f3388009860 tx=0x7f338802a340 comp rx=0 tx=0).stop 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 -- 172.21.15.28:0/1715022893 shutdown_connections 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 --2- 172.21.15.28:0/1715022893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f339c106880 0x7f339c10d110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 --2- 172.21.15.28:0/1715022893 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c106340 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 --2- 172.21.15.28:0/1715022893 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f339c104cd0 0x7f339c1050f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 -- 172.21.15.28:0/1715022893 >> 172.21.15.28:0/1715022893 conn(0x7f339c100240 msgr2=0x7f339c102680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 -- 172.21.15.28:0/1715022893 shutdown_connections 2024-05-20T20:05:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.880+0000 7f33a1400640 1 -- 172.21.15.28:0/1715022893 wait complete. 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.881+0000 7f33a1400640 1 Processor -- start 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.881+0000 7f33a1400640 1 -- start start 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f33a1400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f339c104cd0 0x7f339c19af20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f33a1400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c19b460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f33a1400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f339c106880 0x7f339c1a24d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f33a1400640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f339c110800 con 0x7f339c104cd0 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f33a1400640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f339c110680 con 0x7f339c106880 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f33a1400640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f339c110980 con 0x7f339c105ec0 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f3399e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c19b460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f339a800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f339c104cd0 0x7f339c19af20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f3399e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c19b460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55608/0 (socket says 172.21.15.28:55608) 2024-05-20T20:05:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f3399e00640 1 -- 172.21.15.28:0/3944229740 learned_addr learned my addr 172.21.15.28:0/3944229740 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.882+0000 7f339b200640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f339c106880 0x7f339c1a24d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3399e00640 1 -- 172.21.15.28:0/3944229740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f339c106880 msgr2=0x7f339c1a24d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3399e00640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f339c106880 0x7f339c1a24d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3399e00640 1 -- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f339c104cd0 msgr2=0x7f339c19af20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3399e00640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f339c104cd0 0x7f339c19af20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3399e00640 1 -- 172.21.15.28:0/3944229740 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f339c1a2b40 con 0x7f339c105ec0 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3399e00640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c19b460 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f33880057c0 tx=0x7f3388032eb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3393400640 1 -- 172.21.15.28:0/3944229740 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3388037040 con 0x7f339c105ec0 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3393400640 1 -- 172.21.15.28:0/3944229740 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3388030cd0 con 0x7f339c105ec0 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f339c1a2dd0 con 0x7f339c105ec0 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.883+0000 7f3393400640 1 -- 172.21.15.28:0/3944229740 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3388005280 con 0x7f339c105ec0 2024-05-20T20:05:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.884+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f339c1a3220 con 0x7f339c105ec0 2024-05-20T20:05:44.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.885+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f339c10c110 con 0x7f339c105ec0 2024-05-20T20:05:44.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.889+0000 7f3393400640 1 -- 172.21.15.28:0/3944229740 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f3388043020 con 0x7f339c105ec0 2024-05-20T20:05:44.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.890+0000 7f3393400640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33800b2b00 0x7f33800b4fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:44.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.890+0000 7f3393400640 1 -- 172.21.15.28:0/3944229740 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f3388101e60 con 0x7f339c105ec0 2024-05-20T20:05:44.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.891+0000 7f339a800640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33800b2b00 0x7f33800b4fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:44.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.891+0000 7f339a800640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33800b2b00 0x7f33800b4fc0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f339c105d20 tx=0x7f338c009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:44.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:44.895+0000 7f3393400640 1 -- 172.21.15.28:0/3944229740 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f33880c96b0 con 0x7f339c105ec0 2024-05-20T20:05:45.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.111+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f339c062e90 con 0x7f33800b2b00 2024-05-20T20:05:45.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.121+0000 7f3393400640 1 -- 172.21.15.28:0/3944229740 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258090 (secure 0 0 0) 0x7f339c062e90 con 0x7f33800b2b00 2024-05-20T20:05:45.126 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:45.127 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:05:45.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33800b2b00 msgr2=0x7f33800b4fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33800b2b00 0x7f33800b4fc0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f339c105d20 tx=0x7f338c009260 comp rx=0 tx=0).stop 2024-05-20T20:05:45.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 msgr2=0x7f339c19b460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c19b460 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f33880057c0 tx=0x7f3388032eb0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 shutdown_connections 2024-05-20T20:05:45.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f339c106880 0x7f339c1a24d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33800b2b00 0x7f33800b4fc0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f339c105ec0 0x7f339c19b460 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 --2- 172.21.15.28:0/3944229740 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f339c104cd0 0x7f339c19af20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 >> 172.21.15.28:0/3944229740 conn(0x7f339c100240 msgr2=0x7f339c1092e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:45.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 shutdown_connections 2024-05-20T20:05:45.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.126+0000 7f33a1400640 1 -- 172.21.15.28:0/3944229740 wait complete. 2024-05-20T20:05:45.147 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":6,"stamp":"2024-05-20T20:05:43.627844+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1195147,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":2575,"num_read_kb":1437,"num_write":503,"num_write_kb":3622,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":898,"ondisk_log_size":898,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347748,"kb_used_data":11056,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124594076,"statfs":{"total":1151940427776,"available":1151584333824,"internally_reserved":0,"allocated":11321344,"data_stored":8741958,"data_compressed":279771,"data_compressed_allocated":368640,"data_compressed_original":3588096,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3302,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":33,"num_read_kb":20,"num_write":3,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.982270"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603392+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:05:37.603392+0000","last_peered":"2024-05-20T20:05:37.603392+0000","last_clean":"2024-05-20T20:05:37.603392+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:05:37.603392+0000","last_undegraded":"2024-05-20T20:05:37.603392+0000","last_fullsized":"2024-05-20T20:05:37.603392+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750621+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:05:37.750621+0000","last_peered":"2024-05-20T20:05:37.750621+0000","last_clean":"2024-05-20T20:05:37.750621+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:05:37.750621+0000","last_undegraded":"2024-05-20T20:05:37.750621+0000","last_fullsized":"2024-05-20T20:05:37.750621+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750419+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:05:37.750419+0000","last_peered":"2024-05-20T20:05:37.750419+0000","last_clean":"2024-05-20T20:05:37.750419+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:05:37.750419+0000","last_undegraded":"2024-05-20T20:05:37.750419+0000","last_fullsized":"2024-05-20T20:05:37.750419+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946345+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:05:37.946345+0000","last_peered":"2024-05-20T20:05:37.946345+0000","last_clean":"2024-05-20T20:05:37.946345+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:05:37.946345+0000","last_undegraded":"2024-05-20T20:05:37.946345+0000","last_fullsized":"2024-05-20T20:05:37.946345+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":27,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950341+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:05:37.950341+0000","last_peered":"2024-05-20T20:05:37.950341+0000","last_clean":"2024-05-20T20:05:37.950341+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:05:37.950341+0000","last_undegraded":"2024-05-20T20:05:37.950341+0000","last_fullsized":"2024-05-20T20:05:37.950341+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601481+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:05:37.601481+0000","last_peered":"2024-05-20T20:05:37.601481+0000","last_clean":"2024-05-20T20:05:37.601481+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:05:37.601481+0000","last_undegraded":"2024-05-20T20:05:37.601481+0000","last_fullsized":"2024-05-20T20:05:37.601481+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946714+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:05:37.946714+0000","last_peered":"2024-05-20T20:05:37.946714+0000","last_clean":"2024-05-20T20:05:37.946714+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:05:37.946714+0000","last_undegraded":"2024-05-20T20:05:37.946714+0000","last_fullsized":"2024-05-20T20:05:37.946714+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740698+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:05:37.740698+0000","last_peered":"2024-05-20T20:05:37.740698+0000","last_clean":"2024-05-20T20:05:37.740698+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:05:37.740698+0000","last_undegraded":"2024-05-20T20:05:37.740698+0000","last_fullsized":"2024-05-20T20:05:37.740698+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.942016+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:05:37.942016+0000","last_peered":"2024-05-20T20:05:37.942016+0000","last_clean":"2024-05-20T20:05:37.942016+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:05:37.942016+0000","last_undegraded":"2024-05-20T20:05:37.942016+0000","last_fullsized":"2024-05-20T20:05:37.942016+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":45,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601908+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:05:37.601908+0000","last_peered":"2024-05-20T20:05:37.601908+0000","last_clean":"2024-05-20T20:05:37.601908+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:05:37.601908+0000","last_undegraded":"2024-05-20T20:05:37.601908+0000","last_fullsized":"2024-05-20T20:05:37.601908+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947610+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:05:37.947610+0000","last_peered":"2024-05-20T20:05:37.947610+0000","last_clean":"2024-05-20T20:05:37.947610+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:05:37.947610+0000","last_undegraded":"2024-05-20T20:05:37.947610+0000","last_fullsized":"2024-05-20T20:05:37.947610+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946190+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:05:37.946190+0000","last_peered":"2024-05-20T20:05:37.946190+0000","last_clean":"2024-05-20T20:05:37.946190+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:05:37.946190+0000","last_undegraded":"2024-05-20T20:05:37.946190+0000","last_fullsized":"2024-05-20T20:05:37.946190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951933+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:05:37.951933+0000","last_peered":"2024-05-20T20:05:37.951933+0000","last_clean":"2024-05-20T20:05:37.951933+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:05:37.951933+0000","last_undegraded":"2024-05-20T20:05:37.951933+0000","last_fullsized":"2024-05-20T20:05:37.951933+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749940+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:05:37.749940+0000","last_peered":"2024-05-20T20:05:37.749940+0000","last_clean":"2024-05-20T20:05:37.749940+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:05:37.749940+0000","last_undegraded":"2024-05-20T20:05:37.749940+0000","last_fullsized":"2024-05-20T20:05:37.749940+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749763+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:05:37.749763+0000","last_peered":"2024-05-20T20:05:37.749763+0000","last_clean":"2024-05-20T20:05:37.749763+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:05:37.749763+0000","last_undegraded":"2024-05-20T20:05:37.749763+0000","last_fullsized":"2024-05-20T20:05:37.749763+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603526+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:05:37.603526+0000","last_peered":"2024-05-20T20:05:37.603526+0000","last_clean":"2024-05-20T20:05:37.603526+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:05:37.603526+0000","last_undegraded":"2024-05-20T20:05:37.603526+0000","last_fullsized":"2024-05-20T20:05:37.603526+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602253+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:05:37.602253+0000","last_peered":"2024-05-20T20:05:37.602253+0000","last_clean":"2024-05-20T20:05:37.602253+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:05:37.602253+0000","last_undegraded":"2024-05-20T20:05:37.602253+0000","last_fullsized":"2024-05-20T20:05:37.602253+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940646+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:05:37.940646+0000","last_peered":"2024-05-20T20:05:37.940646+0000","last_clean":"2024-05-20T20:05:37.940646+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:05:37.940646+0000","last_undegraded":"2024-05-20T20:05:37.940646+0000","last_fullsized":"2024-05-20T20:05:37.940646+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'14","reported_seq":277,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.219827+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:05:38.219827+0000","last_peered":"2024-05-20T20:05:38.219827+0000","last_clean":"2024-05-20T20:05:38.219827+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:05:38.219827+0000","last_undegraded":"2024-05-20T20:05:38.219827+0000","last_fullsized":"2024-05-20T20:05:38.219827+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":233,"num_read_kb":233,"num_write":15,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741585+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:05:37.741585+0000","last_peered":"2024-05-20T20:05:37.741585+0000","last_clean":"2024-05-20T20:05:37.741585+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:05:37.741585+0000","last_undegraded":"2024-05-20T20:05:37.741585+0000","last_fullsized":"2024-05-20T20:05:37.741585+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741764+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:05:37.741764+0000","last_peered":"2024-05-20T20:05:37.741764+0000","last_clean":"2024-05-20T20:05:37.741764+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:05:37.741764+0000","last_undegraded":"2024-05-20T20:05:37.741764+0000","last_fullsized":"2024-05-20T20:05:37.741764+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750654+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:05:37.750654+0000","last_peered":"2024-05-20T20:05:37.750654+0000","last_clean":"2024-05-20T20:05:37.750654+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:05:37.750654+0000","last_undegraded":"2024-05-20T20:05:37.750654+0000","last_fullsized":"2024-05-20T20:05:37.750654+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"78'12","reported_seq":42,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950953+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:05:37.950953+0000","last_peered":"2024-05-20T20:05:37.950953+0000","last_clean":"2024-05-20T20:05:37.950953+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:05:37.950953+0000","last_undegraded":"2024-05-20T20:05:37.950953+0000","last_fullsized":"2024-05-20T20:05:37.950953+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950749+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:05:37.950749+0000","last_peered":"2024-05-20T20:05:37.950749+0000","last_clean":"2024-05-20T20:05:37.950749+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:05:37.950749+0000","last_undegraded":"2024-05-20T20:05:37.950749+0000","last_fullsized":"2024-05-20T20:05:37.950749+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602506+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:05:37.602506+0000","last_peered":"2024-05-20T20:05:37.602506+0000","last_clean":"2024-05-20T20:05:37.602506+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:05:37.602506+0000","last_undegraded":"2024-05-20T20:05:37.602506+0000","last_fullsized":"2024-05-20T20:05:37.602506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"81'8","reported_seq":279,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.037384+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:05:42.037384+0000","last_peered":"2024-05-20T20:05:42.037384+0000","last_clean":"2024-05-20T20:05:42.037384+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:05:42.037384+0000","last_undegraded":"2024-05-20T20:05:42.037384+0000","last_fullsized":"2024-05-20T20:05:42.037384+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1545,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":242,"num_read_kb":5,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950345+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:05:37.950345+0000","last_peered":"2024-05-20T20:05:37.950345+0000","last_clean":"2024-05-20T20:05:37.950345+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:05:37.950345+0000","last_undegraded":"2024-05-20T20:05:37.950345+0000","last_fullsized":"2024-05-20T20:05:37.950345+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601320+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:05:37.601320+0000","last_peered":"2024-05-20T20:05:37.601320+0000","last_clean":"2024-05-20T20:05:37.601320+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:05:37.601320+0000","last_undegraded":"2024-05-20T20:05:37.601320+0000","last_fullsized":"2024-05-20T20:05:37.601320+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751407+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:05:37.751407+0000","last_peered":"2024-05-20T20:05:37.751407+0000","last_clean":"2024-05-20T20:05:37.751407+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:05:37.751407+0000","last_undegraded":"2024-05-20T20:05:37.751407+0000","last_fullsized":"2024-05-20T20:05:37.751407+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950714+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:05:37.950714+0000","last_peered":"2024-05-20T20:05:37.950714+0000","last_clean":"2024-05-20T20:05:37.950714+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:05:37.950714+0000","last_undegraded":"2024-05-20T20:05:37.950714+0000","last_fullsized":"2024-05-20T20:05:37.950714+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740756+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:05:37.740756+0000","last_peered":"2024-05-20T20:05:37.740756+0000","last_clean":"2024-05-20T20:05:37.740756+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:05:37.740756+0000","last_undegraded":"2024-05-20T20:05:37.740756+0000","last_fullsized":"2024-05-20T20:05:37.740756+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751178+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:05:37.751178+0000","last_peered":"2024-05-20T20:05:37.751178+0000","last_clean":"2024-05-20T20:05:37.751178+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:05:37.751178+0000","last_undegraded":"2024-05-20T20:05:37.751178+0000","last_fullsized":"2024-05-20T20:05:37.751178+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601398+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:05:37.601398+0000","last_peered":"2024-05-20T20:05:37.601398+0000","last_clean":"2024-05-20T20:05:37.601398+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:05:37.601398+0000","last_undegraded":"2024-05-20T20:05:37.601398+0000","last_fullsized":"2024-05-20T20:05:37.601398+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'7","reported_seq":510,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:43.038090+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:05:43.038090+0000","last_peered":"2024-05-20T20:05:43.038090+0000","last_clean":"2024-05-20T20:05:43.038090+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:05:43.038090+0000","last_undegraded":"2024-05-20T20:05:43.038090+0000","last_fullsized":"2024-05-20T20:05:43.038090+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":474,"num_read_kb":474,"num_write":8,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941795+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:05:37.941795+0000","last_peered":"2024-05-20T20:05:37.941795+0000","last_clean":"2024-05-20T20:05:37.941795+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:05:37.941795+0000","last_undegraded":"2024-05-20T20:05:37.941795+0000","last_fullsized":"2024-05-20T20:05:37.941795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947471+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:05:37.947471+0000","last_peered":"2024-05-20T20:05:37.947471+0000","last_clean":"2024-05-20T20:05:37.947471+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:05:37.947471+0000","last_undegraded":"2024-05-20T20:05:37.947471+0000","last_fullsized":"2024-05-20T20:05:37.947471+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750655+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:05:37.750655+0000","last_peered":"2024-05-20T20:05:37.750655+0000","last_clean":"2024-05-20T20:05:37.750655+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:05:37.750655+0000","last_undegraded":"2024-05-20T20:05:37.750655+0000","last_fullsized":"2024-05-20T20:05:37.750655+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751776+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:05:37.751776+0000","last_peered":"2024-05-20T20:05:37.751776+0000","last_clean":"2024-05-20T20:05:37.751776+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:05:37.751776+0000","last_undegraded":"2024-05-20T20:05:37.751776+0000","last_fullsized":"2024-05-20T20:05:37.751776+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941312+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:05:37.941312+0000","last_peered":"2024-05-20T20:05:37.941312+0000","last_clean":"2024-05-20T20:05:37.941312+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:05:37.941312+0000","last_undegraded":"2024-05-20T20:05:37.941312+0000","last_fullsized":"2024-05-20T20:05:37.941312+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'39","reported_seq":556,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:43.038486+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:05:43.038486+0000","last_peered":"2024-05-20T20:05:43.038486+0000","last_clean":"2024-05-20T20:05:43.038486+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:05:43.038486+0000","last_undegraded":"2024-05-20T20:05:43.038486+0000","last_fullsized":"2024-05-20T20:05:43.038486+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70510,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":488,"num_read_kb":313,"num_write":41,"num_write_kb":85,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950750+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:05:37.950750+0000","last_peered":"2024-05-20T20:05:37.950750+0000","last_clean":"2024-05-20T20:05:37.950750+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:05:37.950750+0000","last_undegraded":"2024-05-20T20:05:37.950750+0000","last_fullsized":"2024-05-20T20:05:37.950750+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602898+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:05:37.602898+0000","last_peered":"2024-05-20T20:05:37.602898+0000","last_clean":"2024-05-20T20:05:37.602898+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:05:37.602898+0000","last_undegraded":"2024-05-20T20:05:37.602898+0000","last_fullsized":"2024-05-20T20:05:37.602898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950828+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:05:37.950828+0000","last_peered":"2024-05-20T20:05:37.950828+0000","last_clean":"2024-05-20T20:05:37.950828+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:05:37.950828+0000","last_undegraded":"2024-05-20T20:05:37.950828+0000","last_fullsized":"2024-05-20T20:05:37.950828+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751260+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:05:37.751260+0000","last_peered":"2024-05-20T20:05:37.751260+0000","last_clean":"2024-05-20T20:05:37.751260+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:05:37.751260+0000","last_undegraded":"2024-05-20T20:05:37.751260+0000","last_fullsized":"2024-05-20T20:05:37.751260+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947376+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:05:37.947376+0000","last_peered":"2024-05-20T20:05:37.947376+0000","last_clean":"2024-05-20T20:05:37.947376+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:05:37.947376+0000","last_undegraded":"2024-05-20T20:05:37.947376+0000","last_fullsized":"2024-05-20T20:05:37.947376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'8","reported_seq":279,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.228671+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:05:42.228671+0000","last_peered":"2024-05-20T20:05:42.228671+0000","last_clean":"2024-05-20T20:05:42.228671+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:05:42.228671+0000","last_undegraded":"2024-05-20T20:05:42.228671+0000","last_fullsized":"2024-05-20T20:05:42.228671+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1680,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":243,"num_read_kb":6,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946134+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:05:37.946134+0000","last_peered":"2024-05-20T20:05:37.946134+0000","last_clean":"2024-05-20T20:05:37.946134+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:05:37.946134+0000","last_undegraded":"2024-05-20T20:05:37.946134+0000","last_fullsized":"2024-05-20T20:05:37.946134+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":71,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950299+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:05:37.950299+0000","last_peered":"2024-05-20T20:05:37.950299+0000","last_clean":"2024-05-20T20:05:37.950299+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:05:37.950299+0000","last_undegraded":"2024-05-20T20:05:37.950299+0000","last_fullsized":"2024-05-20T20:05:37.950299+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751376+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:05:37.751376+0000","last_peered":"2024-05-20T20:05:37.751376+0000","last_clean":"2024-05-20T20:05:37.751376+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:05:37.751376+0000","last_undegraded":"2024-05-20T20:05:37.751376+0000","last_fullsized":"2024-05-20T20:05:37.751376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"80'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950961+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:05:37.950961+0000","last_peered":"2024-05-20T20:05:37.950961+0000","last_clean":"2024-05-20T20:05:37.950961+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:05:37.950961+0000","last_undegraded":"2024-05-20T20:05:37.950961+0000","last_fullsized":"2024-05-20T20:05:37.950961+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603114+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:05:37.603114+0000","last_peered":"2024-05-20T20:05:37.603114+0000","last_clean":"2024-05-20T20:05:37.603114+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:05:37.603114+0000","last_undegraded":"2024-05-20T20:05:37.603114+0000","last_fullsized":"2024-05-20T20:05:37.603114+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602390+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:05:37.602390+0000","last_peered":"2024-05-20T20:05:37.602390+0000","last_clean":"2024-05-20T20:05:37.602390+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:05:37.602390+0000","last_undegraded":"2024-05-20T20:05:37.602390+0000","last_fullsized":"2024-05-20T20:05:37.602390+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601741+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:05:37.601741+0000","last_peered":"2024-05-20T20:05:37.601741+0000","last_clean":"2024-05-20T20:05:37.601741+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:05:37.601741+0000","last_undegraded":"2024-05-20T20:05:37.601741+0000","last_fullsized":"2024-05-20T20:05:37.601741+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'6","reported_seq":270,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.221448+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:05:38.221448+0000","last_peered":"2024-05-20T20:05:38.221448+0000","last_clean":"2024-05-20T20:05:38.221448+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:05:38.221448+0000","last_undegraded":"2024-05-20T20:05:38.221448+0000","last_fullsized":"2024-05-20T20:05:38.221448+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":234,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601417+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:05:37.601417+0000","last_peered":"2024-05-20T20:05:37.601417+0000","last_clean":"2024-05-20T20:05:37.601417+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:05:37.601417+0000","last_undegraded":"2024-05-20T20:05:37.601417+0000","last_fullsized":"2024-05-20T20:05:37.601417+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751541+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:05:37.751541+0000","last_peered":"2024-05-20T20:05:37.751541+0000","last_clean":"2024-05-20T20:05:37.751541+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:05:37.751541+0000","last_undegraded":"2024-05-20T20:05:37.751541+0000","last_fullsized":"2024-05-20T20:05:37.751541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"80'6","reported_seq":269,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.221082+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:05:38.221082+0000","last_peered":"2024-05-20T20:05:38.221082+0000","last_clean":"2024-05-20T20:05:38.221082+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:05:38.221082+0000","last_undegraded":"2024-05-20T20:05:38.221082+0000","last_fullsized":"2024-05-20T20:05:38.221082+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":233,"num_read_kb":233,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741186+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:05:37.741186+0000","last_peered":"2024-05-20T20:05:37.741186+0000","last_clean":"2024-05-20T20:05:37.741186+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:05:37.741186+0000","last_undegraded":"2024-05-20T20:05:37.741186+0000","last_fullsized":"2024-05-20T20:05:37.741186+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.945964+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:05:37.945964+0000","last_peered":"2024-05-20T20:05:37.945964+0000","last_clean":"2024-05-20T20:05:37.945964+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:05:37.945964+0000","last_undegraded":"2024-05-20T20:05:37.945964+0000","last_fullsized":"2024-05-20T20:05:37.945964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"20'92","reported_seq":103,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946374+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:05:37.946374+0000","last_peered":"2024-05-20T20:05:37.946374+0000","last_clean":"2024-05-20T20:05:37.946374+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:05:37.946374+0000","last_undegraded":"2024-05-20T20:05:37.946374+0000","last_fullsized":"2024-05-20T20:05:37.946374+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950636+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:05:37.950636+0000","last_peered":"2024-05-20T20:05:37.950636+0000","last_clean":"2024-05-20T20:05:37.950636+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:05:37.950636+0000","last_undegraded":"2024-05-20T20:05:37.950636+0000","last_fullsized":"2024-05-20T20:05:37.950636+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946570+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:05:37.946570+0000","last_peered":"2024-05-20T20:05:37.946570+0000","last_clean":"2024-05-20T20:05:37.946570+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:05:37.946570+0000","last_undegraded":"2024-05-20T20:05:37.946570+0000","last_fullsized":"2024-05-20T20:05:37.946570+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950840+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:05:37.950840+0000","last_peered":"2024-05-20T20:05:37.950840+0000","last_clean":"2024-05-20T20:05:37.950840+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:05:37.950840+0000","last_undegraded":"2024-05-20T20:05:37.950840+0000","last_fullsized":"2024-05-20T20:05:37.950840+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952108+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:05:37.952108+0000","last_peered":"2024-05-20T20:05:37.952108+0000","last_clean":"2024-05-20T20:05:37.952108+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:05:37.952108+0000","last_undegraded":"2024-05-20T20:05:37.952108+0000","last_fullsized":"2024-05-20T20:05:37.952108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'51","reported_seq":120,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.896768+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:05:37.896768+0000","last_peered":"2024-05-20T20:05:37.896768+0000","last_clean":"2024-05-20T20:05:37.896768+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:05:37.896768+0000","last_undegraded":"2024-05-20T20:05:37.896768+0000","last_fullsized":"2024-05-20T20:05:37.896768+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":51,"log_dups_size":0,"ondisk_log_size":51,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":91,"num_write_kb":838,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741235+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:05:37.741235+0000","last_peered":"2024-05-20T20:05:37.741235+0000","last_clean":"2024-05-20T20:05:37.741235+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:05:37.741235+0000","last_undegraded":"2024-05-20T20:05:37.741235+0000","last_fullsized":"2024-05-20T20:05:37.741235+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946101+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:05:37.946101+0000","last_peered":"2024-05-20T20:05:37.946101+0000","last_clean":"2024-05-20T20:05:37.946101+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:05:37.946101+0000","last_undegraded":"2024-05-20T20:05:37.946101+0000","last_fullsized":"2024-05-20T20:05:37.946101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750578+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:05:37.750578+0000","last_peered":"2024-05-20T20:05:37.750578+0000","last_clean":"2024-05-20T20:05:37.750578+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:05:37.750578+0000","last_undegraded":"2024-05-20T20:05:37.750578+0000","last_fullsized":"2024-05-20T20:05:37.750578+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941387+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:05:37.941387+0000","last_peered":"2024-05-20T20:05:37.941387+0000","last_clean":"2024-05-20T20:05:37.941387+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:05:37.941387+0000","last_undegraded":"2024-05-20T20:05:37.941387+0000","last_fullsized":"2024-05-20T20:05:37.941387+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":25,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950298+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:05:37.950298+0000","last_peered":"2024-05-20T20:05:37.950298+0000","last_clean":"2024-05-20T20:05:37.950298+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:05:37.950298+0000","last_undegraded":"2024-05-20T20:05:37.950298+0000","last_fullsized":"2024-05-20T20:05:37.950298+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740964+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:05:37.740964+0000","last_peered":"2024-05-20T20:05:37.740964+0000","last_clean":"2024-05-20T20:05:37.740964+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:05:37.740964+0000","last_undegraded":"2024-05-20T20:05:37.740964+0000","last_fullsized":"2024-05-20T20:05:37.740964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602823+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:05:37.602823+0000","last_peered":"2024-05-20T20:05:37.602823+0000","last_clean":"2024-05-20T20:05:37.602823+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:05:37.602823+0000","last_undegraded":"2024-05-20T20:05:37.602823+0000","last_fullsized":"2024-05-20T20:05:37.602823+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940442+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:05:37.940442+0000","last_peered":"2024-05-20T20:05:37.940442+0000","last_clean":"2024-05-20T20:05:37.940442+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:05:37.940442+0000","last_undegraded":"2024-05-20T20:05:37.940442+0000","last_fullsized":"2024-05-20T20:05:37.940442+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741376+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:05:37.741376+0000","last_peered":"2024-05-20T20:05:37.741376+0000","last_clean":"2024-05-20T20:05:37.741376+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:05:37.741376+0000","last_undegraded":"2024-05-20T20:05:37.741376+0000","last_fullsized":"2024-05-20T20:05:37.741376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950472+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:05:37.950472+0000","last_peered":"2024-05-20T20:05:37.950472+0000","last_clean":"2024-05-20T20:05:37.950472+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:05:37.950472+0000","last_undegraded":"2024-05-20T20:05:37.950472+0000","last_fullsized":"2024-05-20T20:05:37.950472+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947444+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:05:37.947444+0000","last_peered":"2024-05-20T20:05:37.947444+0000","last_clean":"2024-05-20T20:05:37.947444+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:05:37.947444+0000","last_undegraded":"2024-05-20T20:05:37.947444+0000","last_fullsized":"2024-05-20T20:05:37.947444+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'6","reported_seq":270,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.031306+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:05:38.031306+0000","last_peered":"2024-05-20T20:05:38.031306+0000","last_clean":"2024-05-20T20:05:38.031306+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:05:38.031306+0000","last_undegraded":"2024-05-20T20:05:38.031306+0000","last_fullsized":"2024-05-20T20:05:38.031306+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1561,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":235,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947761+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:05:37.947761+0000","last_peered":"2024-05-20T20:05:37.947761+0000","last_clean":"2024-05-20T20:05:37.947761+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:05:37.947761+0000","last_undegraded":"2024-05-20T20:05:37.947761+0000","last_fullsized":"2024-05-20T20:05:37.947761+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"81'41","reported_seq":78,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.890409+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:05:37.890409+0000","last_peered":"2024-05-20T20:05:37.890409+0000","last_clean":"2024-05-20T20:05:37.890409+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:05:37.890409+0000","last_undegraded":"2024-05-20T20:05:37.890409+0000","last_fullsized":"2024-05-20T20:05:37.890409+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":41,"log_dups_size":0,"ondisk_log_size":41,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947433+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:05:37.947433+0000","last_peered":"2024-05-20T20:05:37.947433+0000","last_clean":"2024-05-20T20:05:37.947433+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:05:37.947433+0000","last_undegraded":"2024-05-20T20:05:37.947433+0000","last_fullsized":"2024-05-20T20:05:37.947433+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952316+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:05:37.952316+0000","last_peered":"2024-05-20T20:05:37.952316+0000","last_clean":"2024-05-20T20:05:37.952316+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:05:37.952316+0000","last_undegraded":"2024-05-20T20:05:37.952316+0000","last_fullsized":"2024-05-20T20:05:37.952316+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947760+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:05:37.947760+0000","last_peered":"2024-05-20T20:05:37.947760+0000","last_clean":"2024-05-20T20:05:37.947760+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:05:37.947760+0000","last_undegraded":"2024-05-20T20:05:37.947760+0000","last_fullsized":"2024-05-20T20:05:37.947760+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603108+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:05:37.603108+0000","last_peered":"2024-05-20T20:05:37.603108+0000","last_clean":"2024-05-20T20:05:37.603108+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:05:37.603108+0000","last_undegraded":"2024-05-20T20:05:37.603108+0000","last_fullsized":"2024-05-20T20:05:37.603108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940537+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:05:37.940537+0000","last_peered":"2024-05-20T20:05:37.940537+0000","last_clean":"2024-05-20T20:05:37.940537+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:05:37.940537+0000","last_undegraded":"2024-05-20T20:05:37.940537+0000","last_fullsized":"2024-05-20T20:05:37.940537+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940554+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:05:37.940554+0000","last_peered":"2024-05-20T20:05:37.940554+0000","last_clean":"2024-05-20T20:05:37.940554+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:05:37.940554+0000","last_undegraded":"2024-05-20T20:05:37.940554+0000","last_fullsized":"2024-05-20T20:05:37.940554+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750191+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:05:37.750191+0000","last_peered":"2024-05-20T20:05:37.750191+0000","last_clean":"2024-05-20T20:05:37.750191+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:05:37.750191+0000","last_undegraded":"2024-05-20T20:05:37.750191+0000","last_fullsized":"2024-05-20T20:05:37.750191+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952180+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:05:37.952180+0000","last_peered":"2024-05-20T20:05:37.952180+0000","last_clean":"2024-05-20T20:05:37.952180+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:05:37.952180+0000","last_undegraded":"2024-05-20T20:05:37.952180+0000","last_fullsized":"2024-05-20T20:05:37.952180+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603150+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:05:37.603150+0000","last_peered":"2024-05-20T20:05:37.603150+0000","last_clean":"2024-05-20T20:05:37.603150+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:05:37.603150+0000","last_undegraded":"2024-05-20T20:05:37.603150+0000","last_fullsized":"2024-05-20T20:05:37.603150+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940773+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:05:37.940773+0000","last_peered":"2024-05-20T20:05:37.940773+0000","last_clean":"2024-05-20T20:05:37.940773+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:05:37.940773+0000","last_undegraded":"2024-05-20T20:05:37.940773+0000","last_fullsized":"2024-05-20T20:05:37.940773+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":25,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741108+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:05:37.741108+0000","last_peered":"2024-05-20T20:05:37.741108+0000","last_clean":"2024-05-20T20:05:37.741108+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:05:37.741108+0000","last_undegraded":"2024-05-20T20:05:37.741108+0000","last_fullsized":"2024-05-20T20:05:37.741108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751255+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:05:37.751255+0000","last_peered":"2024-05-20T20:05:37.751255+0000","last_clean":"2024-05-20T20:05:37.751255+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:05:37.751255+0000","last_undegraded":"2024-05-20T20:05:37.751255+0000","last_fullsized":"2024-05-20T20:05:37.751255+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740671+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:05:37.740671+0000","last_peered":"2024-05-20T20:05:37.740671+0000","last_clean":"2024-05-20T20:05:37.740671+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:05:37.740671+0000","last_undegraded":"2024-05-20T20:05:37.740671+0000","last_fullsized":"2024-05-20T20:05:37.740671+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941782+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:05:37.941782+0000","last_peered":"2024-05-20T20:05:37.941782+0000","last_clean":"2024-05-20T20:05:37.941782+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:05:37.941782+0000","last_undegraded":"2024-05-20T20:05:37.941782+0000","last_fullsized":"2024-05-20T20:05:37.941782+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951595+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:05:37.951595+0000","last_peered":"2024-05-20T20:05:37.951595+0000","last_clean":"2024-05-20T20:05:37.951595+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:05:37.951595+0000","last_undegraded":"2024-05-20T20:05:37.951595+0000","last_fullsized":"2024-05-20T20:05:37.951595+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602929+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:05:37.602929+0000","last_peered":"2024-05-20T20:05:37.602929+0000","last_clean":"2024-05-20T20:05:37.602929+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:05:37.602929+0000","last_undegraded":"2024-05-20T20:05:37.602929+0000","last_fullsized":"2024-05-20T20:05:37.602929+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":108381,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":604545,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":2494,"num_read_kb":1368,"num_write":378,"num_write_kb":1470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":729088,"data_stored":1807422,"data_compressed":75906,"data_compressed_allocated":98304,"data_compressed_original":1548288,"omap_allocated":0,"internal_metadata":0},"log_size":304,"ondisk_log_size":304,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710789,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":917504,"data_stored":828458,"data_compressed":27826,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877907018,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28060,"kb_used_data":1044,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717092,"statfs":{"total":95995035648,"available":95966302208,"internally_reserved":0,"allocated":1069056,"data_stored":1226494,"data_compressed":50963,"data_compressed_allocated":65536,"data_compressed_original":737280,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070542,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32092,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713060,"statfs":{"total":95995035648,"available":95962173440,"internally_reserved":0,"allocated":933888,"data_stored":827929,"data_compressed":27826,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266772,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":905216,"data_stored":435091,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863463002,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32116,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713036,"statfs":{"total":95995035648,"available":95962148864,"internally_reserved":0,"allocated":958464,"data_stored":1029233,"data_compressed":44807,"data_compressed_allocated":57344,"data_compressed_original":614400,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626528,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":892928,"data_stored":434821,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574972,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":489281,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738498,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":426813,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411447,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":928,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":950272,"data_stored":489853,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182513,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27948,"kb_used_data":932,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717204,"statfs":{"total":95995035648,"available":95966416896,"internally_reserved":0,"allocated":954368,"data_stored":959071,"data_compressed":33982,"data_compressed_allocated":45056,"data_compressed_original":540672,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986285,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27900,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717252,"statfs":{"total":95995035648,"available":95966466048,"internally_reserved":0,"allocated":901120,"data_stored":435495,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822759,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32144,"kb_used_data":968,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713008,"statfs":{"total":95995035648,"available":95962120192,"internally_reserved":0,"allocated":991232,"data_stored":1159419,"data_compressed":50963,"data_compressed_allocated":65536,"data_compressed_original":737280,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":69742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":70705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":73728,"data_stored":525246,"data_compressed":25302,"data_compressed_allocated":32768,"data_compressed_original":516096,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":135343,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":394338,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":204938,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394867,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:05:45.152 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-05-20T20:05:45.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 -- 172.21.15.28:0/2362571111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935410a960 msgr2=0x7f935410cdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 --2- 172.21.15.28:0/2362571111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935410a960 0x7f935410cdf0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f933c009860 tx=0x7f933c02a340 comp rx=0 tx=0).stop 2024-05-20T20:05:45.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 -- 172.21.15.28:0/2362571111 shutdown_connections 2024-05-20T20:05:45.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 --2- 172.21.15.28:0/2362571111 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93540754b0 0x7f9354075930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 --2- 172.21.15.28:0/2362571111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935410a960 0x7f935410cdf0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 --2- 172.21.15.28:0/2362571111 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9354076a90 0x7f9354074ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 -- 172.21.15.28:0/2362571111 >> 172.21.15.28:0/2362571111 conn(0x7f93540fe030 msgr2=0x7f9354100450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:45.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 -- 172.21.15.28:0/2362571111 shutdown_connections 2024-05-20T20:05:45.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.377+0000 7f935ac00640 1 -- 172.21.15.28:0/2362571111 wait complete. 2024-05-20T20:05:45.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.378+0000 7f935ac00640 1 Processor -- start 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f935ac00640 1 -- start start 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f935ac00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93540754b0 0x7f935419ae50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f935ac00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9354076a90 0x7f935419b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f935ac00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f935410a960 0x7f93541a2400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f935ac00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9354110790 con 0x7f9354076a90 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f935ac00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9354110610 con 0x7f935410a960 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f935ac00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9354110910 con 0x7f93540754b0 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f9353e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93540754b0 0x7f935419ae50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f9353400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9354076a90 0x7f935419b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f9358c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f935410a960 0x7f93541a2400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f9353e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93540754b0 0x7f935419ae50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55636/0 (socket says 172.21.15.28:55636) 2024-05-20T20:05:45.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f9358c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f935410a960 0x7f93541a2400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37952/0 (socket says 172.21.15.28:37952) 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.379+0000 7f9353e00640 1 -- 172.21.15.28:0/4251543876 learned_addr learned my addr 172.21.15.28:0/4251543876 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9353e00640 1 -- 172.21.15.28:0/4251543876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f935410a960 msgr2=0x7f93541a2400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9353e00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f935410a960 0x7f93541a2400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9353e00640 1 -- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9354076a90 msgr2=0x7f935419b390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9353e00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9354076a90 0x7f935419b390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9353e00640 1 -- 172.21.15.28:0/4251543876 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f93541a2a70 con 0x7f93540754b0 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9353e00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93540754b0 0x7f935419ae50 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f934c00efb0 tx=0x7f934c00c800 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9350c00640 1 -- 172.21.15.28:0/4251543876 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f934c01b070 con 0x7f93540754b0 2024-05-20T20:05:45.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9350c00640 1 -- 172.21.15.28:0/4251543876 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f934c00ba20 con 0x7f93540754b0 2024-05-20T20:05:45.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f93541a2d60 con 0x7f93540754b0 2024-05-20T20:05:45.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f9350c00640 1 -- 172.21.15.28:0/4251543876 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f934c016b30 con 0x7f93540754b0 2024-05-20T20:05:45.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.380+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f93541a31f0 con 0x7f93540754b0 2024-05-20T20:05:45.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.382+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9354109690 con 0x7f93540754b0 2024-05-20T20:05:45.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.385+0000 7f9350c00640 1 -- 172.21.15.28:0/4251543876 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f934c01e070 con 0x7f93540754b0 2024-05-20T20:05:45.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.386+0000 7f9350c00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93380b2a80 0x7f93380b4f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.387+0000 7f9350c00640 1 -- 172.21.15.28:0/4251543876 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f934c0e2880 con 0x7f93540754b0 2024-05-20T20:05:45.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.387+0000 7f9353400640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93380b2a80 0x7f93380b4f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.387+0000 7f9353400640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93380b2a80 0x7f93380b4f40 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f935419c360 tx=0x7f9344005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:45.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.394+0000 7f9350c00640 1 -- 172.21.15.28:0/4251543876 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f934c0a9970 con 0x7f93540754b0 2024-05-20T20:05:45.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.606+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f9354062e90 con 0x7f93380b2a80 2024-05-20T20:05:45.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.614+0000 7f9350c00640 1 -- 172.21.15.28:0/4251543876 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258090 (secure 0 0 0) 0x7f9354062e90 con 0x7f93380b2a80 2024-05-20T20:05:45.619 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:45.620 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93380b2a80 msgr2=0x7f93380b4f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93380b2a80 0x7f93380b4f40 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f935419c360 tx=0x7f9344005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93540754b0 msgr2=0x7f935419ae50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93540754b0 0x7f935419ae50 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f934c00efb0 tx=0x7f934c00c800 comp rx=0 tx=0).stop 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 shutdown_connections 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f935410a960 0x7f93541a2400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93380b2a80 0x7f93380b4f40 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9354076a90 0x7f935419b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 --2- 172.21.15.28:0/4251543876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93540754b0 0x7f935419ae50 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 >> 172.21.15.28:0/4251543876 conn(0x7f93540fe030 msgr2=0x7f9354106860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:45.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 shutdown_connections 2024-05-20T20:05:45.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.619+0000 7f935ac00640 1 -- 172.21.15.28:0/4251543876 wait complete. 2024-05-20T20:05:45.639 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":6,"stamp":"2024-05-20T20:05:43.627844+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1195147,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":2575,"num_read_kb":1437,"num_write":503,"num_write_kb":3622,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":898,"ondisk_log_size":898,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347748,"kb_used_data":11056,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124594076,"statfs":{"total":1151940427776,"available":1151584333824,"internally_reserved":0,"allocated":11321344,"data_stored":8741958,"data_compressed":279771,"data_compressed_allocated":368640,"data_compressed_original":3588096,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3302,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":33,"num_read_kb":20,"num_write":3,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.982270"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603392+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:05:37.603392+0000","last_peered":"2024-05-20T20:05:37.603392+0000","last_clean":"2024-05-20T20:05:37.603392+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:05:37.603392+0000","last_undegraded":"2024-05-20T20:05:37.603392+0000","last_fullsized":"2024-05-20T20:05:37.603392+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750621+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:05:37.750621+0000","last_peered":"2024-05-20T20:05:37.750621+0000","last_clean":"2024-05-20T20:05:37.750621+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:05:37.750621+0000","last_undegraded":"2024-05-20T20:05:37.750621+0000","last_fullsized":"2024-05-20T20:05:37.750621+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750419+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:05:37.750419+0000","last_peered":"2024-05-20T20:05:37.750419+0000","last_clean":"2024-05-20T20:05:37.750419+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:05:37.750419+0000","last_undegraded":"2024-05-20T20:05:37.750419+0000","last_fullsized":"2024-05-20T20:05:37.750419+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946345+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:05:37.946345+0000","last_peered":"2024-05-20T20:05:37.946345+0000","last_clean":"2024-05-20T20:05:37.946345+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:05:37.946345+0000","last_undegraded":"2024-05-20T20:05:37.946345+0000","last_fullsized":"2024-05-20T20:05:37.946345+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":27,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950341+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:05:37.950341+0000","last_peered":"2024-05-20T20:05:37.950341+0000","last_clean":"2024-05-20T20:05:37.950341+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:05:37.950341+0000","last_undegraded":"2024-05-20T20:05:37.950341+0000","last_fullsized":"2024-05-20T20:05:37.950341+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601481+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:05:37.601481+0000","last_peered":"2024-05-20T20:05:37.601481+0000","last_clean":"2024-05-20T20:05:37.601481+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:05:37.601481+0000","last_undegraded":"2024-05-20T20:05:37.601481+0000","last_fullsized":"2024-05-20T20:05:37.601481+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946714+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:05:37.946714+0000","last_peered":"2024-05-20T20:05:37.946714+0000","last_clean":"2024-05-20T20:05:37.946714+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:05:37.946714+0000","last_undegraded":"2024-05-20T20:05:37.946714+0000","last_fullsized":"2024-05-20T20:05:37.946714+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740698+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:05:37.740698+0000","last_peered":"2024-05-20T20:05:37.740698+0000","last_clean":"2024-05-20T20:05:37.740698+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:05:37.740698+0000","last_undegraded":"2024-05-20T20:05:37.740698+0000","last_fullsized":"2024-05-20T20:05:37.740698+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.942016+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:05:37.942016+0000","last_peered":"2024-05-20T20:05:37.942016+0000","last_clean":"2024-05-20T20:05:37.942016+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:05:37.942016+0000","last_undegraded":"2024-05-20T20:05:37.942016+0000","last_fullsized":"2024-05-20T20:05:37.942016+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":45,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601908+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:05:37.601908+0000","last_peered":"2024-05-20T20:05:37.601908+0000","last_clean":"2024-05-20T20:05:37.601908+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:05:37.601908+0000","last_undegraded":"2024-05-20T20:05:37.601908+0000","last_fullsized":"2024-05-20T20:05:37.601908+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947610+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:05:37.947610+0000","last_peered":"2024-05-20T20:05:37.947610+0000","last_clean":"2024-05-20T20:05:37.947610+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:05:37.947610+0000","last_undegraded":"2024-05-20T20:05:37.947610+0000","last_fullsized":"2024-05-20T20:05:37.947610+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946190+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:05:37.946190+0000","last_peered":"2024-05-20T20:05:37.946190+0000","last_clean":"2024-05-20T20:05:37.946190+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:05:37.946190+0000","last_undegraded":"2024-05-20T20:05:37.946190+0000","last_fullsized":"2024-05-20T20:05:37.946190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951933+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:05:37.951933+0000","last_peered":"2024-05-20T20:05:37.951933+0000","last_clean":"2024-05-20T20:05:37.951933+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:05:37.951933+0000","last_undegraded":"2024-05-20T20:05:37.951933+0000","last_fullsized":"2024-05-20T20:05:37.951933+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749940+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:05:37.749940+0000","last_peered":"2024-05-20T20:05:37.749940+0000","last_clean":"2024-05-20T20:05:37.749940+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:05:37.749940+0000","last_undegraded":"2024-05-20T20:05:37.749940+0000","last_fullsized":"2024-05-20T20:05:37.749940+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749763+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:05:37.749763+0000","last_peered":"2024-05-20T20:05:37.749763+0000","last_clean":"2024-05-20T20:05:37.749763+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:05:37.749763+0000","last_undegraded":"2024-05-20T20:05:37.749763+0000","last_fullsized":"2024-05-20T20:05:37.749763+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603526+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:05:37.603526+0000","last_peered":"2024-05-20T20:05:37.603526+0000","last_clean":"2024-05-20T20:05:37.603526+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:05:37.603526+0000","last_undegraded":"2024-05-20T20:05:37.603526+0000","last_fullsized":"2024-05-20T20:05:37.603526+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602253+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:05:37.602253+0000","last_peered":"2024-05-20T20:05:37.602253+0000","last_clean":"2024-05-20T20:05:37.602253+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:05:37.602253+0000","last_undegraded":"2024-05-20T20:05:37.602253+0000","last_fullsized":"2024-05-20T20:05:37.602253+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940646+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:05:37.940646+0000","last_peered":"2024-05-20T20:05:37.940646+0000","last_clean":"2024-05-20T20:05:37.940646+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:05:37.940646+0000","last_undegraded":"2024-05-20T20:05:37.940646+0000","last_fullsized":"2024-05-20T20:05:37.940646+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'14","reported_seq":277,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.219827+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:05:38.219827+0000","last_peered":"2024-05-20T20:05:38.219827+0000","last_clean":"2024-05-20T20:05:38.219827+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:05:38.219827+0000","last_undegraded":"2024-05-20T20:05:38.219827+0000","last_fullsized":"2024-05-20T20:05:38.219827+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":233,"num_read_kb":233,"num_write":15,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741585+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:05:37.741585+0000","last_peered":"2024-05-20T20:05:37.741585+0000","last_clean":"2024-05-20T20:05:37.741585+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:05:37.741585+0000","last_undegraded":"2024-05-20T20:05:37.741585+0000","last_fullsized":"2024-05-20T20:05:37.741585+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741764+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:05:37.741764+0000","last_peered":"2024-05-20T20:05:37.741764+0000","last_clean":"2024-05-20T20:05:37.741764+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:05:37.741764+0000","last_undegraded":"2024-05-20T20:05:37.741764+0000","last_fullsized":"2024-05-20T20:05:37.741764+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750654+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:05:37.750654+0000","last_peered":"2024-05-20T20:05:37.750654+0000","last_clean":"2024-05-20T20:05:37.750654+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:05:37.750654+0000","last_undegraded":"2024-05-20T20:05:37.750654+0000","last_fullsized":"2024-05-20T20:05:37.750654+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"78'12","reported_seq":42,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950953+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:05:37.950953+0000","last_peered":"2024-05-20T20:05:37.950953+0000","last_clean":"2024-05-20T20:05:37.950953+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:05:37.950953+0000","last_undegraded":"2024-05-20T20:05:37.950953+0000","last_fullsized":"2024-05-20T20:05:37.950953+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950749+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:05:37.950749+0000","last_peered":"2024-05-20T20:05:37.950749+0000","last_clean":"2024-05-20T20:05:37.950749+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:05:37.950749+0000","last_undegraded":"2024-05-20T20:05:37.950749+0000","last_fullsized":"2024-05-20T20:05:37.950749+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602506+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:05:37.602506+0000","last_peered":"2024-05-20T20:05:37.602506+0000","last_clean":"2024-05-20T20:05:37.602506+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:05:37.602506+0000","last_undegraded":"2024-05-20T20:05:37.602506+0000","last_fullsized":"2024-05-20T20:05:37.602506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"81'8","reported_seq":279,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.037384+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:05:42.037384+0000","last_peered":"2024-05-20T20:05:42.037384+0000","last_clean":"2024-05-20T20:05:42.037384+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:05:42.037384+0000","last_undegraded":"2024-05-20T20:05:42.037384+0000","last_fullsized":"2024-05-20T20:05:42.037384+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1545,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":242,"num_read_kb":5,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950345+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:05:37.950345+0000","last_peered":"2024-05-20T20:05:37.950345+0000","last_clean":"2024-05-20T20:05:37.950345+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:05:37.950345+0000","last_undegraded":"2024-05-20T20:05:37.950345+0000","last_fullsized":"2024-05-20T20:05:37.950345+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601320+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:05:37.601320+0000","last_peered":"2024-05-20T20:05:37.601320+0000","last_clean":"2024-05-20T20:05:37.601320+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:05:37.601320+0000","last_undegraded":"2024-05-20T20:05:37.601320+0000","last_fullsized":"2024-05-20T20:05:37.601320+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751407+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:05:37.751407+0000","last_peered":"2024-05-20T20:05:37.751407+0000","last_clean":"2024-05-20T20:05:37.751407+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:05:37.751407+0000","last_undegraded":"2024-05-20T20:05:37.751407+0000","last_fullsized":"2024-05-20T20:05:37.751407+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950714+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:05:37.950714+0000","last_peered":"2024-05-20T20:05:37.950714+0000","last_clean":"2024-05-20T20:05:37.950714+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:05:37.950714+0000","last_undegraded":"2024-05-20T20:05:37.950714+0000","last_fullsized":"2024-05-20T20:05:37.950714+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740756+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:05:37.740756+0000","last_peered":"2024-05-20T20:05:37.740756+0000","last_clean":"2024-05-20T20:05:37.740756+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:05:37.740756+0000","last_undegraded":"2024-05-20T20:05:37.740756+0000","last_fullsized":"2024-05-20T20:05:37.740756+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751178+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:05:37.751178+0000","last_peered":"2024-05-20T20:05:37.751178+0000","last_clean":"2024-05-20T20:05:37.751178+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:05:37.751178+0000","last_undegraded":"2024-05-20T20:05:37.751178+0000","last_fullsized":"2024-05-20T20:05:37.751178+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601398+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:05:37.601398+0000","last_peered":"2024-05-20T20:05:37.601398+0000","last_clean":"2024-05-20T20:05:37.601398+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:05:37.601398+0000","last_undegraded":"2024-05-20T20:05:37.601398+0000","last_fullsized":"2024-05-20T20:05:37.601398+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'7","reported_seq":510,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:43.038090+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:05:43.038090+0000","last_peered":"2024-05-20T20:05:43.038090+0000","last_clean":"2024-05-20T20:05:43.038090+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:05:43.038090+0000","last_undegraded":"2024-05-20T20:05:43.038090+0000","last_fullsized":"2024-05-20T20:05:43.038090+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":474,"num_read_kb":474,"num_write":8,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941795+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:05:37.941795+0000","last_peered":"2024-05-20T20:05:37.941795+0000","last_clean":"2024-05-20T20:05:37.941795+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:05:37.941795+0000","last_undegraded":"2024-05-20T20:05:37.941795+0000","last_fullsized":"2024-05-20T20:05:37.941795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947471+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:05:37.947471+0000","last_peered":"2024-05-20T20:05:37.947471+0000","last_clean":"2024-05-20T20:05:37.947471+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:05:37.947471+0000","last_undegraded":"2024-05-20T20:05:37.947471+0000","last_fullsized":"2024-05-20T20:05:37.947471+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750655+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:05:37.750655+0000","last_peered":"2024-05-20T20:05:37.750655+0000","last_clean":"2024-05-20T20:05:37.750655+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:05:37.750655+0000","last_undegraded":"2024-05-20T20:05:37.750655+0000","last_fullsized":"2024-05-20T20:05:37.750655+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751776+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:05:37.751776+0000","last_peered":"2024-05-20T20:05:37.751776+0000","last_clean":"2024-05-20T20:05:37.751776+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:05:37.751776+0000","last_undegraded":"2024-05-20T20:05:37.751776+0000","last_fullsized":"2024-05-20T20:05:37.751776+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941312+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:05:37.941312+0000","last_peered":"2024-05-20T20:05:37.941312+0000","last_clean":"2024-05-20T20:05:37.941312+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:05:37.941312+0000","last_undegraded":"2024-05-20T20:05:37.941312+0000","last_fullsized":"2024-05-20T20:05:37.941312+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'39","reported_seq":556,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:43.038486+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:05:43.038486+0000","last_peered":"2024-05-20T20:05:43.038486+0000","last_clean":"2024-05-20T20:05:43.038486+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:05:43.038486+0000","last_undegraded":"2024-05-20T20:05:43.038486+0000","last_fullsized":"2024-05-20T20:05:43.038486+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70510,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":488,"num_read_kb":313,"num_write":41,"num_write_kb":85,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950750+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:05:37.950750+0000","last_peered":"2024-05-20T20:05:37.950750+0000","last_clean":"2024-05-20T20:05:37.950750+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:05:37.950750+0000","last_undegraded":"2024-05-20T20:05:37.950750+0000","last_fullsized":"2024-05-20T20:05:37.950750+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602898+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:05:37.602898+0000","last_peered":"2024-05-20T20:05:37.602898+0000","last_clean":"2024-05-20T20:05:37.602898+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:05:37.602898+0000","last_undegraded":"2024-05-20T20:05:37.602898+0000","last_fullsized":"2024-05-20T20:05:37.602898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950828+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:05:37.950828+0000","last_peered":"2024-05-20T20:05:37.950828+0000","last_clean":"2024-05-20T20:05:37.950828+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:05:37.950828+0000","last_undegraded":"2024-05-20T20:05:37.950828+0000","last_fullsized":"2024-05-20T20:05:37.950828+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751260+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:05:37.751260+0000","last_peered":"2024-05-20T20:05:37.751260+0000","last_clean":"2024-05-20T20:05:37.751260+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:05:37.751260+0000","last_undegraded":"2024-05-20T20:05:37.751260+0000","last_fullsized":"2024-05-20T20:05:37.751260+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947376+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:05:37.947376+0000","last_peered":"2024-05-20T20:05:37.947376+0000","last_clean":"2024-05-20T20:05:37.947376+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:05:37.947376+0000","last_undegraded":"2024-05-20T20:05:37.947376+0000","last_fullsized":"2024-05-20T20:05:37.947376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'8","reported_seq":279,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.228671+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:05:42.228671+0000","last_peered":"2024-05-20T20:05:42.228671+0000","last_clean":"2024-05-20T20:05:42.228671+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:05:42.228671+0000","last_undegraded":"2024-05-20T20:05:42.228671+0000","last_fullsized":"2024-05-20T20:05:42.228671+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1680,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":243,"num_read_kb":6,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946134+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:05:37.946134+0000","last_peered":"2024-05-20T20:05:37.946134+0000","last_clean":"2024-05-20T20:05:37.946134+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:05:37.946134+0000","last_undegraded":"2024-05-20T20:05:37.946134+0000","last_fullsized":"2024-05-20T20:05:37.946134+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":71,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950299+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:05:37.950299+0000","last_peered":"2024-05-20T20:05:37.950299+0000","last_clean":"2024-05-20T20:05:37.950299+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:05:37.950299+0000","last_undegraded":"2024-05-20T20:05:37.950299+0000","last_fullsized":"2024-05-20T20:05:37.950299+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751376+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:05:37.751376+0000","last_peered":"2024-05-20T20:05:37.751376+0000","last_clean":"2024-05-20T20:05:37.751376+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:05:37.751376+0000","last_undegraded":"2024-05-20T20:05:37.751376+0000","last_fullsized":"2024-05-20T20:05:37.751376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"80'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950961+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:05:37.950961+0000","last_peered":"2024-05-20T20:05:37.950961+0000","last_clean":"2024-05-20T20:05:37.950961+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:05:37.950961+0000","last_undegraded":"2024-05-20T20:05:37.950961+0000","last_fullsized":"2024-05-20T20:05:37.950961+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603114+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:05:37.603114+0000","last_peered":"2024-05-20T20:05:37.603114+0000","last_clean":"2024-05-20T20:05:37.603114+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:05:37.603114+0000","last_undegraded":"2024-05-20T20:05:37.603114+0000","last_fullsized":"2024-05-20T20:05:37.603114+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602390+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:05:37.602390+0000","last_peered":"2024-05-20T20:05:37.602390+0000","last_clean":"2024-05-20T20:05:37.602390+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:05:37.602390+0000","last_undegraded":"2024-05-20T20:05:37.602390+0000","last_fullsized":"2024-05-20T20:05:37.602390+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601741+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:05:37.601741+0000","last_peered":"2024-05-20T20:05:37.601741+0000","last_clean":"2024-05-20T20:05:37.601741+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:05:37.601741+0000","last_undegraded":"2024-05-20T20:05:37.601741+0000","last_fullsized":"2024-05-20T20:05:37.601741+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'6","reported_seq":270,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.221448+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:05:38.221448+0000","last_peered":"2024-05-20T20:05:38.221448+0000","last_clean":"2024-05-20T20:05:38.221448+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:05:38.221448+0000","last_undegraded":"2024-05-20T20:05:38.221448+0000","last_fullsized":"2024-05-20T20:05:38.221448+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":234,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601417+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:05:37.601417+0000","last_peered":"2024-05-20T20:05:37.601417+0000","last_clean":"2024-05-20T20:05:37.601417+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:05:37.601417+0000","last_undegraded":"2024-05-20T20:05:37.601417+0000","last_fullsized":"2024-05-20T20:05:37.601417+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751541+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:05:37.751541+0000","last_peered":"2024-05-20T20:05:37.751541+0000","last_clean":"2024-05-20T20:05:37.751541+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:05:37.751541+0000","last_undegraded":"2024-05-20T20:05:37.751541+0000","last_fullsized":"2024-05-20T20:05:37.751541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"80'6","reported_seq":269,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.221082+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:05:38.221082+0000","last_peered":"2024-05-20T20:05:38.221082+0000","last_clean":"2024-05-20T20:05:38.221082+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:05:38.221082+0000","last_undegraded":"2024-05-20T20:05:38.221082+0000","last_fullsized":"2024-05-20T20:05:38.221082+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":233,"num_read_kb":233,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741186+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:05:37.741186+0000","last_peered":"2024-05-20T20:05:37.741186+0000","last_clean":"2024-05-20T20:05:37.741186+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:05:37.741186+0000","last_undegraded":"2024-05-20T20:05:37.741186+0000","last_fullsized":"2024-05-20T20:05:37.741186+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.945964+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:05:37.945964+0000","last_peered":"2024-05-20T20:05:37.945964+0000","last_clean":"2024-05-20T20:05:37.945964+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:05:37.945964+0000","last_undegraded":"2024-05-20T20:05:37.945964+0000","last_fullsized":"2024-05-20T20:05:37.945964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"20'92","reported_seq":103,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946374+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:05:37.946374+0000","last_peered":"2024-05-20T20:05:37.946374+0000","last_clean":"2024-05-20T20:05:37.946374+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:05:37.946374+0000","last_undegraded":"2024-05-20T20:05:37.946374+0000","last_fullsized":"2024-05-20T20:05:37.946374+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950636+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:05:37.950636+0000","last_peered":"2024-05-20T20:05:37.950636+0000","last_clean":"2024-05-20T20:05:37.950636+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:05:37.950636+0000","last_undegraded":"2024-05-20T20:05:37.950636+0000","last_fullsized":"2024-05-20T20:05:37.950636+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946570+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:05:37.946570+0000","last_peered":"2024-05-20T20:05:37.946570+0000","last_clean":"2024-05-20T20:05:37.946570+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:05:37.946570+0000","last_undegraded":"2024-05-20T20:05:37.946570+0000","last_fullsized":"2024-05-20T20:05:37.946570+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950840+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:05:37.950840+0000","last_peered":"2024-05-20T20:05:37.950840+0000","last_clean":"2024-05-20T20:05:37.950840+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:05:37.950840+0000","last_undegraded":"2024-05-20T20:05:37.950840+0000","last_fullsized":"2024-05-20T20:05:37.950840+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952108+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:05:37.952108+0000","last_peered":"2024-05-20T20:05:37.952108+0000","last_clean":"2024-05-20T20:05:37.952108+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:05:37.952108+0000","last_undegraded":"2024-05-20T20:05:37.952108+0000","last_fullsized":"2024-05-20T20:05:37.952108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'51","reported_seq":120,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.896768+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:05:37.896768+0000","last_peered":"2024-05-20T20:05:37.896768+0000","last_clean":"2024-05-20T20:05:37.896768+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:05:37.896768+0000","last_undegraded":"2024-05-20T20:05:37.896768+0000","last_fullsized":"2024-05-20T20:05:37.896768+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":51,"log_dups_size":0,"ondisk_log_size":51,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":91,"num_write_kb":838,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741235+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:05:37.741235+0000","last_peered":"2024-05-20T20:05:37.741235+0000","last_clean":"2024-05-20T20:05:37.741235+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:05:37.741235+0000","last_undegraded":"2024-05-20T20:05:37.741235+0000","last_fullsized":"2024-05-20T20:05:37.741235+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946101+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:05:37.946101+0000","last_peered":"2024-05-20T20:05:37.946101+0000","last_clean":"2024-05-20T20:05:37.946101+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:05:37.946101+0000","last_undegraded":"2024-05-20T20:05:37.946101+0000","last_fullsized":"2024-05-20T20:05:37.946101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750578+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:05:37.750578+0000","last_peered":"2024-05-20T20:05:37.750578+0000","last_clean":"2024-05-20T20:05:37.750578+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:05:37.750578+0000","last_undegraded":"2024-05-20T20:05:37.750578+0000","last_fullsized":"2024-05-20T20:05:37.750578+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941387+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:05:37.941387+0000","last_peered":"2024-05-20T20:05:37.941387+0000","last_clean":"2024-05-20T20:05:37.941387+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:05:37.941387+0000","last_undegraded":"2024-05-20T20:05:37.941387+0000","last_fullsized":"2024-05-20T20:05:37.941387+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":25,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950298+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:05:37.950298+0000","last_peered":"2024-05-20T20:05:37.950298+0000","last_clean":"2024-05-20T20:05:37.950298+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:05:37.950298+0000","last_undegraded":"2024-05-20T20:05:37.950298+0000","last_fullsized":"2024-05-20T20:05:37.950298+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740964+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:05:37.740964+0000","last_peered":"2024-05-20T20:05:37.740964+0000","last_clean":"2024-05-20T20:05:37.740964+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:05:37.740964+0000","last_undegraded":"2024-05-20T20:05:37.740964+0000","last_fullsized":"2024-05-20T20:05:37.740964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602823+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:05:37.602823+0000","last_peered":"2024-05-20T20:05:37.602823+0000","last_clean":"2024-05-20T20:05:37.602823+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:05:37.602823+0000","last_undegraded":"2024-05-20T20:05:37.602823+0000","last_fullsized":"2024-05-20T20:05:37.602823+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940442+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:05:37.940442+0000","last_peered":"2024-05-20T20:05:37.940442+0000","last_clean":"2024-05-20T20:05:37.940442+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:05:37.940442+0000","last_undegraded":"2024-05-20T20:05:37.940442+0000","last_fullsized":"2024-05-20T20:05:37.940442+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741376+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:05:37.741376+0000","last_peered":"2024-05-20T20:05:37.741376+0000","last_clean":"2024-05-20T20:05:37.741376+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:05:37.741376+0000","last_undegraded":"2024-05-20T20:05:37.741376+0000","last_fullsized":"2024-05-20T20:05:37.741376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950472+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:05:37.950472+0000","last_peered":"2024-05-20T20:05:37.950472+0000","last_clean":"2024-05-20T20:05:37.950472+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:05:37.950472+0000","last_undegraded":"2024-05-20T20:05:37.950472+0000","last_fullsized":"2024-05-20T20:05:37.950472+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947444+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:05:37.947444+0000","last_peered":"2024-05-20T20:05:37.947444+0000","last_clean":"2024-05-20T20:05:37.947444+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:05:37.947444+0000","last_undegraded":"2024-05-20T20:05:37.947444+0000","last_fullsized":"2024-05-20T20:05:37.947444+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'6","reported_seq":270,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.031306+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:05:38.031306+0000","last_peered":"2024-05-20T20:05:38.031306+0000","last_clean":"2024-05-20T20:05:38.031306+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:05:38.031306+0000","last_undegraded":"2024-05-20T20:05:38.031306+0000","last_fullsized":"2024-05-20T20:05:38.031306+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1561,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":235,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947761+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:05:37.947761+0000","last_peered":"2024-05-20T20:05:37.947761+0000","last_clean":"2024-05-20T20:05:37.947761+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:05:37.947761+0000","last_undegraded":"2024-05-20T20:05:37.947761+0000","last_fullsized":"2024-05-20T20:05:37.947761+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"81'41","reported_seq":78,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.890409+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:05:37.890409+0000","last_peered":"2024-05-20T20:05:37.890409+0000","last_clean":"2024-05-20T20:05:37.890409+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:05:37.890409+0000","last_undegraded":"2024-05-20T20:05:37.890409+0000","last_fullsized":"2024-05-20T20:05:37.890409+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":41,"log_dups_size":0,"ondisk_log_size":41,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947433+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:05:37.947433+0000","last_peered":"2024-05-20T20:05:37.947433+0000","last_clean":"2024-05-20T20:05:37.947433+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:05:37.947433+0000","last_undegraded":"2024-05-20T20:05:37.947433+0000","last_fullsized":"2024-05-20T20:05:37.947433+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952316+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:05:37.952316+0000","last_peered":"2024-05-20T20:05:37.952316+0000","last_clean":"2024-05-20T20:05:37.952316+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:05:37.952316+0000","last_undegraded":"2024-05-20T20:05:37.952316+0000","last_fullsized":"2024-05-20T20:05:37.952316+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947760+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:05:37.947760+0000","last_peered":"2024-05-20T20:05:37.947760+0000","last_clean":"2024-05-20T20:05:37.947760+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:05:37.947760+0000","last_undegraded":"2024-05-20T20:05:37.947760+0000","last_fullsized":"2024-05-20T20:05:37.947760+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603108+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:05:37.603108+0000","last_peered":"2024-05-20T20:05:37.603108+0000","last_clean":"2024-05-20T20:05:37.603108+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:05:37.603108+0000","last_undegraded":"2024-05-20T20:05:37.603108+0000","last_fullsized":"2024-05-20T20:05:37.603108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940537+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:05:37.940537+0000","last_peered":"2024-05-20T20:05:37.940537+0000","last_clean":"2024-05-20T20:05:37.940537+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:05:37.940537+0000","last_undegraded":"2024-05-20T20:05:37.940537+0000","last_fullsized":"2024-05-20T20:05:37.940537+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940554+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:05:37.940554+0000","last_peered":"2024-05-20T20:05:37.940554+0000","last_clean":"2024-05-20T20:05:37.940554+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:05:37.940554+0000","last_undegraded":"2024-05-20T20:05:37.940554+0000","last_fullsized":"2024-05-20T20:05:37.940554+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750191+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:05:37.750191+0000","last_peered":"2024-05-20T20:05:37.750191+0000","last_clean":"2024-05-20T20:05:37.750191+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:05:37.750191+0000","last_undegraded":"2024-05-20T20:05:37.750191+0000","last_fullsized":"2024-05-20T20:05:37.750191+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952180+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:05:37.952180+0000","last_peered":"2024-05-20T20:05:37.952180+0000","last_clean":"2024-05-20T20:05:37.952180+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:05:37.952180+0000","last_undegraded":"2024-05-20T20:05:37.952180+0000","last_fullsized":"2024-05-20T20:05:37.952180+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603150+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:05:37.603150+0000","last_peered":"2024-05-20T20:05:37.603150+0000","last_clean":"2024-05-20T20:05:37.603150+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:05:37.603150+0000","last_undegraded":"2024-05-20T20:05:37.603150+0000","last_fullsized":"2024-05-20T20:05:37.603150+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940773+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:05:37.940773+0000","last_peered":"2024-05-20T20:05:37.940773+0000","last_clean":"2024-05-20T20:05:37.940773+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:05:37.940773+0000","last_undegraded":"2024-05-20T20:05:37.940773+0000","last_fullsized":"2024-05-20T20:05:37.940773+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":25,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741108+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:05:37.741108+0000","last_peered":"2024-05-20T20:05:37.741108+0000","last_clean":"2024-05-20T20:05:37.741108+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:05:37.741108+0000","last_undegraded":"2024-05-20T20:05:37.741108+0000","last_fullsized":"2024-05-20T20:05:37.741108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751255+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:05:37.751255+0000","last_peered":"2024-05-20T20:05:37.751255+0000","last_clean":"2024-05-20T20:05:37.751255+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:05:37.751255+0000","last_undegraded":"2024-05-20T20:05:37.751255+0000","last_fullsized":"2024-05-20T20:05:37.751255+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740671+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:05:37.740671+0000","last_peered":"2024-05-20T20:05:37.740671+0000","last_clean":"2024-05-20T20:05:37.740671+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:05:37.740671+0000","last_undegraded":"2024-05-20T20:05:37.740671+0000","last_fullsized":"2024-05-20T20:05:37.740671+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941782+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:05:37.941782+0000","last_peered":"2024-05-20T20:05:37.941782+0000","last_clean":"2024-05-20T20:05:37.941782+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:05:37.941782+0000","last_undegraded":"2024-05-20T20:05:37.941782+0000","last_fullsized":"2024-05-20T20:05:37.941782+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951595+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:05:37.951595+0000","last_peered":"2024-05-20T20:05:37.951595+0000","last_clean":"2024-05-20T20:05:37.951595+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:05:37.951595+0000","last_undegraded":"2024-05-20T20:05:37.951595+0000","last_fullsized":"2024-05-20T20:05:37.951595+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602929+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:05:37.602929+0000","last_peered":"2024-05-20T20:05:37.602929+0000","last_clean":"2024-05-20T20:05:37.602929+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:05:37.602929+0000","last_undegraded":"2024-05-20T20:05:37.602929+0000","last_fullsized":"2024-05-20T20:05:37.602929+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":81,"num_read_kb":69,"num_write":123,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":108381,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":92,"ondisk_log_size":92,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":604545,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":2494,"num_read_kb":1368,"num_write":378,"num_write_kb":1470,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":729088,"data_stored":1807422,"data_compressed":75906,"data_compressed_allocated":98304,"data_compressed_original":1548288,"omap_allocated":0,"internal_metadata":0},"log_size":304,"ondisk_log_size":304,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710789,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":917504,"data_stored":828458,"data_compressed":27826,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877907018,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28060,"kb_used_data":1044,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717092,"statfs":{"total":95995035648,"available":95966302208,"internally_reserved":0,"allocated":1069056,"data_stored":1226494,"data_compressed":50963,"data_compressed_allocated":65536,"data_compressed_original":737280,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070542,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32092,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713060,"statfs":{"total":95995035648,"available":95962173440,"internally_reserved":0,"allocated":933888,"data_stored":827929,"data_compressed":27826,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266772,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":905216,"data_stored":435091,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863463002,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32116,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713036,"statfs":{"total":95995035648,"available":95962148864,"internally_reserved":0,"allocated":958464,"data_stored":1029233,"data_compressed":44807,"data_compressed_allocated":57344,"data_compressed_original":614400,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626528,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":892928,"data_stored":434821,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574972,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":946176,"data_stored":489281,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738498,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":426813,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411447,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":928,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":950272,"data_stored":489853,"data_compressed":5788,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182513,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27948,"kb_used_data":932,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717204,"statfs":{"total":95995035648,"available":95966416896,"internally_reserved":0,"allocated":954368,"data_stored":959071,"data_compressed":33982,"data_compressed_allocated":45056,"data_compressed_original":540672,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986285,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27900,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717252,"statfs":{"total":95995035648,"available":95966466048,"internally_reserved":0,"allocated":901120,"data_stored":435495,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822759,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32144,"kb_used_data":968,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713008,"statfs":{"total":95995035648,"available":95962120192,"internally_reserved":0,"allocated":991232,"data_stored":1159419,"data_compressed":50963,"data_compressed_allocated":65536,"data_compressed_original":737280,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":69742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":70705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":73728,"data_stored":525246,"data_compressed":25302,"data_compressed_allocated":32768,"data_compressed_original":516096,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":135343,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":394338,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":204938,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394867,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:05:45.643 INFO:tasks.fwd_scrub.ceph_manager:clean! 2024-05-20T20:05:45.643 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-05-20T20:05:45.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:45 smithi028 ceph-mon[31178]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.824 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.824 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.824 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:45.824 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:45.824 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:45.824 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.825 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.825 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.825 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.825 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:45 smithi154 ceph-mon[36314]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.864+0000 7fe1f8c00640 1 -- 172.21.15.28:0/2042516922 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4075740 msgr2=0x7fe1f410ebd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.864+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/2042516922 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4075740 0x7fe1f410ebd0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fe1e0009860 tx=0x7fe1e002a3d0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.864+0000 7fe1f8c00640 1 -- 172.21.15.28:0/2042516922 shutdown_connections 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.864+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/2042516922 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4075740 0x7fe1f410ebd0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.864+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/2042516922 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe1f410f110 0x7fe1f4111520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.864+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/2042516922 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4074e20 0x7fe1f4075200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.864+0000 7fe1f8c00640 1 -- 172.21.15.28:0/2042516922 >> 172.21.15.28:0/2042516922 conn(0x7fe1f40fe380 msgr2=0x7fe1f41007a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.865+0000 7fe1f8c00640 1 -- 172.21.15.28:0/2042516922 shutdown_connections 2024-05-20T20:05:45.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.865+0000 7fe1f8c00640 1 -- 172.21.15.28:0/2042516922 wait complete. 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 Processor -- start 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 -- start start 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4074e20 0x7fe1f419b350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4075740 0x7fe1f419b890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe1f410f110 0x7fe1f419fc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe1f4114d90 con 0x7fe1f410f110 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe1f4114c10 con 0x7fe1f4074e20 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.866+0000 7fe1f8c00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe1f4114f10 con 0x7fe1f4075740 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4075740 0x7fe1f419b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4075740 0x7fe1f419b890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55650/0 (socket says 172.21.15.28:55650) 2024-05-20T20:05:45.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 -- 172.21.15.28:0/769106337 learned_addr learned my addr 172.21.15.28:0/769106337 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f3e00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe1f410f110 0x7fe1f419fc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f3400640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4074e20 0x7fe1f419b350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 -- 172.21.15.28:0/769106337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4074e20 msgr2=0x7fe1f419b350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4074e20 0x7fe1f419b350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 -- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe1f410f110 msgr2=0x7fe1f419fc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe1f410f110 0x7fe1f419fc10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 -- 172.21.15.28:0/769106337 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe1f41a0340 con 0x7fe1f4075740 2024-05-20T20:05:45.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f3400640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4074e20 0x7fe1f419b350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:45.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.867+0000 7fe1f2a00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4075740 0x7fe1f419b890 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fe1e0009980 tx=0x7fe1e0030fa0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:45.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.868+0000 7fe1ebe00640 1 -- 172.21.15.28:0/769106337 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1e0040070 con 0x7fe1f4075740 2024-05-20T20:05:45.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.868+0000 7fe1ebe00640 1 -- 172.21.15.28:0/769106337 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe1e0005570 con 0x7fe1f4075740 2024-05-20T20:05:45.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.868+0000 7fe1ebe00640 1 -- 172.21.15.28:0/769106337 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1e00046c0 con 0x7fe1f4075740 2024-05-20T20:05:45.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.868+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe1f41a05d0 con 0x7fe1f4075740 2024-05-20T20:05:45.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.868+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe1f41a7ec0 con 0x7fe1f4075740 2024-05-20T20:05:45.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.869+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe1f40764d0 con 0x7fe1f4075740 2024-05-20T20:05:45.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.871+0000 7fe1ebe00640 1 -- 172.21.15.28:0/769106337 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7fe1e0043070 con 0x7fe1f4075740 2024-05-20T20:05:45.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.873+0000 7fe1ebe00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1d80b2a80 0x7fe1d80b4f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:45.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.873+0000 7fe1ebe00640 1 -- 172.21.15.28:0/769106337 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fe1e01032e0 con 0x7fe1f4075740 2024-05-20T20:05:45.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.873+0000 7fe1f3400640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1d80b2a80 0x7fe1d80b4f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:45.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.874+0000 7fe1f3400640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1d80b2a80 0x7fe1d80b4f40 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fe1e4009fc0 tx=0x7fe1e4005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: Updating smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: Updating smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:45.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:45 smithi033 ceph-mon[36179]: Updating smithi028:/etc/ceph/ceph.client.admin.keyring 2024-05-20T20:05:45.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:45.877+0000 7fe1ebe00640 1 -- 172.21.15.28:0/769106337 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fe1e00ca530 con 0x7fe1f4075740 2024-05-20T20:05:46.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.090+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fe1f4062e90 con 0x7fe1d80b2a80 2024-05-20T20:05:46.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.112+0000 7fe1ebe00640 1 -- 172.21.15.28:0/769106337 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258064 (secure 0 0 0) 0x7fe1f4062e90 con 0x7fe1d80b2a80 2024-05-20T20:05:46.117 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:46.118 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:05:46.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1d80b2a80 msgr2=0x7fe1d80b4f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1d80b2a80 0x7fe1d80b4f40 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fe1e4009fc0 tx=0x7fe1e4005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4075740 msgr2=0x7fe1f419b890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4075740 0x7fe1f419b890 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fe1e0009980 tx=0x7fe1e0030fa0 comp rx=0 tx=0).stop 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 shutdown_connections 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1d80b2a80 0x7fe1d80b4f40 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe1f410f110 0x7fe1f419fc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe1f4075740 0x7fe1f419b890 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 --2- 172.21.15.28:0/769106337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe1f4074e20 0x7fe1f419b350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 >> 172.21.15.28:0/769106337 conn(0x7fe1f40fe380 msgr2=0x7fe1f40ff0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 shutdown_connections 2024-05-20T20:05:46.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:46.117+0000 7fe1f8c00640 1 -- 172.21.15.28:0/769106337 wait complete. 2024-05-20T20:05:46.137 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":7,"stamp":"2024-05-20T20:05:45.628284+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1195147,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":2658,"num_read_kb":1626,"num_write":523,"num_write_kb":3626,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":909,"ondisk_log_size":909,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347800,"kb_used_data":11100,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124594024,"statfs":{"total":1151940427776,"available":1151584280576,"internally_reserved":0,"allocated":11366400,"data_stored":8780973,"data_compressed":288447,"data_compressed_allocated":380928,"data_compressed_original":3612672,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3302,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":116,"num_read_kb":209,"num_write":23,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.982709"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603392+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:05:37.603392+0000","last_peered":"2024-05-20T20:05:37.603392+0000","last_clean":"2024-05-20T20:05:37.603392+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:05:37.603392+0000","last_undegraded":"2024-05-20T20:05:37.603392+0000","last_fullsized":"2024-05-20T20:05:37.603392+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750621+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:05:37.750621+0000","last_peered":"2024-05-20T20:05:37.750621+0000","last_clean":"2024-05-20T20:05:37.750621+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:05:37.750621+0000","last_undegraded":"2024-05-20T20:05:37.750621+0000","last_fullsized":"2024-05-20T20:05:37.750621+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750419+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:05:37.750419+0000","last_peered":"2024-05-20T20:05:37.750419+0000","last_clean":"2024-05-20T20:05:37.750419+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:05:37.750419+0000","last_undegraded":"2024-05-20T20:05:37.750419+0000","last_fullsized":"2024-05-20T20:05:37.750419+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946345+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:05:37.946345+0000","last_peered":"2024-05-20T20:05:37.946345+0000","last_clean":"2024-05-20T20:05:37.946345+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:05:37.946345+0000","last_undegraded":"2024-05-20T20:05:37.946345+0000","last_fullsized":"2024-05-20T20:05:37.946345+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":27,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950341+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:05:37.950341+0000","last_peered":"2024-05-20T20:05:37.950341+0000","last_clean":"2024-05-20T20:05:37.950341+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:05:37.950341+0000","last_undegraded":"2024-05-20T20:05:37.950341+0000","last_fullsized":"2024-05-20T20:05:37.950341+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601481+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:05:37.601481+0000","last_peered":"2024-05-20T20:05:37.601481+0000","last_clean":"2024-05-20T20:05:37.601481+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:05:37.601481+0000","last_undegraded":"2024-05-20T20:05:37.601481+0000","last_fullsized":"2024-05-20T20:05:37.601481+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946714+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:05:37.946714+0000","last_peered":"2024-05-20T20:05:37.946714+0000","last_clean":"2024-05-20T20:05:37.946714+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:05:37.946714+0000","last_undegraded":"2024-05-20T20:05:37.946714+0000","last_fullsized":"2024-05-20T20:05:37.946714+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740698+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:05:37.740698+0000","last_peered":"2024-05-20T20:05:37.740698+0000","last_clean":"2024-05-20T20:05:37.740698+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:05:37.740698+0000","last_undegraded":"2024-05-20T20:05:37.740698+0000","last_fullsized":"2024-05-20T20:05:37.740698+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.942016+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:05:37.942016+0000","last_peered":"2024-05-20T20:05:37.942016+0000","last_clean":"2024-05-20T20:05:37.942016+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:05:37.942016+0000","last_undegraded":"2024-05-20T20:05:37.942016+0000","last_fullsized":"2024-05-20T20:05:37.942016+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":45,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601908+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:05:37.601908+0000","last_peered":"2024-05-20T20:05:37.601908+0000","last_clean":"2024-05-20T20:05:37.601908+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:05:37.601908+0000","last_undegraded":"2024-05-20T20:05:37.601908+0000","last_fullsized":"2024-05-20T20:05:37.601908+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947610+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:05:37.947610+0000","last_peered":"2024-05-20T20:05:37.947610+0000","last_clean":"2024-05-20T20:05:37.947610+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:05:37.947610+0000","last_undegraded":"2024-05-20T20:05:37.947610+0000","last_fullsized":"2024-05-20T20:05:37.947610+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946190+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:05:37.946190+0000","last_peered":"2024-05-20T20:05:37.946190+0000","last_clean":"2024-05-20T20:05:37.946190+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:05:37.946190+0000","last_undegraded":"2024-05-20T20:05:37.946190+0000","last_fullsized":"2024-05-20T20:05:37.946190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951933+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:05:37.951933+0000","last_peered":"2024-05-20T20:05:37.951933+0000","last_clean":"2024-05-20T20:05:37.951933+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:05:37.951933+0000","last_undegraded":"2024-05-20T20:05:37.951933+0000","last_fullsized":"2024-05-20T20:05:37.951933+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749940+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:05:37.749940+0000","last_peered":"2024-05-20T20:05:37.749940+0000","last_clean":"2024-05-20T20:05:37.749940+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:05:37.749940+0000","last_undegraded":"2024-05-20T20:05:37.749940+0000","last_fullsized":"2024-05-20T20:05:37.749940+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749763+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:05:37.749763+0000","last_peered":"2024-05-20T20:05:37.749763+0000","last_clean":"2024-05-20T20:05:37.749763+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:05:37.749763+0000","last_undegraded":"2024-05-20T20:05:37.749763+0000","last_fullsized":"2024-05-20T20:05:37.749763+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603526+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:05:37.603526+0000","last_peered":"2024-05-20T20:05:37.603526+0000","last_clean":"2024-05-20T20:05:37.603526+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:05:37.603526+0000","last_undegraded":"2024-05-20T20:05:37.603526+0000","last_fullsized":"2024-05-20T20:05:37.603526+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602253+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:05:37.602253+0000","last_peered":"2024-05-20T20:05:37.602253+0000","last_clean":"2024-05-20T20:05:37.602253+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:05:37.602253+0000","last_undegraded":"2024-05-20T20:05:37.602253+0000","last_fullsized":"2024-05-20T20:05:37.602253+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940646+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:05:37.940646+0000","last_peered":"2024-05-20T20:05:37.940646+0000","last_clean":"2024-05-20T20:05:37.940646+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:05:37.940646+0000","last_undegraded":"2024-05-20T20:05:37.940646+0000","last_fullsized":"2024-05-20T20:05:37.940646+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'14","reported_seq":282,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.227223+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:05:42.227223+0000","last_peered":"2024-05-20T20:05:42.227223+0000","last_clean":"2024-05-20T20:05:42.227223+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:05:42.227223+0000","last_undegraded":"2024-05-20T20:05:42.227223+0000","last_fullsized":"2024-05-20T20:05:42.227223+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":238,"num_read_kb":238,"num_write":15,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741585+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:05:37.741585+0000","last_peered":"2024-05-20T20:05:37.741585+0000","last_clean":"2024-05-20T20:05:37.741585+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:05:37.741585+0000","last_undegraded":"2024-05-20T20:05:37.741585+0000","last_fullsized":"2024-05-20T20:05:37.741585+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741764+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:05:37.741764+0000","last_peered":"2024-05-20T20:05:37.741764+0000","last_clean":"2024-05-20T20:05:37.741764+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:05:37.741764+0000","last_undegraded":"2024-05-20T20:05:37.741764+0000","last_fullsized":"2024-05-20T20:05:37.741764+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750654+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:05:37.750654+0000","last_peered":"2024-05-20T20:05:37.750654+0000","last_clean":"2024-05-20T20:05:37.750654+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:05:37.750654+0000","last_undegraded":"2024-05-20T20:05:37.750654+0000","last_fullsized":"2024-05-20T20:05:37.750654+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'13","reported_seq":43,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950989+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:05:37.950989+0000","last_peered":"2024-05-20T20:05:37.950989+0000","last_clean":"2024-05-20T20:05:37.950989+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:05:37.950989+0000","last_undegraded":"2024-05-20T20:05:37.950989+0000","last_fullsized":"2024-05-20T20:05:37.950989+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":18,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950749+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:05:37.950749+0000","last_peered":"2024-05-20T20:05:37.950749+0000","last_clean":"2024-05-20T20:05:37.950749+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:05:37.950749+0000","last_undegraded":"2024-05-20T20:05:37.950749+0000","last_fullsized":"2024-05-20T20:05:37.950749+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602506+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:05:37.602506+0000","last_peered":"2024-05-20T20:05:37.602506+0000","last_clean":"2024-05-20T20:05:37.602506+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:05:37.602506+0000","last_undegraded":"2024-05-20T20:05:37.602506+0000","last_fullsized":"2024-05-20T20:05:37.602506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"81'8","reported_seq":279,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.037384+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:05:42.037384+0000","last_peered":"2024-05-20T20:05:42.037384+0000","last_clean":"2024-05-20T20:05:42.037384+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:05:42.037384+0000","last_undegraded":"2024-05-20T20:05:42.037384+0000","last_fullsized":"2024-05-20T20:05:42.037384+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1545,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":242,"num_read_kb":5,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950345+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:05:37.950345+0000","last_peered":"2024-05-20T20:05:37.950345+0000","last_clean":"2024-05-20T20:05:37.950345+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:05:37.950345+0000","last_undegraded":"2024-05-20T20:05:37.950345+0000","last_fullsized":"2024-05-20T20:05:37.950345+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601320+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:05:37.601320+0000","last_peered":"2024-05-20T20:05:37.601320+0000","last_clean":"2024-05-20T20:05:37.601320+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:05:37.601320+0000","last_undegraded":"2024-05-20T20:05:37.601320+0000","last_fullsized":"2024-05-20T20:05:37.601320+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751407+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:05:37.751407+0000","last_peered":"2024-05-20T20:05:37.751407+0000","last_clean":"2024-05-20T20:05:37.751407+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:05:37.751407+0000","last_undegraded":"2024-05-20T20:05:37.751407+0000","last_fullsized":"2024-05-20T20:05:37.751407+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950714+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:05:37.950714+0000","last_peered":"2024-05-20T20:05:37.950714+0000","last_clean":"2024-05-20T20:05:37.950714+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:05:37.950714+0000","last_undegraded":"2024-05-20T20:05:37.950714+0000","last_fullsized":"2024-05-20T20:05:37.950714+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740756+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:05:37.740756+0000","last_peered":"2024-05-20T20:05:37.740756+0000","last_clean":"2024-05-20T20:05:37.740756+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:05:37.740756+0000","last_undegraded":"2024-05-20T20:05:37.740756+0000","last_fullsized":"2024-05-20T20:05:37.740756+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751178+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:05:37.751178+0000","last_peered":"2024-05-20T20:05:37.751178+0000","last_clean":"2024-05-20T20:05:37.751178+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:05:37.751178+0000","last_undegraded":"2024-05-20T20:05:37.751178+0000","last_fullsized":"2024-05-20T20:05:37.751178+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601398+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:05:37.601398+0000","last_peered":"2024-05-20T20:05:37.601398+0000","last_clean":"2024-05-20T20:05:37.601398+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:05:37.601398+0000","last_undegraded":"2024-05-20T20:05:37.601398+0000","last_fullsized":"2024-05-20T20:05:37.601398+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'7","reported_seq":510,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:43.038090+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:05:43.038090+0000","last_peered":"2024-05-20T20:05:43.038090+0000","last_clean":"2024-05-20T20:05:43.038090+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:05:43.038090+0000","last_undegraded":"2024-05-20T20:05:43.038090+0000","last_fullsized":"2024-05-20T20:05:43.038090+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":474,"num_read_kb":474,"num_write":8,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941795+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:05:37.941795+0000","last_peered":"2024-05-20T20:05:37.941795+0000","last_clean":"2024-05-20T20:05:37.941795+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:05:37.941795+0000","last_undegraded":"2024-05-20T20:05:37.941795+0000","last_fullsized":"2024-05-20T20:05:37.941795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947471+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:05:37.947471+0000","last_peered":"2024-05-20T20:05:37.947471+0000","last_clean":"2024-05-20T20:05:37.947471+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:05:37.947471+0000","last_undegraded":"2024-05-20T20:05:37.947471+0000","last_fullsized":"2024-05-20T20:05:37.947471+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750655+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:05:37.750655+0000","last_peered":"2024-05-20T20:05:37.750655+0000","last_clean":"2024-05-20T20:05:37.750655+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:05:37.750655+0000","last_undegraded":"2024-05-20T20:05:37.750655+0000","last_fullsized":"2024-05-20T20:05:37.750655+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751776+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:05:37.751776+0000","last_peered":"2024-05-20T20:05:37.751776+0000","last_clean":"2024-05-20T20:05:37.751776+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:05:37.751776+0000","last_undegraded":"2024-05-20T20:05:37.751776+0000","last_fullsized":"2024-05-20T20:05:37.751776+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941312+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:05:37.941312+0000","last_peered":"2024-05-20T20:05:37.941312+0000","last_clean":"2024-05-20T20:05:37.941312+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:05:37.941312+0000","last_undegraded":"2024-05-20T20:05:37.941312+0000","last_fullsized":"2024-05-20T20:05:37.941312+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'39","reported_seq":556,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:43.038486+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:05:43.038486+0000","last_peered":"2024-05-20T20:05:43.038486+0000","last_clean":"2024-05-20T20:05:43.038486+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:05:43.038486+0000","last_undegraded":"2024-05-20T20:05:43.038486+0000","last_fullsized":"2024-05-20T20:05:43.038486+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70510,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":488,"num_read_kb":313,"num_write":41,"num_write_kb":85,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950750+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:05:37.950750+0000","last_peered":"2024-05-20T20:05:37.950750+0000","last_clean":"2024-05-20T20:05:37.950750+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:05:37.950750+0000","last_undegraded":"2024-05-20T20:05:37.950750+0000","last_fullsized":"2024-05-20T20:05:37.950750+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602898+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:05:37.602898+0000","last_peered":"2024-05-20T20:05:37.602898+0000","last_clean":"2024-05-20T20:05:37.602898+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:05:37.602898+0000","last_undegraded":"2024-05-20T20:05:37.602898+0000","last_fullsized":"2024-05-20T20:05:37.602898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950828+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:05:37.950828+0000","last_peered":"2024-05-20T20:05:37.950828+0000","last_clean":"2024-05-20T20:05:37.950828+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:05:37.950828+0000","last_undegraded":"2024-05-20T20:05:37.950828+0000","last_fullsized":"2024-05-20T20:05:37.950828+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751260+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:05:37.751260+0000","last_peered":"2024-05-20T20:05:37.751260+0000","last_clean":"2024-05-20T20:05:37.751260+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:05:37.751260+0000","last_undegraded":"2024-05-20T20:05:37.751260+0000","last_fullsized":"2024-05-20T20:05:37.751260+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947376+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:05:37.947376+0000","last_peered":"2024-05-20T20:05:37.947376+0000","last_clean":"2024-05-20T20:05:37.947376+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:05:37.947376+0000","last_undegraded":"2024-05-20T20:05:37.947376+0000","last_fullsized":"2024-05-20T20:05:37.947376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'8","reported_seq":279,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.228671+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:05:42.228671+0000","last_peered":"2024-05-20T20:05:42.228671+0000","last_clean":"2024-05-20T20:05:42.228671+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:05:42.228671+0000","last_undegraded":"2024-05-20T20:05:42.228671+0000","last_fullsized":"2024-05-20T20:05:42.228671+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1680,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":243,"num_read_kb":6,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946134+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:05:37.946134+0000","last_peered":"2024-05-20T20:05:37.946134+0000","last_clean":"2024-05-20T20:05:37.946134+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:05:37.946134+0000","last_undegraded":"2024-05-20T20:05:37.946134+0000","last_fullsized":"2024-05-20T20:05:37.946134+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":71,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950299+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:05:37.950299+0000","last_peered":"2024-05-20T20:05:37.950299+0000","last_clean":"2024-05-20T20:05:37.950299+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:05:37.950299+0000","last_undegraded":"2024-05-20T20:05:37.950299+0000","last_fullsized":"2024-05-20T20:05:37.950299+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751376+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:05:37.751376+0000","last_peered":"2024-05-20T20:05:37.751376+0000","last_clean":"2024-05-20T20:05:37.751376+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:05:37.751376+0000","last_undegraded":"2024-05-20T20:05:37.751376+0000","last_fullsized":"2024-05-20T20:05:37.751376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.658836+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:05:38.658836+0000","last_peered":"2024-05-20T20:05:38.658836+0000","last_clean":"2024-05-20T20:05:38.658836+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:05:38.658836+0000","last_undegraded":"2024-05-20T20:05:38.658836+0000","last_fullsized":"2024-05-20T20:05:38.658836+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603114+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:05:37.603114+0000","last_peered":"2024-05-20T20:05:37.603114+0000","last_clean":"2024-05-20T20:05:37.603114+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:05:37.603114+0000","last_undegraded":"2024-05-20T20:05:37.603114+0000","last_fullsized":"2024-05-20T20:05:37.603114+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602390+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:05:37.602390+0000","last_peered":"2024-05-20T20:05:37.602390+0000","last_clean":"2024-05-20T20:05:37.602390+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:05:37.602390+0000","last_undegraded":"2024-05-20T20:05:37.602390+0000","last_fullsized":"2024-05-20T20:05:37.602390+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601741+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:05:37.601741+0000","last_peered":"2024-05-20T20:05:37.601741+0000","last_clean":"2024-05-20T20:05:37.601741+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:05:37.601741+0000","last_undegraded":"2024-05-20T20:05:37.601741+0000","last_fullsized":"2024-05-20T20:05:37.601741+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'6","reported_seq":275,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.228369+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:05:42.228369+0000","last_peered":"2024-05-20T20:05:42.228369+0000","last_clean":"2024-05-20T20:05:42.228369+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:05:42.228369+0000","last_undegraded":"2024-05-20T20:05:42.228369+0000","last_fullsized":"2024-05-20T20:05:42.228369+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":239,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601417+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:05:37.601417+0000","last_peered":"2024-05-20T20:05:37.601417+0000","last_clean":"2024-05-20T20:05:37.601417+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:05:37.601417+0000","last_undegraded":"2024-05-20T20:05:37.601417+0000","last_fullsized":"2024-05-20T20:05:37.601417+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751541+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:05:37.751541+0000","last_peered":"2024-05-20T20:05:37.751541+0000","last_clean":"2024-05-20T20:05:37.751541+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:05:37.751541+0000","last_undegraded":"2024-05-20T20:05:37.751541+0000","last_fullsized":"2024-05-20T20:05:37.751541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"80'6","reported_seq":274,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.227823+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:05:42.227823+0000","last_peered":"2024-05-20T20:05:42.227823+0000","last_clean":"2024-05-20T20:05:42.227823+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:05:42.227823+0000","last_undegraded":"2024-05-20T20:05:42.227823+0000","last_fullsized":"2024-05-20T20:05:42.227823+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":238,"num_read_kb":238,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741186+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:05:37.741186+0000","last_peered":"2024-05-20T20:05:37.741186+0000","last_clean":"2024-05-20T20:05:37.741186+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:05:37.741186+0000","last_undegraded":"2024-05-20T20:05:37.741186+0000","last_fullsized":"2024-05-20T20:05:37.741186+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.945964+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:05:37.945964+0000","last_peered":"2024-05-20T20:05:37.945964+0000","last_clean":"2024-05-20T20:05:37.945964+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:05:37.945964+0000","last_undegraded":"2024-05-20T20:05:37.945964+0000","last_fullsized":"2024-05-20T20:05:37.945964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"81'99","reported_seq":131,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:39.743703+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:05:39.743703+0000","last_peered":"2024-05-20T20:05:39.743703+0000","last_clean":"2024-05-20T20:05:39.743703+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:05:39.743703+0000","last_undegraded":"2024-05-20T20:05:39.743703+0000","last_fullsized":"2024-05-20T20:05:39.743703+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950636+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:05:37.950636+0000","last_peered":"2024-05-20T20:05:37.950636+0000","last_clean":"2024-05-20T20:05:37.950636+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:05:37.950636+0000","last_undegraded":"2024-05-20T20:05:37.950636+0000","last_fullsized":"2024-05-20T20:05:37.950636+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946570+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:05:37.946570+0000","last_peered":"2024-05-20T20:05:37.946570+0000","last_clean":"2024-05-20T20:05:37.946570+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:05:37.946570+0000","last_undegraded":"2024-05-20T20:05:37.946570+0000","last_fullsized":"2024-05-20T20:05:37.946570+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950840+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:05:37.950840+0000","last_peered":"2024-05-20T20:05:37.950840+0000","last_clean":"2024-05-20T20:05:37.950840+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:05:37.950840+0000","last_undegraded":"2024-05-20T20:05:37.950840+0000","last_fullsized":"2024-05-20T20:05:37.950840+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952108+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:05:37.952108+0000","last_peered":"2024-05-20T20:05:37.952108+0000","last_clean":"2024-05-20T20:05:37.952108+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:05:37.952108+0000","last_undegraded":"2024-05-20T20:05:37.952108+0000","last_fullsized":"2024-05-20T20:05:37.952108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'52","reported_seq":121,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:38.634031+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:05:38.634031+0000","last_peered":"2024-05-20T20:05:38.634031+0000","last_clean":"2024-05-20T20:05:38.634031+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:05:38.634031+0000","last_undegraded":"2024-05-20T20:05:38.634031+0000","last_fullsized":"2024-05-20T20:05:38.634031+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":93,"num_write_kb":839,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741294+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:05:37.741294+0000","last_peered":"2024-05-20T20:05:37.741294+0000","last_clean":"2024-05-20T20:05:37.741294+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:05:37.741294+0000","last_undegraded":"2024-05-20T20:05:37.741294+0000","last_fullsized":"2024-05-20T20:05:37.741294+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946101+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:05:37.946101+0000","last_peered":"2024-05-20T20:05:37.946101+0000","last_clean":"2024-05-20T20:05:37.946101+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:05:37.946101+0000","last_undegraded":"2024-05-20T20:05:37.946101+0000","last_fullsized":"2024-05-20T20:05:37.946101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750609+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:05:37.750609+0000","last_peered":"2024-05-20T20:05:37.750609+0000","last_clean":"2024-05-20T20:05:37.750609+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:05:37.750609+0000","last_undegraded":"2024-05-20T20:05:37.750609+0000","last_fullsized":"2024-05-20T20:05:37.750609+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941387+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:05:37.941387+0000","last_peered":"2024-05-20T20:05:37.941387+0000","last_clean":"2024-05-20T20:05:37.941387+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:05:37.941387+0000","last_undegraded":"2024-05-20T20:05:37.941387+0000","last_fullsized":"2024-05-20T20:05:37.941387+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":26,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950366+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:05:37.950366+0000","last_peered":"2024-05-20T20:05:37.950366+0000","last_clean":"2024-05-20T20:05:37.950366+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:05:37.950366+0000","last_undegraded":"2024-05-20T20:05:37.950366+0000","last_fullsized":"2024-05-20T20:05:37.950366+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741016+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:05:37.741016+0000","last_peered":"2024-05-20T20:05:37.741016+0000","last_clean":"2024-05-20T20:05:37.741016+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:05:37.741016+0000","last_undegraded":"2024-05-20T20:05:37.741016+0000","last_fullsized":"2024-05-20T20:05:37.741016+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602823+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:05:37.602823+0000","last_peered":"2024-05-20T20:05:37.602823+0000","last_clean":"2024-05-20T20:05:37.602823+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:05:37.602823+0000","last_undegraded":"2024-05-20T20:05:37.602823+0000","last_fullsized":"2024-05-20T20:05:37.602823+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940518+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:05:37.940518+0000","last_peered":"2024-05-20T20:05:37.940518+0000","last_clean":"2024-05-20T20:05:37.940518+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:05:37.940518+0000","last_undegraded":"2024-05-20T20:05:37.940518+0000","last_fullsized":"2024-05-20T20:05:37.940518+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741376+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:05:37.741376+0000","last_peered":"2024-05-20T20:05:37.741376+0000","last_clean":"2024-05-20T20:05:37.741376+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:05:37.741376+0000","last_undegraded":"2024-05-20T20:05:37.741376+0000","last_fullsized":"2024-05-20T20:05:37.741376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950472+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:05:37.950472+0000","last_peered":"2024-05-20T20:05:37.950472+0000","last_clean":"2024-05-20T20:05:37.950472+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:05:37.950472+0000","last_undegraded":"2024-05-20T20:05:37.950472+0000","last_fullsized":"2024-05-20T20:05:37.950472+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947444+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:05:37.947444+0000","last_peered":"2024-05-20T20:05:37.947444+0000","last_clean":"2024-05-20T20:05:37.947444+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:05:37.947444+0000","last_undegraded":"2024-05-20T20:05:37.947444+0000","last_fullsized":"2024-05-20T20:05:37.947444+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'6","reported_seq":275,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:42.037171+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:05:42.037171+0000","last_peered":"2024-05-20T20:05:42.037171+0000","last_clean":"2024-05-20T20:05:42.037171+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:05:42.037171+0000","last_undegraded":"2024-05-20T20:05:42.037171+0000","last_fullsized":"2024-05-20T20:05:42.037171+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1561,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":240,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947761+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:05:37.947761+0000","last_peered":"2024-05-20T20:05:37.947761+0000","last_clean":"2024-05-20T20:05:37.947761+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:05:37.947761+0000","last_undegraded":"2024-05-20T20:05:37.947761+0000","last_fullsized":"2024-05-20T20:05:37.947761+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"81'41","reported_seq":78,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.890409+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:05:37.890409+0000","last_peered":"2024-05-20T20:05:37.890409+0000","last_clean":"2024-05-20T20:05:37.890409+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:05:37.890409+0000","last_undegraded":"2024-05-20T20:05:37.890409+0000","last_fullsized":"2024-05-20T20:05:37.890409+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":41,"log_dups_size":0,"ondisk_log_size":41,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947433+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:05:37.947433+0000","last_peered":"2024-05-20T20:05:37.947433+0000","last_clean":"2024-05-20T20:05:37.947433+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:05:37.947433+0000","last_undegraded":"2024-05-20T20:05:37.947433+0000","last_fullsized":"2024-05-20T20:05:37.947433+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952316+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:05:37.952316+0000","last_peered":"2024-05-20T20:05:37.952316+0000","last_clean":"2024-05-20T20:05:37.952316+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:05:37.952316+0000","last_undegraded":"2024-05-20T20:05:37.952316+0000","last_fullsized":"2024-05-20T20:05:37.952316+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947760+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:05:37.947760+0000","last_peered":"2024-05-20T20:05:37.947760+0000","last_clean":"2024-05-20T20:05:37.947760+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:05:37.947760+0000","last_undegraded":"2024-05-20T20:05:37.947760+0000","last_fullsized":"2024-05-20T20:05:37.947760+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603108+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:05:37.603108+0000","last_peered":"2024-05-20T20:05:37.603108+0000","last_clean":"2024-05-20T20:05:37.603108+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:05:37.603108+0000","last_undegraded":"2024-05-20T20:05:37.603108+0000","last_fullsized":"2024-05-20T20:05:37.603108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940537+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:05:37.940537+0000","last_peered":"2024-05-20T20:05:37.940537+0000","last_clean":"2024-05-20T20:05:37.940537+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:05:37.940537+0000","last_undegraded":"2024-05-20T20:05:37.940537+0000","last_fullsized":"2024-05-20T20:05:37.940537+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940554+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:05:37.940554+0000","last_peered":"2024-05-20T20:05:37.940554+0000","last_clean":"2024-05-20T20:05:37.940554+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:05:37.940554+0000","last_undegraded":"2024-05-20T20:05:37.940554+0000","last_fullsized":"2024-05-20T20:05:37.940554+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750191+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:05:37.750191+0000","last_peered":"2024-05-20T20:05:37.750191+0000","last_clean":"2024-05-20T20:05:37.750191+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:05:37.750191+0000","last_undegraded":"2024-05-20T20:05:37.750191+0000","last_fullsized":"2024-05-20T20:05:37.750191+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952180+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:05:37.952180+0000","last_peered":"2024-05-20T20:05:37.952180+0000","last_clean":"2024-05-20T20:05:37.952180+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:05:37.952180+0000","last_undegraded":"2024-05-20T20:05:37.952180+0000","last_fullsized":"2024-05-20T20:05:37.952180+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603150+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:05:37.603150+0000","last_peered":"2024-05-20T20:05:37.603150+0000","last_clean":"2024-05-20T20:05:37.603150+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:05:37.603150+0000","last_undegraded":"2024-05-20T20:05:37.603150+0000","last_fullsized":"2024-05-20T20:05:37.603150+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940832+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:05:37.940832+0000","last_peered":"2024-05-20T20:05:37.940832+0000","last_clean":"2024-05-20T20:05:37.940832+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:05:37.940832+0000","last_undegraded":"2024-05-20T20:05:37.940832+0000","last_fullsized":"2024-05-20T20:05:37.940832+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":25,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741108+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:05:37.741108+0000","last_peered":"2024-05-20T20:05:37.741108+0000","last_clean":"2024-05-20T20:05:37.741108+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:05:37.741108+0000","last_undegraded":"2024-05-20T20:05:37.741108+0000","last_fullsized":"2024-05-20T20:05:37.741108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751255+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:05:37.751255+0000","last_peered":"2024-05-20T20:05:37.751255+0000","last_clean":"2024-05-20T20:05:37.751255+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:05:37.751255+0000","last_undegraded":"2024-05-20T20:05:37.751255+0000","last_fullsized":"2024-05-20T20:05:37.751255+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740671+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:05:37.740671+0000","last_peered":"2024-05-20T20:05:37.740671+0000","last_clean":"2024-05-20T20:05:37.740671+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:05:37.740671+0000","last_undegraded":"2024-05-20T20:05:37.740671+0000","last_fullsized":"2024-05-20T20:05:37.740671+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941782+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:05:37.941782+0000","last_peered":"2024-05-20T20:05:37.941782+0000","last_clean":"2024-05-20T20:05:37.941782+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:05:37.941782+0000","last_undegraded":"2024-05-20T20:05:37.941782+0000","last_fullsized":"2024-05-20T20:05:37.941782+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951595+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:05:37.951595+0000","last_peered":"2024-05-20T20:05:37.951595+0000","last_clean":"2024-05-20T20:05:37.951595+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:05:37.951595+0000","last_undegraded":"2024-05-20T20:05:37.951595+0000","last_fullsized":"2024-05-20T20:05:37.951595+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602929+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:05:37.602929+0000","last_peered":"2024-05-20T20:05:37.602929+0000","last_clean":"2024-05-20T20:05:37.602929+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:05:37.602929+0000","last_undegraded":"2024-05-20T20:05:37.602929+0000","last_fullsized":"2024-05-20T20:05:37.602929+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":108381,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":604545,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":2514,"num_read_kb":1378,"num_write":386,"num_write_kb":1474,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":737280,"data_stored":1813635,"data_compressed":75906,"data_compressed_allocated":98304,"data_compressed_original":1548288,"omap_allocated":0,"internal_metadata":0},"log_size":308,"ondisk_log_size":308,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710789,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":917504,"data_stored":828458,"data_compressed":27826,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877907018,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28060,"kb_used_data":1044,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717092,"statfs":{"total":95995035648,"available":95966302208,"internally_reserved":0,"allocated":1069056,"data_stored":1226494,"data_compressed":50963,"data_compressed_allocated":65536,"data_compressed_original":737280,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070543,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32092,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713060,"statfs":{"total":95995035648,"available":95962173440,"internally_reserved":0,"allocated":933888,"data_stored":827929,"data_compressed":27826,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266773,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":905216,"data_stored":435091,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863463003,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32112,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713040,"statfs":{"total":95995035648,"available":95962152960,"internally_reserved":0,"allocated":958464,"data_stored":1029624,"data_compressed":44807,"data_compressed_allocated":57344,"data_compressed_original":614400,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626529,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":892928,"data_stored":434821,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574973,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27960,"kb_used_data":940,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717192,"statfs":{"total":95995035648,"available":95966404608,"internally_reserved":0,"allocated":962560,"data_stored":503724,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738499,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":892,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":913408,"data_stored":436160,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411448,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":966656,"data_stored":504687,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182514,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27948,"kb_used_data":932,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717204,"statfs":{"total":95995035648,"available":95966416896,"internally_reserved":0,"allocated":954368,"data_stored":959071,"data_compressed":33982,"data_compressed_allocated":45056,"data_compressed_original":540672,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986285,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27900,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717252,"statfs":{"total":95995035648,"available":95966466048,"internally_reserved":0,"allocated":901120,"data_stored":435495,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822759,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32144,"kb_used_data":968,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713008,"statfs":{"total":95995035648,"available":95962120192,"internally_reserved":0,"allocated":991232,"data_stored":1159419,"data_compressed":50963,"data_compressed_allocated":65536,"data_compressed_original":737280,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":72653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":73616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":73728,"data_stored":525246,"data_compressed":25302,"data_compressed_allocated":32768,"data_compressed_original":516096,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":135343,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":394338,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":204938,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394867,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:05:46.142 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:05:46.637 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='client.25216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='client.25222 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: pgmap v7: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 1.3 KiB/s wr, 19 op/s 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.785 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:46.786 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:46.786 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:46.786 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:46 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='client.25216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='client.25222 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: pgmap v7: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 1.3 KiB/s wr, 19 op/s 2024-05-20T20:05:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:46.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:46.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:46.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:46 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='client.25216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: Updating smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='client.25222 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: pgmap v7: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 1.3 KiB/s wr, 19 op/s 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:46.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:46.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:46.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:46 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:47 smithi028 ceph-mon[31178]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:47.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:47 smithi154 ceph-mon[36314]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:47.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:47 smithi033 ceph-mon[36179]: from='client.25228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:05:48.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:48 smithi028 ceph-mon[31178]: pgmap v8: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 1.0 KiB/s wr, 15 op/s 2024-05-20T20:05:48.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:48 smithi154 ceph-mon[36314]: pgmap v8: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 1.0 KiB/s wr, 15 op/s 2024-05-20T20:05:48.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:48 smithi033 ceph-mon[36179]: pgmap v8: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 1.0 KiB/s wr, 15 op/s 2024-05-20T20:05:49.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 -- 172.21.15.28:0/3847329408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2648104690 msgr2=0x7f2648104a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 --2- 172.21.15.28:0/3847329408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2648104690 0x7f2648104a70 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f2638009990 tx=0x7f2638009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 -- 172.21.15.28:0/3847329408 shutdown_connections 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 --2- 172.21.15.28:0/3847329408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f264810da70 0x7f264810ff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 --2- 172.21.15.28:0/3847329408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2648104690 0x7f2648104a70 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 --2- 172.21.15.28:0/3847329408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2648104fb0 0x7f264810d4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 -- 172.21.15.28:0/3847329408 >> 172.21.15.28:0/3847329408 conn(0x7f26480fde60 msgr2=0x7f2648100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 -- 172.21.15.28:0/3847329408 shutdown_connections 2024-05-20T20:05:49.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.121+0000 7f2647e00700 1 -- 172.21.15.28:0/3847329408 wait complete. 2024-05-20T20:05:49.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.122+0000 7f2647e00700 1 Processor -- start 2024-05-20T20:05:49.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.123+0000 7f2647e00700 1 -- start start 2024-05-20T20:05:49.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.123+0000 7f2647e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2648104690 0x7f264819be40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.123+0000 7f2647e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2648104fb0 0x7f264819c380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.123+0000 7f2647e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f264810da70 0x7f26481a0710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.123+0000 7f2647e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2648113690 con 0x7f264810da70 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.123+0000 7f2647e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2648113510 con 0x7f2648104690 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.123+0000 7f2647e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2648113810 con 0x7f2648104fb0 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2647400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f264810da70 0x7f26481a0710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2646a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2648104690 0x7f264819be40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2647400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f264810da70 0x7f26481a0710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:44570/0 (socket says 172.21.15.28:44570) 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2647400700 1 -- 172.21.15.28:0/3506903490 learned_addr learned my addr 172.21.15.28:0/3506903490 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:49.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2646000700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2648104fb0 0x7f264819c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:49.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2646a00700 1 -- 172.21.15.28:0/3506903490 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2648104fb0 msgr2=0x7f264819c380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:49.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2646a00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2648104fb0 0x7f264819c380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2646a00700 1 -- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f264810da70 msgr2=0x7f26481a0710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2646a00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f264810da70 0x7f26481a0710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.124+0000 7f2646a00700 1 -- 172.21.15.28:0/3506903490 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f26481a0e50 con 0x7f2648104690 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.125+0000 7f2646a00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2648104690 0x7f264819be40 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f26380098e0 tx=0x7f26380057c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.125+0000 7f263f400700 1 -- 172.21.15.28:0/3506903490 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2638027070 con 0x7f2648104690 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.125+0000 7f263f400700 1 -- 172.21.15.28:0/3506903490 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2638022470 con 0x7f2648104690 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.125+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f26481a10e0 con 0x7f2648104690 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.125+0000 7f263f400700 1 -- 172.21.15.28:0/3506903490 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2638017650 con 0x7f2648104690 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.125+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f26481a89c0 con 0x7f2648104690 2024-05-20T20:05:49.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.126+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f264806a230 con 0x7f2648104690 2024-05-20T20:05:49.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.129+0000 7f263f400700 1 -- 172.21.15.28:0/3506903490 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f263802a070 con 0x7f2648104690 2024-05-20T20:05:49.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.130+0000 7f263f400700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f262c0b2b10 0x7f262c0b4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:49.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.130+0000 7f263f400700 1 -- 172.21.15.28:0/3506903490 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f26380e37e0 con 0x7f2648104690 2024-05-20T20:05:49.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.131+0000 7f2646000700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f262c0b2b10 0x7f262c0b4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:49.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.131+0000 7f2646000700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f262c0b2b10 0x7f262c0b4fd0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f2634009680 tx=0x7f2634009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:49.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.133+0000 7f263f400700 1 -- 172.21.15.28:0/3506903490 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f2638031080 con 0x7f2648104690 2024-05-20T20:05:49.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.425+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f26481a8b90 con 0x7f2648104690 2024-05-20T20:05:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.426+0000 7f263f400700 1 -- 172.21.15.28:0/3506903490 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f2638013070 con 0x7f2648104690 2024-05-20T20:05:49.426 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:49.427 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:05:49.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f262c0b2b10 msgr2=0x7f262c0b4fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:49.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f262c0b2b10 0x7f262c0b4fd0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f2634009680 tx=0x7f2634009260 comp rx=0 tx=0).stop 2024-05-20T20:05:49.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2648104690 msgr2=0x7f264819be40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:49.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2648104690 0x7f264819be40 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f26380098e0 tx=0x7f26380057c0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 shutdown_connections 2024-05-20T20:05:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f264810da70 0x7f26481a0710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f262c0b2b10 0x7f262c0b4fd0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2648104690 0x7f264819be40 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 --2- 172.21.15.28:0/3506903490 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2648104fb0 0x7f264819c380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.429+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 >> 172.21.15.28:0/3506903490 conn(0x7f26480fde60 msgr2=0x7f2648100250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.430+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 shutdown_connections 2024-05-20T20:05:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:49.430+0000 7f2647e00700 1 -- 172.21.15.28:0/3506903490 wait complete. 2024-05-20T20:05:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:05:49.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:49 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3506903490' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:49.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:49 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3506903490' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:49.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:49 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3506903490' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:05:50.331 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:05:50.804 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:50.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:50 smithi028 ceph-mon[31178]: pgmap v9: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 839 B/s wr, 17 op/s 2024-05-20T20:05:50.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:50 smithi154 ceph-mon[36314]: pgmap v9: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 839 B/s wr, 17 op/s 2024-05-20T20:05:50.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:50 smithi033 ceph-mon[36179]: pgmap v9: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 839 B/s wr, 17 op/s 2024-05-20T20:05:53.009 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:52 smithi028 ceph-mon[31178]: pgmap v10: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 767 B/s wr, 15 op/s 2024-05-20T20:05:53.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:52 smithi154 ceph-mon[36314]: pgmap v10: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 767 B/s wr, 15 op/s 2024-05-20T20:05:53.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:52 smithi033 ceph-mon[36179]: pgmap v10: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 767 B/s wr, 15 op/s 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.178+0000 7fa19f000700 1 -- 172.21.15.28:0/521638405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa198076090 msgr2=0x7fa19810fa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.178+0000 7fa19f000700 1 --2- 172.21.15.28:0/521638405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa198076090 0x7fa19810fa00 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fa184009990 tx=0x7fa184009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.179+0000 7fa19f000700 1 -- 172.21.15.28:0/521638405 shutdown_connections 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.179+0000 7fa19f000700 1 --2- 172.21.15.28:0/521638405 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa19810ff40 0x7fa198112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.179+0000 7fa19f000700 1 --2- 172.21.15.28:0/521638405 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 0x7fa198075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.179+0000 7fa19f000700 1 --2- 172.21.15.28:0/521638405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa198076090 0x7fa19810fa00 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.179+0000 7fa19f000700 1 -- 172.21.15.28:0/521638405 >> 172.21.15.28:0/521638405 conn(0x7fa19807a6a0 msgr2=0x7fa19807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.179+0000 7fa19f000700 1 -- 172.21.15.28:0/521638405 shutdown_connections 2024-05-20T20:05:53.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.179+0000 7fa19f000700 1 -- 172.21.15.28:0/521638405 wait complete. 2024-05-20T20:05:53.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.180+0000 7fa19f000700 1 Processor -- start 2024-05-20T20:05:53.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.181+0000 7fa19f000700 1 -- start start 2024-05-20T20:05:53.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.181+0000 7fa19f000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 0x7fa198103660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:53.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.181+0000 7fa19f000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa198076090 0x7fa198103ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:53.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.181+0000 7fa19f000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa19810ff40 0x7fa1981040e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:53.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.181+0000 7fa19f000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa198115950 con 0x7fa19810ff40 2024-05-20T20:05:53.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.181+0000 7fa19f000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa1981157d0 con 0x7fa198076090 2024-05-20T20:05:53.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.181+0000 7fa19f000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa198115ad0 con 0x7fa198075770 2024-05-20T20:05:53.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa19d000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa19810ff40 0x7fa1981040e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:53.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 0x7fa198103660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:53.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 0x7fa198103660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38248/0 (socket says 172.21.15.28:38248) 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 -- 172.21.15.28:0/2842673303 learned_addr learned my addr 172.21.15.28:0/2842673303 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197400700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa198076090 0x7fa198103ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 -- 172.21.15.28:0/2842673303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa198076090 msgr2=0x7fa198103ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa198076090 0x7fa198103ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 -- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa19810ff40 msgr2=0x7fa1981040e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa19810ff40 0x7fa1981040e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa197e00700 1 -- 172.21.15.28:0/2842673303 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa1981ac530 con 0x7fa198075770 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.182+0000 7fa19d000700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa19810ff40 0x7fa1981040e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.183+0000 7fa197e00700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 0x7fa198103660 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fa18800e9d0 tx=0x7fa18800ece0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.183+0000 7fa194c00700 1 -- 172.21.15.28:0/2842673303 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa18800cbd0 con 0x7fa198075770 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.183+0000 7fa194c00700 1 -- 172.21.15.28:0/2842673303 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa18800ee70 con 0x7fa198075770 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.183+0000 7fa194c00700 1 -- 172.21.15.28:0/2842673303 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa188018680 con 0x7fa198075770 2024-05-20T20:05:53.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.183+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa1981ac820 con 0x7fa198075770 2024-05-20T20:05:53.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.183+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa1981acd80 con 0x7fa198075770 2024-05-20T20:05:53.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.185+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa198076e60 con 0x7fa198075770 2024-05-20T20:05:53.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.189+0000 7fa194c00700 1 -- 172.21.15.28:0/2842673303 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7fa18804ab90 con 0x7fa198075770 2024-05-20T20:05:53.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.191+0000 7fa194c00700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa17c0b2bc0 0x7fa17c0b5080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:53.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.191+0000 7fa194c00700 1 -- 172.21.15.28:0/2842673303 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa1880e2b80 con 0x7fa198075770 2024-05-20T20:05:53.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.191+0000 7fa197400700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa17c0b2bc0 0x7fa17c0b5080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:53.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.191+0000 7fa194c00700 1 -- 172.21.15.28:0/2842673303 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa1880aa2c0 con 0x7fa198075770 2024-05-20T20:05:53.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.192+0000 7fa197400700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa17c0b2bc0 0x7fa17c0b5080 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fa198104e60 tx=0x7fa184005fd0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:53.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.437+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fa19802e560 con 0x7fa198075770 2024-05-20T20:05:53.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.439+0000 7fa194c00700 1 -- 172.21.15.28:0/2842673303 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) ==== 74+0+22947 (secure 0 0 0) 0x7fa1880a9bc0 con 0x7fa198075770 2024-05-20T20:05:53.439 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:05:53.440 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":81,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:05:37.589166+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6800/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/2759010565":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/1496067961":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1975849601":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/19609679":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa17c0b2bc0 msgr2=0x7fa17c0b5080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa17c0b2bc0 0x7fa17c0b5080 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fa198104e60 tx=0x7fa184005fd0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 msgr2=0x7fa198103660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 0x7fa198103660 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fa18800e9d0 tx=0x7fa18800ece0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 shutdown_connections 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa19810ff40 0x7fa1981040e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa198075770 0x7fa198103660 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa17c0b2bc0 0x7fa17c0b5080 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 --2- 172.21.15.28:0/2842673303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa198076090 0x7fa198103ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.442+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 >> 172.21.15.28:0/2842673303 conn(0x7fa19807a6a0 msgr2=0x7fa1981007f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.443+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 shutdown_connections 2024-05-20T20:05:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:53.443+0000 7fa19f000700 1 -- 172.21.15.28:0/2842673303 wait complete. 2024-05-20T20:05:53.845 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:53 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2842673303' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:54.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:53 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2842673303' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:54.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:53 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2842673303' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:05:54.387 DEBUG:tasks.fwd_scrub:Yielding 2024-05-20T20:05:54.387 INFO:teuthology.run_tasks:Running task exec... 2024-05-20T20:05:54.399 INFO:teuthology.task.exec:Executing custom commands... 2024-05-20T20:05:54.399 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:05:54.399 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/log_level debug' 2024-05-20T20:05:54.400 INFO:tasks.fwd_scrub.fs.[cephfs]:start scrubbing fs: cephfs 2024-05-20T20:05:54.400 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:05:54.400 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 -- 172.21.15.28:0/402129332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 msgr2=0x7f4688105280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 --2- 172.21.15.28:0/402129332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 0x7f4688105280 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f4678009900 tx=0x7f467802a410 comp rx=0 tx=0).stop 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 -- 172.21.15.28:0/402129332 shutdown_connections 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 --2- 172.21.15.28:0/402129332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4688105850 0x7f4688109890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 --2- 172.21.15.28:0/402129332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 0x7f4688111b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 --2- 172.21.15.28:0/402129332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 0x7f4688105280 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 -- 172.21.15.28:0/402129332 >> 172.21.15.28:0/402129332 conn(0x7f4688100740 msgr2=0x7f4688102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.604+0000 7f468ce00640 1 -- 172.21.15.28:0/402129332 shutdown_connections 2024-05-20T20:05:54.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.605+0000 7f468ce00640 1 -- 172.21.15.28:0/402129332 wait complete. 2024-05-20T20:05:54.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 Processor -- start 2024-05-20T20:05:54.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 -- start start 2024-05-20T20:05:54.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 0x7f468819b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:54.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 0x7f468819b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:54.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f468819fc40 0x7f46881a00f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4688114ed0 con 0x7f4688104ea0 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4688114d50 con 0x7f468819fc40 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.606+0000 7f468ce00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4688115050 con 0x7f4688109f70 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 0x7f468819b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 0x7f468819b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 0x7f468819b910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38276/0 (socket says 172.21.15.28:38276) 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 -- 172.21.15.28:0/4158175930 learned_addr learned my addr 172.21.15.28:0/4158175930 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4686800640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f468819fc40 0x7f46881a00f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 -- 172.21.15.28:0/4158175930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f468819fc40 msgr2=0x7f46881a00f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f468819fc40 0x7f46881a00f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 -- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 msgr2=0x7f468819b3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 0x7f468819b3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 -- 172.21.15.28:0/4158175930 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f46881a0730 con 0x7f4688109f70 2024-05-20T20:05:54.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.607+0000 7f4685400640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 0x7f468819b910 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f467400e9c0 tx=0x7f467400ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:54.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.608+0000 7f467ea00640 1 -- 172.21.15.28:0/4158175930 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f467400cd80 con 0x7f4688109f70 2024-05-20T20:05:54.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.608+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f46881a0a20 con 0x7f4688109f70 2024-05-20T20:05:54.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.608+0000 7f467ea00640 1 -- 172.21.15.28:0/4158175930 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f46740054d0 con 0x7f4688109f70 2024-05-20T20:05:54.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.608+0000 7f467ea00640 1 -- 172.21.15.28:0/4158175930 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4674010430 con 0x7f4688109f70 2024-05-20T20:05:54.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.608+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f46881a8200 con 0x7f4688109f70 2024-05-20T20:05:54.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.609+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f46880507e0 con 0x7f4688109f70 2024-05-20T20:05:54.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.612+0000 7f467ea00640 1 -- 172.21.15.28:0/4158175930 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f4674004fd0 con 0x7f4688109f70 2024-05-20T20:05:54.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.614+0000 7f467ea00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f466c0b2b30 0x7f466c0b4ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:54.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.614+0000 7f467ea00640 1 -- 172.21.15.28:0/4158175930 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f46740e2d60 con 0x7f4688109f70 2024-05-20T20:05:54.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.614+0000 7f4685e00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f466c0b2b30 0x7f466c0b4ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:54.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.615+0000 7f467ea00640 1 -- 172.21.15.28:0/4158175930 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f46740aa4a0 con 0x7f4688109f70 2024-05-20T20:05:54.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.615+0000 7f4685e00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f466c0b2b30 0x7f466c0b4ff0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f4678031440 tx=0x7f4678005b50 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:54.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.842+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/log_level}] v 0) -- 0x7f468819c6d0 con 0x7f4688109f70 2024-05-20T20:05:54.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.850+0000 7f467ea00640 1 -- 172.21.15.28:0/4158175930 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/log_level}]=0 v26)=0 v26) ==== 129+0+0 (secure 0 0 0) 0x7f468819c6d0 con 0x7f4688109f70 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f466c0b2b30 msgr2=0x7f466c0b4ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f466c0b2b30 0x7f466c0b4ff0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f4678031440 tx=0x7f4678005b50 comp rx=0 tx=0).stop 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 msgr2=0x7f468819b910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 0x7f468819b910 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f467400e9c0 tx=0x7f467400ee80 comp rx=0 tx=0).stop 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 shutdown_connections 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f468819fc40 0x7f46881a00f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f466c0b2b30 0x7f466c0b4ff0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4688109f70 0x7f468819b910 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 --2- 172.21.15.28:0/4158175930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4688104ea0 0x7f468819b3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:54.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 >> 172.21.15.28:0/4158175930 conn(0x7f4688100740 msgr2=0x7f4688102a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:54.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.853+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 shutdown_connections 2024-05-20T20:05:54.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:54.854+0000 7f468ce00640 1 -- 172.21.15.28:0/4158175930 wait complete. 2024-05-20T20:05:54.856 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:54.874 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/allow_m_granularity true' 2024-05-20T20:05:55.011 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:54 smithi028 ceph-mon[31178]: pgmap v11: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 767 B/s wr, 18 op/s 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.066+0000 7f0c48800640 1 -- 172.21.15.28:0/3498572225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40103ce0 msgr2=0x7f0c401040c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.066+0000 7f0c48800640 1 --2- 172.21.15.28:0/3498572225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40103ce0 0x7f0c401040c0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f0c34009900 tx=0x7f0c3402a480 comp rx=0 tx=0).stop 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.067+0000 7f0c48800640 1 -- 172.21.15.28:0/3498572225 shutdown_connections 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.067+0000 7f0c48800640 1 --2- 172.21.15.28:0/3498572225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 0x7f0c4010f4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.067+0000 7f0c48800640 1 --2- 172.21.15.28:0/3498572225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c40104600 0x7f0c4010cae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.067+0000 7f0c48800640 1 --2- 172.21.15.28:0/3498572225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40103ce0 0x7f0c401040c0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.067+0000 7f0c48800640 1 -- 172.21.15.28:0/3498572225 >> 172.21.15.28:0/3498572225 conn(0x7f0c40078a60 msgr2=0x7f0c40078e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.067+0000 7f0c48800640 1 -- 172.21.15.28:0/3498572225 shutdown_connections 2024-05-20T20:05:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.067+0000 7f0c48800640 1 -- 172.21.15.28:0/3498572225 wait complete. 2024-05-20T20:05:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 Processor -- start 2024-05-20T20:05:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 -- start start 2024-05-20T20:05:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c40103ce0 0x7f0c4019b430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40104600 0x7f0c4019b970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 0x7f0c4019fcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0c40112d40 con 0x7f0c40103ce0 2024-05-20T20:05:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0c40112bc0 con 0x7f0c4010d0b0 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.068+0000 7f0c48800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0c40112ec0 con 0x7f0c40104600 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c45e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c40103ce0 0x7f0c4019b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 0x7f0c4019fcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 0x7f0c4019fcf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:50256/0 (socket says 172.21.15.28:50256) 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 -- 172.21.15.28:0/2157108735 learned_addr learned my addr 172.21.15.28:0/2157108735 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c45400640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40104600 0x7f0c4019b970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 -- 172.21.15.28:0/2157108735 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40104600 msgr2=0x7f0c4019b970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40104600 0x7f0c4019b970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 -- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c40103ce0 msgr2=0x7f0c4019b430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c40103ce0 0x7f0c4019b430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 -- 172.21.15.28:0/2157108735 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c401a0420 con 0x7f0c4010d0b0 2024-05-20T20:05:55.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.069+0000 7f0c46800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 0x7f0c4019fcf0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f0c3c00eb40 tx=0x7f0c3c00ef40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.070+0000 7f0c3aa00640 1 -- 172.21.15.28:0/2157108735 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c3c00ce40 con 0x7f0c4010d0b0 2024-05-20T20:05:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.070+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c401a0710 con 0x7f0c4010d0b0 2024-05-20T20:05:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.070+0000 7f0c3aa00640 1 -- 172.21.15.28:0/2157108735 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0c3c005020 con 0x7f0c4010d0b0 2024-05-20T20:05:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.070+0000 7f0c3aa00640 1 -- 172.21.15.28:0/2157108735 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c3c010670 con 0x7f0c4010d0b0 2024-05-20T20:05:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.070+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c401a7fa0 con 0x7f0c4010d0b0 2024-05-20T20:05:55.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.071+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c400507e0 con 0x7f0c4010d0b0 2024-05-20T20:05:55.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.073+0000 7f0c3aa00640 1 -- 172.21.15.28:0/2157108735 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f0c3c0054b0 con 0x7f0c4010d0b0 2024-05-20T20:05:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.075+0000 7f0c3aa00640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2bb0 0x7f0c280b5070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.075+0000 7f0c3aa00640 1 -- 172.21.15.28:0/2157108735 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0c3c0e2ef0 con 0x7f0c4010d0b0 2024-05-20T20:05:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.075+0000 7f0c45e00640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2bb0 0x7f0c280b5070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.076+0000 7f0c45e00640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2bb0 0x7f0c280b5070 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0c34005ab0 tx=0x7f0c34005a20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:55.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.077+0000 7f0c3aa00640 1 -- 172.21.15.28:0/2157108735 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f0c3c0aa630 con 0x7f0c4010d0b0 2024-05-20T20:05:55.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:54 smithi154 ceph-mon[36314]: pgmap v11: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 767 B/s wr, 18 op/s 2024-05-20T20:05:55.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:54 smithi033 ceph-mon[36179]: pgmap v11: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 767 B/s wr, 18 op/s 2024-05-20T20:05:55.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.298+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}] v 0) -- 0x7f0c401a8230 con 0x7f0c4010d0b0 2024-05-20T20:05:55.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.305+0000 7f0c3aa00640 1 -- 172.21.15.28:0/2157108735 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}]=0 v27)=0 v27) ==== 138+0+0 (secure 0 0 0) 0x7f0c3c0a9f30 con 0x7f0c4010d0b0 2024-05-20T20:05:55.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.308+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2bb0 msgr2=0x7f0c280b5070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.308+0000 7f0c48800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2bb0 0x7f0c280b5070 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0c34005ab0 tx=0x7f0c34005a20 comp rx=0 tx=0).stop 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.308+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 msgr2=0x7f0c4019fcf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.308+0000 7f0c48800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 0x7f0c4019fcf0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f0c3c00eb40 tx=0x7f0c3c00ef40 comp rx=0 tx=0).stop 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 shutdown_connections 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c4010d0b0 0x7f0c4019fcf0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2bb0 0x7f0c280b5070 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c40104600 0x7f0c4019b970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 --2- 172.21.15.28:0/2157108735 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c40103ce0 0x7f0c4019b430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 >> 172.21.15.28:0/2157108735 conn(0x7f0c40078a60 msgr2=0x7f0c400fe840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 shutdown_connections 2024-05-20T20:05:55.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.309+0000 7f0c48800640 1 -- 172.21.15.28:0/2157108735 wait complete. 2024-05-20T20:05:55.330 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/dump_on_update true' 2024-05-20T20:05:55.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 -- 172.21.15.28:0/2703342124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 msgr2=0x7f892c115d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 --2- 172.21.15.28:0/2703342124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c115d50 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f8924009ca0 tx=0x7f892402a8b0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 -- 172.21.15.28:0/2703342124 shutdown_connections 2024-05-20T20:05:55.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 --2- 172.21.15.28:0/2703342124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c115d50 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 --2- 172.21.15.28:0/2703342124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f892c108f90 0x7f892c1093d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 --2- 172.21.15.28:0/2703342124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f892c10ef20 0x7f892c10f300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 -- 172.21.15.28:0/2703342124 >> 172.21.15.28:0/2703342124 conn(0x7f892c078830 msgr2=0x7f892c107870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:55.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 -- 172.21.15.28:0/2703342124 shutdown_connections 2024-05-20T20:05:55.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.539+0000 7f892b200640 1 -- 172.21.15.28:0/2703342124 wait complete. 2024-05-20T20:05:55.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.540+0000 7f892b200640 1 Processor -- start 2024-05-20T20:05:55.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f892b200640 1 -- start start 2024-05-20T20:05:55.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f892b200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f892c108f90 0x7f892c19f730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f892b200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c19fc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f892b200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f892c10ef20 0x7f892c1a3ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f892b200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f892c119150 con 0x7f892c108f90 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f892b200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f892c118fd0 con 0x7f892c109910 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f892b200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f892c1192d0 con 0x7f892c10ef20 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f8929400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c19fc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.541+0000 7f8929400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c19fc70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:50276/0 (socket says 172.21.15.28:50276) 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929400640 1 -- 172.21.15.28:0/589152767 learned_addr learned my addr 172.21.15.28:0/589152767 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f892c108f90 0x7f892c19f730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f892a800640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f892c10ef20 0x7f892c1a3ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929400640 1 -- 172.21.15.28:0/589152767 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f892c10ef20 msgr2=0x7f892c1a3ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929400640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f892c10ef20 0x7f892c1a3ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929400640 1 -- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f892c108f90 msgr2=0x7f892c19f730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929400640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f892c108f90 0x7f892c19f730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929400640 1 -- 172.21.15.28:0/589152767 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f892c1a4720 con 0x7f892c109910 2024-05-20T20:05:55.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.542+0000 7f8929400640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c19fc70 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f891800e9f0 tx=0x7f891800edf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:55.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.543+0000 7f8922a00640 1 -- 172.21.15.28:0/589152767 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f891800cd80 con 0x7f892c109910 2024-05-20T20:05:55.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.543+0000 7f8922a00640 1 -- 172.21.15.28:0/589152767 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f89180054d0 con 0x7f892c109910 2024-05-20T20:05:55.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.543+0000 7f8922a00640 1 -- 172.21.15.28:0/589152767 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8918010690 con 0x7f892c109910 2024-05-20T20:05:55.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.543+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f892c1a4a10 con 0x7f892c109910 2024-05-20T20:05:55.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.543+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f892c1ac2a0 con 0x7f892c109910 2024-05-20T20:05:55.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.544+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f892c051030 con 0x7f892c109910 2024-05-20T20:05:55.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.547+0000 7f8922a00640 1 -- 172.21.15.28:0/589152767 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f8918004e90 con 0x7f892c109910 2024-05-20T20:05:55.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.549+0000 7f8922a00640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89100b2a70 0x7f89100b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:55.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.550+0000 7f8922a00640 1 -- 172.21.15.28:0/589152767 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f89180e2d10 con 0x7f892c109910 2024-05-20T20:05:55.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.550+0000 7f8929e00640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89100b2a70 0x7f89100b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:55.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.551+0000 7f8929e00640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89100b2a70 0x7f89100b4f30 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f891c009b50 tx=0x7f891c005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:55.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.554+0000 7f8922a00640 1 -- 172.21.15.28:0/589152767 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f89180e8020 con 0x7f892c109910 2024-05-20T20:05:55.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.771+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/dump_on_update}] v 0) -- 0x7f892c1113a0 con 0x7f892c109910 2024-05-20T20:05:55.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.783+0000 7f8922a00640 1 -- 172.21.15.28:0/589152767 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/dump_on_update}]=0 v28)=0 v28) ==== 133+0+0 (secure 0 0 0) 0x7f89180aa450 con 0x7f892c109910 2024-05-20T20:05:55.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89100b2a70 msgr2=0x7f89100b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89100b2a70 0x7f89100b4f30 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f891c009b50 tx=0x7f891c005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:55.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 msgr2=0x7f892c19fc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c19fc70 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f891800e9f0 tx=0x7f891800edf0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 shutdown_connections 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f892c109910 0x7f892c19fc70 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89100b2a70 0x7f89100b4f30 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f892c10ef20 0x7f892c1a3ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 --2- 172.21.15.28:0/589152767 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f892c108f90 0x7f892c19f730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.786+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 >> 172.21.15.28:0/589152767 conn(0x7f892c078830 msgr2=0x7f892c079850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.787+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 shutdown_connections 2024-05-20T20:05:55.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:55.787+0000 7f892b200640 1 -- 172.21.15.28:0/589152767 wait complete. 2024-05-20T20:05:55.808 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1m' 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.015+0000 7f3b9d200640 1 -- 172.21.15.28:0/854696478 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98102850 msgr2=0x7f3b98102cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.015+0000 7f3b9d200640 1 --2- 172.21.15.28:0/854696478 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98102850 0x7f3b98102cd0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f3b80009900 tx=0x7f3b8002a3f0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.016+0000 7f3b9d200640 1 -- 172.21.15.28:0/854696478 shutdown_connections 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.016+0000 7f3b9d200640 1 --2- 172.21.15.28:0/854696478 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98102850 0x7f3b98102cd0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.016+0000 7f3b9d200640 1 --2- 172.21.15.28:0/854696478 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 0x7f3b9810f720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.016+0000 7f3b9d200640 1 --2- 172.21.15.28:0/854696478 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b98108860 0x7f3b98108c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.016+0000 7f3b9d200640 1 -- 172.21.15.28:0/854696478 >> 172.21.15.28:0/854696478 conn(0x7f3b980fe3a0 msgr2=0x7f3b981007c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:56.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.016+0000 7f3b9d200640 1 -- 172.21.15.28:0/854696478 shutdown_connections 2024-05-20T20:05:56.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.016+0000 7f3b9d200640 1 -- 172.21.15.28:0/854696478 wait complete. 2024-05-20T20:05:56.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 Processor -- start 2024-05-20T20:05:56.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 -- start start 2024-05-20T20:05:56.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b98102850 0x7f3b9819b270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 0x7f3b9819b7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98108860 0x7f3b9819fb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3b98112b70 con 0x7f3b98103210 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3b981129f0 con 0x7f3b98108860 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.017+0000 7f3b9d200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3b98112cf0 con 0x7f3b98102850 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 0x7f3b9819b7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 0x7f3b9819b7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:44642/0 (socket says 172.21.15.28:44642) 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 -- 172.21.15.28:0/176301778 learned_addr learned my addr 172.21.15.28:0/176301778 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b97e00640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98108860 0x7f3b9819fb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b97400640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b98102850 0x7f3b9819b270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 -- 172.21.15.28:0/176301778 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b98102850 msgr2=0x7f3b9819b270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b98102850 0x7f3b9819b270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 -- 172.21.15.28:0/176301778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98108860 msgr2=0x7f3b9819fb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98108860 0x7f3b9819fb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b8fe00640 1 -- 172.21.15.28:0/176301778 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3b981a0260 con 0x7f3b98103210 2024-05-20T20:05:56.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.018+0000 7f3b97e00640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98108860 0x7f3b9819fb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:56.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.019+0000 7f3b8fe00640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 0x7f3b9819b7b0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f3b8002a9e0 tx=0x7f3b80003c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:56.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.019+0000 7f3b94c00640 1 -- 172.21.15.28:0/176301778 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b8003f070 con 0x7f3b98103210 2024-05-20T20:05:56.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.019+0000 7f3b94c00640 1 -- 172.21.15.28:0/176301778 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3b8003bb20 con 0x7f3b98103210 2024-05-20T20:05:56.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.019+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3b981a04f0 con 0x7f3b98103210 2024-05-20T20:05:56.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.019+0000 7f3b94c00640 1 -- 172.21.15.28:0/176301778 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b80035030 con 0x7f3b98103210 2024-05-20T20:05:56.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.019+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3b981a7de0 con 0x7f3b98103210 2024-05-20T20:05:56.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.020+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3b98103f90 con 0x7f3b98103210 2024-05-20T20:05:56.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.023+0000 7f3b94c00640 1 -- 172.21.15.28:0/176301778 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f3b80042070 con 0x7f3b98103210 2024-05-20T20:05:56.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.024+0000 7f3b94c00640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b7c0b2a70 0x7f3b7c0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.024+0000 7f3b94c00640 1 -- 172.21.15.28:0/176301778 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f3b80102f70 con 0x7f3b98103210 2024-05-20T20:05:56.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.024+0000 7f3b97400640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b7c0b2a70 0x7f3b7c0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.025+0000 7f3b97400640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b7c0b2a70 0x7f3b7c0b4f30 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f3b88009af0 tx=0x7f3b88005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:56.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.030+0000 7f3b94c00640 1 -- 172.21.15.28:0/176301778 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f3b800ca780 con 0x7f3b98103210 2024-05-20T20:05:56.063 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.063 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: pgmap v12: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 341 B/s wr, 16 op/s 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.064 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:55 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: pgmap v12: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 341 B/s wr, 16 op/s 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:55 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: pgmap v12: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 341 B/s wr, 16 op/s 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:05:56.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:55 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:05:56.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.276+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}) -- 0x7f3b9810e220 con 0x7f3b7c0b2a70 2024-05-20T20:05:56.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.427+0000 7f3b94c00640 1 -- 172.21.15.28:0/176301778 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+24 (secure 0 0 0) 0x7f3b9810e220 con 0x7f3b7c0b2a70 2024-05-20T20:05:56.427 INFO:teuthology.orchestra.run.smithi028.stdout:Schedule set for path / 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b7c0b2a70 msgr2=0x7f3b7c0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b7c0b2a70 0x7f3b7c0b4f30 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f3b88009af0 tx=0x7f3b88005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 msgr2=0x7f3b9819b7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 0x7f3b9819b7b0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f3b8002a9e0 tx=0x7f3b80003c10 comp rx=0 tx=0).stop 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 shutdown_connections 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b98108860 0x7f3b9819fb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b7c0b2a70 0x7f3b7c0b4f30 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b98103210 0x7f3b9819b7b0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 --2- 172.21.15.28:0/176301778 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b98102850 0x7f3b9819b270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.430+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 >> 172.21.15.28:0/176301778 conn(0x7f3b980fe3a0 msgr2=0x7f3b9810db10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:56.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.431+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 shutdown_connections 2024-05-20T20:05:56.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.431+0000 7f3b9d200640 1 -- 172.21.15.28:0/176301778 wait complete. 2024-05-20T20:05:56.453 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6m3h' 2024-05-20T20:05:56.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 -- 172.21.15.28:0/3756070742 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f459410f340 msgr2=0x7f4594111730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 --2- 172.21.15.28:0/3756070742 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f459410f340 0x7f4594111730 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f458c009ca0 tx=0x7f458c02a950 comp rx=0 tx=0).stop 2024-05-20T20:05:56.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 -- 172.21.15.28:0/3756070742 shutdown_connections 2024-05-20T20:05:56.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 --2- 172.21.15.28:0/3756070742 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f459410f340 0x7f4594111730 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 --2- 172.21.15.28:0/3756070742 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4594075670 0x7f459410ee00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 --2- 172.21.15.28:0/3756070742 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4594074d50 0x7f4594075130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 -- 172.21.15.28:0/3756070742 >> 172.21.15.28:0/3756070742 conn(0x7f4594079ee0 msgr2=0x7f459407a2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:56.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 -- 172.21.15.28:0/3756070742 shutdown_connections 2024-05-20T20:05:56.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.640+0000 7f459c600640 1 -- 172.21.15.28:0/3756070742 wait complete. 2024-05-20T20:05:56.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.641+0000 7f459c600640 1 Processor -- start 2024-05-20T20:05:56.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459c600640 1 -- start start 2024-05-20T20:05:56.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459c600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4594074d50 0x7f459419b2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459c600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4594075670 0x7f459419b820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459c600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f459410f340 0x7f459419fba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459c600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4594114f20 con 0x7f459410f340 2024-05-20T20:05:56.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459c600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4594114da0 con 0x7f4594075670 2024-05-20T20:05:56.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459c600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f45941150a0 con 0x7f4594074d50 2024-05-20T20:05:56.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f4599200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4594075670 0x7f459419b820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f4599200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4594075670 0x7f459419b820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:50298/0 (socket says 172.21.15.28:50298) 2024-05-20T20:05:56.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f4599200640 1 -- 172.21.15.28:0/1026236717 learned_addr learned my addr 172.21.15.28:0/1026236717 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f459a600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f459410f340 0x7f459419fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.642+0000 7f4599c00640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4594074d50 0x7f459419b2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.643+0000 7f459a600640 1 -- 172.21.15.28:0/1026236717 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4594074d50 msgr2=0x7f459419b2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.643+0000 7f459a600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4594074d50 0x7f459419b2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.643+0000 7f459a600640 1 -- 172.21.15.28:0/1026236717 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4594075670 msgr2=0x7f459419b820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.643+0000 7f459a600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4594075670 0x7f459419b820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.643+0000 7f459a600640 1 -- 172.21.15.28:0/1026236717 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f45941a0320 con 0x7f459410f340 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.643+0000 7f4599200640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4594075670 0x7f459419b820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:56.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.643+0000 7f459a600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f459410f340 0x7f459419fba0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f458c005cc0 tx=0x7f458c031f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:56.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.644+0000 7f4592000640 1 -- 172.21.15.28:0/1026236717 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f458c034040 con 0x7f459410f340 2024-05-20T20:05:56.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.644+0000 7f4592000640 1 -- 172.21.15.28:0/1026236717 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f458c0308f0 con 0x7f459410f340 2024-05-20T20:05:56.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.644+0000 7f4592000640 1 -- 172.21.15.28:0/1026236717 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f458c0044d0 con 0x7f459410f340 2024-05-20T20:05:56.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.644+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f45941a05b0 con 0x7f459410f340 2024-05-20T20:05:56.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.644+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f45941a7e50 con 0x7f459410f340 2024-05-20T20:05:56.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.645+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f45940507e0 con 0x7f459410f340 2024-05-20T20:05:56.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.647+0000 7f4592000640 1 -- 172.21.15.28:0/1026236717 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f458c037020 con 0x7f459410f340 2024-05-20T20:05:56.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.649+0000 7f4592000640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f457c0b2bb0 0x7f457c0b5070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:56.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.649+0000 7f4592000640 1 -- 172.21.15.28:0/1026236717 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f458c1034c0 con 0x7f459410f340 2024-05-20T20:05:56.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.649+0000 7f4599c00640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f457c0b2bb0 0x7f457c0b5070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:56.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.650+0000 7f4599c00640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f457c0b2bb0 0x7f457c0b5070 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f45880095f0 tx=0x7f4588005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:56.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.653+0000 7f4592000640 1 -- 172.21.15.28:0/1026236717 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f458c0cad10 con 0x7f459410f340 2024-05-20T20:05:56.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.898+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}) -- 0x7f45941011d0 con 0x7f457c0b2bb0 2024-05-20T20:05:56.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:56.998+0000 7f4592000640 1 -- 172.21.15.28:0/1026236717 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+26 (secure 0 0 0) 0x7f45941011d0 con 0x7f457c0b2bb0 2024-05-20T20:05:56.999 INFO:teuthology.orchestra.run.smithi028.stdout:Retention added to path / 2024-05-20T20:05:57.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.001+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f457c0b2bb0 msgr2=0x7f457c0b5070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.001+0000 7f459c600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f457c0b2bb0 0x7f457c0b5070 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f45880095f0 tx=0x7f4588005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:57.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.001+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f459410f340 msgr2=0x7f459419fba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.001+0000 7f459c600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f459410f340 0x7f459419fba0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f458c005cc0 tx=0x7f458c031f40 comp rx=0 tx=0).stop 2024-05-20T20:05:57.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 shutdown_connections 2024-05-20T20:05:57.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4594075670 0x7f459419b820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f457c0b2bb0 0x7f457c0b5070 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f459410f340 0x7f459419fba0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 --2- 172.21.15.28:0/1026236717 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4594074d50 0x7f459419b2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 >> 172.21.15.28:0/1026236717 conn(0x7f4594079ee0 msgr2=0x7f4594100a20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 shutdown_connections 2024-05-20T20:05:57.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.002+0000 7f459c600640 1 -- 172.21.15.28:0/1026236717 wait complete. 2024-05-20T20:05:57.023 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2024-05-20T20:05:57.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.229+0000 7fb8ca600640 1 -- 172.21.15.28:0/3534385055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4103210 msgr2=0x7fb8c410f720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.229+0000 7fb8ca600640 1 --2- 172.21.15.28:0/3534385055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4103210 0x7fb8c410f720 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fb8bc009ca0 tx=0x7fb8bc02a8c0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.230+0000 7fb8ca600640 1 -- 172.21.15.28:0/3534385055 shutdown_connections 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.230+0000 7fb8ca600640 1 --2- 172.21.15.28:0/3534385055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4103210 0x7fb8c410f720 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.230+0000 7fb8ca600640 1 --2- 172.21.15.28:0/3534385055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c4102850 0x7fb8c4102cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.230+0000 7fb8ca600640 1 --2- 172.21.15.28:0/3534385055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c4108860 0x7fb8c4108c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.230+0000 7fb8ca600640 1 -- 172.21.15.28:0/3534385055 >> 172.21.15.28:0/3534385055 conn(0x7fb8c40fe3a0 msgr2=0x7fb8c41007c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.230+0000 7fb8ca600640 1 -- 172.21.15.28:0/3534385055 shutdown_connections 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.230+0000 7fb8ca600640 1 -- 172.21.15.28:0/3534385055 wait complete. 2024-05-20T20:05:57.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 Processor -- start 2024-05-20T20:05:57.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 -- start start 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4102850 0x7fb8c419b210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c4103210 0x7fb8c419b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c4108860 0x7fb8c419fad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb8c4112b70 con 0x7fb8c4103210 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb8c41129f0 con 0x7fb8c4102850 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8ca600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb8c4112cf0 con 0x7fb8c4108860 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8c9200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4102850 0x7fb8c419b210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.231+0000 7fb8c3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c4103210 0x7fb8c419b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4102850 0x7fb8c419b210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53708/0 (socket says 172.21.15.28:53708) 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 -- 172.21.15.28:0/4102779109 learned_addr learned my addr 172.21.15.28:0/4102779109 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9c00640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c4108860 0x7fb8c419fad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 -- 172.21.15.28:0/4102779109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c4108860 msgr2=0x7fb8c419fad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c4108860 0x7fb8c419fad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 -- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c4103210 msgr2=0x7fb8c419b750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c4103210 0x7fb8c419b750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 -- 172.21.15.28:0/4102779109 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb8c41a0200 con 0x7fb8c4102850 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c9200640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4102850 0x7fb8c419b210 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fb8b800e9f0 tx=0x7fb8b800eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.232+0000 7fb8c1600640 1 -- 172.21.15.28:0/4102779109 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8b800cd80 con 0x7fb8c4102850 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.233+0000 7fb8c1600640 1 -- 172.21.15.28:0/4102779109 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb8b80108e0 con 0x7fb8c4102850 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.233+0000 7fb8c1600640 1 -- 172.21.15.28:0/4102779109 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8b8018980 con 0x7fb8c4102850 2024-05-20T20:05:57.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.233+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb8c41a04f0 con 0x7fb8c4102850 2024-05-20T20:05:57.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.233+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb8c41a7d80 con 0x7fb8c4102850 2024-05-20T20:05:57.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.234+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb8c4103f90 con 0x7fb8c4102850 2024-05-20T20:05:57.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.237+0000 7fb8c1600640 1 -- 172.21.15.28:0/4102779109 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7fb8b8010430 con 0x7fb8c4102850 2024-05-20T20:05:57.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.238+0000 7fb8c1600640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2af0 0x7fb8ac0b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.238+0000 7fb8c1600640 1 -- 172.21.15.28:0/4102779109 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb8b80e2b50 con 0x7fb8c4102850 2024-05-20T20:05:57.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.239+0000 7fb8c3e00640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2af0 0x7fb8ac0b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.239+0000 7fb8c3e00640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2af0 0x7fb8ac0b4fb0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fb8c410fac0 tx=0x7fb8b0005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.242+0000 7fb8c1600640 1 -- 172.21.15.28:0/4102779109 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb8b80a9c40 con 0x7fb8c4102850 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 -- 172.21.15.28:0/1461815816 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f8075770 msgr2=0x7fa8f8075b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/1461815816 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f8075770 0x7fa8f8075b50 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fa8e8009990 tx=0x7fa8e8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 -- 172.21.15.28:0/1461815816 shutdown_connections 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/1461815816 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa8f810ff40 0x7fa8f8112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/1461815816 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f8075770 0x7fa8f8075b50 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/1461815816 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8f8076090 0x7fa8f810fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 -- 172.21.15.28:0/1461815816 >> 172.21.15.28:0/1461815816 conn(0x7fa8f807a6a0 msgr2=0x7fa8f807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 -- 172.21.15.28:0/1461815816 shutdown_connections 2024-05-20T20:05:57.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.257+0000 7fa8fdc00700 1 -- 172.21.15.28:0/1461815816 wait complete. 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.258+0000 7fa8fdc00700 1 Processor -- start 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.258+0000 7fa8fdc00700 1 -- start start 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8fdc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa8f8075770 0x7fa8f819be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8fdc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8f8076090 0x7fa8f819c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8fdc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f810ff40 0x7fa8f81a0690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8fdc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f8115940 con 0x7fa8f8076090 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8fdc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f81157c0 con 0x7fa8f810ff40 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8fdc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f8115ac0 con 0x7fa8f8075770 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8f7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f810ff40 0x7fa8f81a0690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8f6000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8f8076090 0x7fa8f819c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8f7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f810ff40 0x7fa8f81a0690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53724/0 (socket says 172.21.15.28:53724) 2024-05-20T20:05:57.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8f7400700 1 -- 172.21.15.28:0/225149654 learned_addr learned my addr 172.21.15.28:0/225149654 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.259+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa8f8075770 0x7fa8f819be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8f7400700 1 -- 172.21.15.28:0/225149654 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa8f8075770 msgr2=0x7fa8f819be10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8f7400700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa8f8075770 0x7fa8f819be10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8f7400700 1 -- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8f8076090 msgr2=0x7fa8f819c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8f7400700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8f8076090 0x7fa8f819c350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8f7400700 1 -- 172.21.15.28:0/225149654 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa8f81a0dd0 con 0x7fa8f810ff40 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa8f8075770 0x7fa8f819be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8f7400700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f810ff40 0x7fa8f81a0690 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fa8f00094f0 tx=0x7fa8f000d3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8ef400700 1 -- 172.21.15.28:0/225149654 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8f000ddd0 con 0x7fa8f810ff40 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8ef400700 1 -- 172.21.15.28:0/225149654 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa8f0009e20 con 0x7fa8f810ff40 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8ef400700 1 -- 172.21.15.28:0/225149654 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8f0015400 con 0x7fa8f810ff40 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.260+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa8f81a10c0 con 0x7fa8f810ff40 2024-05-20T20:05:57.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.261+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa8f81a8940 con 0x7fa8f810ff40 2024-05-20T20:05:57.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.264+0000 7fa8ef400700 1 -- 172.21.15.28:0/225149654 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7fa8f0005020 con 0x7fa8f810ff40 2024-05-20T20:05:57.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.266+0000 7fa8ef400700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8dc0b2b90 0x7fa8dc0b5050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.266+0000 7fa8ef400700 1 -- 172.21.15.28:0/225149654 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa8f00e2390 con 0x7fa8f810ff40 2024-05-20T20:05:57.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.266+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8dc0b2b90 0x7fa8dc0b5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.267+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8dc0b2b90 0x7fa8dc0b5050 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fa8e8006010 tx=0x7fa8e801a040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.268+0000 7fa90855fb80 1 Processor -- start 2024-05-20T20:05:57.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.268+0000 7fa90855fb80 1 -- start start 2024-05-20T20:05:57.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.268+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9690 0x557de5cc9a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cc3ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc4640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa90855fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557de5c21ec0 con 0x557de5cc9690 2024-05-20T20:05:57.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa90855fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557de5c11a60 con 0x557de5cc9fb0 2024-05-20T20:05:57.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa90855fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557de5c15aa0 con 0x557de5cc41c0 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f6a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9690 0x557de5cc9a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cc3ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cc3ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53740/0 (socket says 172.21.15.28:53740) 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f6000700 1 -- 172.21.15.28:0/1829821971 learned_addr learned my addr 172.21.15.28:0/1829821971 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f7400700 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc4640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f7400700 1 -- 172.21.15.28:0/1829821971 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 msgr2=0x557de5cc3ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f7400700 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cc3ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f7400700 1 -- 172.21.15.28:0/1829821971 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9690 msgr2=0x557de5cc9a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:57.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f7400700 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9690 0x557de5cc9a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.269+0000 7fa8f7400700 1 -- 172.21.15.28:0/1829821971 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557de5bda4a0 con 0x557de5cc41c0 2024-05-20T20:05:57.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.270+0000 7fa8f6000700 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cc3ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:57.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.270+0000 7fa8f7400700 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc4640 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa8f00df910 tx=0x7fa8f00a9b50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4c789b5ed25fdb80 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.270+0000 7fa8e3e00700 1 -- 172.21.15.28:0/1829821971 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8f00b2230 con 0x557de5cc41c0 2024-05-20T20:05:57.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.270+0000 7fa8e3e00700 1 -- 172.21.15.28:0/1829821971 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa8f0004dd0 con 0x557de5cc41c0 2024-05-20T20:05:57.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.270+0000 7fa8e3e00700 1 -- 172.21.15.28:0/1829821971 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8f00b2000 con 0x557de5cc41c0 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.271+0000 7fa90855fb80 1 -- 172.21.15.28:0/1829821971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 msgr2=0x557de5cc4640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.271+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc4640 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fa8f00df910 tx=0x7fa8f00a9b50 comp rx=0 tx=0).stop 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.272+0000 7fa90855fb80 1 -- 172.21.15.28:0/1829821971 shutdown_connections 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.272+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc4640 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.272+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9690 0x557de5cc9a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.272+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1829821971 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cc3ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.272+0000 7fa90855fb80 1 -- 172.21.15.28:0/1829821971 >> 172.21.15.28:0/1829821971 conn(0x557de5bf1c60 msgr2=0x557de5cbe710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.272+0000 7fa90855fb80 1 -- 172.21.15.28:0/1829821971 shutdown_connections 2024-05-20T20:05:57.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.272+0000 7fa90855fb80 1 -- 172.21.15.28:0/1829821971 wait complete. 2024-05-20T20:05:57.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.273+0000 7fa90855fb80 1 -- start start 2024-05-20T20:05:57.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.273+0000 7fa90855fb80 1 Processor -- start 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc41c0 0x557de5ccfe10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 0x557de5ccd1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 0x557de5ccd6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa90855fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557de5b18020 con 0x557de5cc9fb0 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa90855fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557de5bb27e0 con 0x557de5cc41c0 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa90855fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557de5cd0a00 con 0x557de5cc9690 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa90855fb80 10 client.0 _ll_register_callbacks cb 0x557de5bbbcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:57.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa8f6a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc41c0 0x557de5ccfe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa8f6a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc41c0 0x557de5ccfe10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53750/0 (socket says 172.21.15.28:53750) 2024-05-20T20:05:57.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa8f6a00700 1 -- 172.21.15.28:0/812889641 learned_addr learned my addr 172.21.15.28:0/812889641 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa8f6a00700 1 -- 172.21.15.28:0/812889641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 msgr2=0x557de5ccd1b0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.274+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 0x557de5ccd1b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa8f6a00700 1 -- 172.21.15.28:0/812889641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 msgr2=0x557de5ccd6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 0x557de5ccd6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa8f6a00700 1 -- 172.21.15.28:0/812889641 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557de5b61950 con 0x557de5cc41c0 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc41c0 0x557de5ccfe10 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fa8e8015000 tx=0x7fa8e8009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f8cfe520ce5befe5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa8e0c00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa8e0c00700 1 -- 172.21.15.28:0/812889641 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8e802f850 con 0x557de5cc41c0 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa8e0c00700 1 -- 172.21.15.28:0/812889641 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa8e802f9f0 con 0x557de5cc41c0 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.275+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa8f0004dd0 con 0x557de5cc41c0 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa8e0c00700 1 -- 172.21.15.28:0/812889641 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa8e803d050 con 0x557de5cc41c0 2024-05-20T20:05:57.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 10 client.25304 fetch_fsmap learned FSMap version 29 2024-05-20T20:05:57.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557de5be20e0 con 0x557de5cc41c0 2024-05-20T20:05:57.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa8e0c00700 1 -- 172.21.15.28:0/812889641 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa8f0004dd0 con 0x557de5cc41c0 2024-05-20T20:05:57.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa8e0c00700 1 -- 172.21.15.28:0/812889641 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa8e80173f0 con 0x557de5cc41c0 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 10 client.25304 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 10 client.25304 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 20 client.25304 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5cdd770 0x557de5cdfc10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 4 client.25304 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.276+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557de5c23e10 con 0x557de5cdd770 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.277+0000 7fa8f6000700 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5cdd770 0x557de5cdfc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.277+0000 7fa8f6000700 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5cdd770 0x557de5cdfc10 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8c65a2749784eebb server_cookie=617aa555700ff5cd in_seq=0 out_seq=0 2024-05-20T20:05:57.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.277+0000 7fa8e0c00700 10 client.25304 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:57.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.278+0000 7fa8e0c00700 1 -- 172.21.15.28:0/812889641 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x557de5c23e10 con 0x557de5cdd770 2024-05-20T20:05:57.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.278+0000 7fa8e0c00700 10 client.25304 handle_command_reply: tid=0 2024-05-20T20:05:57.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.278+0000 7fa90855fb80 1 client.25304 shutdown 2024-05-20T20:05:57.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.278+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc41c0 msgr2=0x557de5ccfe10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.278+0000 7fa90855fb80 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc41c0 0x557de5ccfe10 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fa8e8015000 tx=0x7fa8e8009310 comp rx=0 tx=0).stop 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 shutdown_connections 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5cdd770 0x557de5cdfc10 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 0x557de5ccd6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc41c0 0x557de5ccfe10 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 --2- 172.21.15.28:0/812889641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 0x557de5ccd1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 >> 172.21.15.28:0/812889641 conn(0x557de5bf1c60 msgr2=0x557de5cbddf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa8e0c00700 0 client.25304 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 shutdown_connections 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 1 -- 172.21.15.28:0/812889641 wait complete. 2024-05-20T20:05:57.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.279+0000 7fa90855fb80 20 client.25304 trim_cache size 0 max 16384 2024-05-20T20:05:57.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 Processor -- start 2024-05-20T20:05:57.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 -- start start 2024-05-20T20:05:57.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc3ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9690 0x557de5cd9aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 0x557de5cc7810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f81abae0 con 0x557de5cc9fb0 2024-05-20T20:05:57.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f81ad620 con 0x557de5cc9690 2024-05-20T20:05:57.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.303+0000 7fa90855fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f81ac1b0 con 0x557de5cc41c0 2024-05-20T20:05:57.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9690 0x557de5cd9aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc3ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9690 0x557de5cd9aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53760/0 (socket says 172.21.15.28:53760) 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 0x557de5cc7810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 -- 172.21.15.28:0/1782398393 learned_addr learned my addr 172.21.15.28:0/1782398393 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc3ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55240/0 (socket says 172.21.15.28:55240) 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 -- 172.21.15.28:0/1782398393 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 msgr2=0x557de5cc3ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc3ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 -- 172.21.15.28:0/1782398393 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 msgr2=0x557de5cc7810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 0x557de5cc7810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 -- 172.21.15.28:0/1782398393 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557de5cc9cc0 con 0x557de5cc9690 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc3ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.304+0000 7fa8f6000700 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9690 0x557de5cd9aa0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fa8e4009e90 tx=0x7fa8e4010710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6434ed008c9572ad server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.305+0000 7fa8e3400700 1 -- 172.21.15.28:0/1782398393 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8e4004020 con 0x557de5cc9690 2024-05-20T20:05:57.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.305+0000 7fa8e3400700 1 -- 172.21.15.28:0/1782398393 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa8e4003680 con 0x557de5cc9690 2024-05-20T20:05:57.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.305+0000 7fa8e3400700 1 -- 172.21.15.28:0/1782398393 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8e401c680 con 0x557de5cc9690 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 -- 172.21.15.28:0/1782398393 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9690 msgr2=0x557de5cd9aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9690 0x557de5cd9aa0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fa8e4009e90 tx=0x7fa8e4010710 comp rx=0 tx=0).stop 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 -- 172.21.15.28:0/1782398393 shutdown_connections 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc9fb0 0x557de5cc7810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc41c0 0x557de5cc3ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1782398393 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9690 0x557de5cd9aa0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 -- 172.21.15.28:0/1782398393 >> 172.21.15.28:0/1782398393 conn(0x557de5bf1c60 msgr2=0x557de5ccac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 -- 172.21.15.28:0/1782398393 shutdown_connections 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 -- 172.21.15.28:0/1782398393 wait complete. 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.306+0000 7fa90855fb80 1 -- start start 2024-05-20T20:05:57.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 1 Processor -- start 2024-05-20T20:05:57.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc41c0 0x557de5cd6180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 0x557de5cd2c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cd31a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f81acf50 con 0x557de5cc41c0 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f81ac880 con 0x557de5cc9fb0 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa8f8107700 con 0x557de5cc9690 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.307+0000 7fa90855fb80 10 client.0 _ll_register_callbacks cb 0x557de5bbbcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc41c0 0x557de5cd6180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc41c0 0x557de5cd6180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50776/0 (socket says 172.21.15.28:50776) 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 -- 172.21.15.28:0/1526895059 learned_addr learned my addr 172.21.15.28:0/1526895059 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6000700 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 0x557de5cd2c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f7400700 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cd31a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 -- 172.21.15.28:0/1526895059 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 msgr2=0x557de5cd2c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:57.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 0x557de5cd2c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 -- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 msgr2=0x557de5cd31a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:05:57.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cd31a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.308+0000 7fa8f6a00700 1 -- 172.21.15.28:0/1526895059 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557de5cc92e0 con 0x557de5cc41c0 2024-05-20T20:05:57.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa8f6a00700 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc41c0 0x557de5cd6180 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fa8e8017890 tx=0x7fa8e8017d30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=250aef154d81845d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa8e0c00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:05:57.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa8e0c00700 1 -- 172.21.15.28:0/1526895059 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8e8017600 con 0x557de5cc41c0 2024-05-20T20:05:57.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa8e4003680 con 0x557de5cc41c0 2024-05-20T20:05:57.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa8e0c00700 1 -- 172.21.15.28:0/1526895059 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa8e802f650 con 0x557de5cc41c0 2024-05-20T20:05:57.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa8e0c00700 1 -- 172.21.15.28:0/1526895059 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa8e802f850 con 0x557de5cc41c0 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa90855fb80 10 client.15171 fetch_fsmap learned FSMap version 29 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.309+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557de5d49060 con 0x557de5cc41c0 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa8e0c00700 1 -- 172.21.15.28:0/1526895059 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa8e4003680 con 0x557de5cc41c0 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa8e0c00700 1 -- 172.21.15.28:0/1526895059 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa8e8024b50 con 0x557de5cc41c0 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa90855fb80 10 client.15171 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa90855fb80 10 client.15171 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa90855fb80 20 client.15171 populate_metadata read hostname 'smithi028' 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5d49740 0x557de5d4bb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa90855fb80 4 client.15171 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.310+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x557de5c1d5f0 con 0x557de5d49740 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.311+0000 7fa8f6000700 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5d49740 0x557de5d4bb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.311+0000 7fa8f6000700 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5d49740 0x557de5d4bb60 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5330a9eff3084b4e server_cookie=13414e277bdf2352 in_seq=0 out_seq=0 2024-05-20T20:05:57.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.311+0000 7fa8e0c00700 10 client.15171 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:57.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.312+0000 7fa8e0c00700 1 -- 172.21.15.28:0/1526895059 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x557de5c1d5f0 con 0x557de5d49740 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.312+0000 7fa8e0c00700 10 client.15171 handle_command_reply: tid=0 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.312+0000 7fa90855fb80 1 client.15171 shutdown 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc41c0 msgr2=0x557de5cd6180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc41c0 0x557de5cd6180 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fa8e8017890 tx=0x7fa8e8017d30 comp rx=0 tx=0).stop 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 shutdown_connections 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557de5d49740 0x557de5d4bb60 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557de5cc9fb0 0x557de5cd31a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557de5cc41c0 0x557de5cd6180 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 --2- 172.21.15.28:0/1526895059 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557de5cc9690 0x557de5cd2c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 >> 172.21.15.28:0/1526895059 conn(0x557de5bf1c60 msgr2=0x557de5cca390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa8e0c00700 0 client.15171 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:05:57.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 shutdown_connections 2024-05-20T20:05:57.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 1 -- 172.21.15.28:0/1526895059 wait complete. 2024-05-20T20:05:57.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.313+0000 7fa90855fb80 20 client.15171 trim_cache size 0 max 16384 2024-05-20T20:05:57.315 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:05:57.315 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:05:57.315 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "9f22e454-6796-47fe-b349-05531a40a16c", 2024-05-20T20:05:57.315 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:05:57.315 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.314+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8dc0b2b90 msgr2=0x7fa8dc0b5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.314+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8dc0b2b90 0x7fa8dc0b5050 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fa8e8006010 tx=0x7fa8e801a040 comp rx=0 tx=0).stop 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.314+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f810ff40 msgr2=0x7fa8f81a0690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.314+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f810ff40 0x7fa8f81a0690 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fa8f00094f0 tx=0x7fa8f000d3b0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 shutdown_connections 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8dc0b2b90 0x7fa8dc0b5050 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8f810ff40 0x7fa8f81a0690 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa8f8075770 0x7fa8f819be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 --2- 172.21.15.28:0/225149654 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8f8076090 0x7fa8f819c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 >> 172.21.15.28:0/225149654 conn(0x7fa8f807a6a0 msgr2=0x7fa8f8100750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 shutdown_connections 2024-05-20T20:05:57.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.315+0000 7fa8fdc00700 1 -- 172.21.15.28:0/225149654 wait complete. 2024-05-20T20:05:57.504 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.504+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) -- 0x7fb8c40ff860 con 0x7fb8ac0b2af0 2024-05-20T20:05:57.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.522+0000 7fb8c1600640 1 -- 172.21.15.28:0/4102779109 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+298 (secure 0 0 0) 0x7fb8c40ff860 con 0x7fb8ac0b2af0 2024-05-20T20:05:57.522 INFO:teuthology.orchestra.run.smithi028.stdout:{"fs": "cephfs", "subvol": null, "group": null, "path": "/", "rel_path": "/", "schedule": "1m", "retention": {"m": 6, "h": 3}, "start": "2024-05-20T00:00:00", "created": "2024-05-20T20:05:56", "first": null, "last": null, "last_pruned": null, "created_count": 0, "pruned_count": 0, "active": true} 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.524+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2af0 msgr2=0x7fb8ac0b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.524+0000 7fb8ca600640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2af0 0x7fb8ac0b4fb0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fb8c410fac0 tx=0x7fb8b0005f90 comp rx=0 tx=0).stop 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.524+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4102850 msgr2=0x7fb8c419b210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.524+0000 7fb8ca600640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4102850 0x7fb8c419b210 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fb8b800e9f0 tx=0x7fb8b800eeb0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.524+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 shutdown_connections 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.524+0000 7fb8ca600640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2af0 0x7fb8ac0b4fb0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.524+0000 7fb8ca600640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c4108860 0x7fb8c419fad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.525+0000 7fb8ca600640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c4103210 0x7fb8c419b750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.525+0000 7fb8ca600640 1 --2- 172.21.15.28:0/4102779109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c4102850 0x7fb8c419b210 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.525+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 >> 172.21.15.28:0/4102779109 conn(0x7fb8c40fe3a0 msgr2=0x7fb8c410dfe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.525+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 shutdown_connections 2024-05-20T20:05:57.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.525+0000 7fb8ca600640 1 -- 172.21.15.28:0/4102779109 wait complete. 2024-05-20T20:05:57.541 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:57 smithi028 ceph-mon[31178]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:57.544 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2024-05-20T20:05:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:57 smithi154 ceph-mon[36314]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:57.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:57 smithi033 ceph-mon[36179]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:57.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 -- 172.21.15.28:0/2869138649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 msgr2=0x7f93ac104f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 --2- 172.21.15.28:0/2869138649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac104f10 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f9394009860 tx=0x7f939402a340 comp rx=0 tx=0).stop 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 -- 172.21.15.28:0/2869138649 shutdown_connections 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 --2- 172.21.15.28:0/2869138649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac104f10 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 --2- 172.21.15.28:0/2869138649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93ac105450 0x7f93ac111960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 --2- 172.21.15.28:0/2869138649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93ac10aaa0 0x7f93ac10ae80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 -- 172.21.15.28:0/2869138649 >> 172.21.15.28:0/2869138649 conn(0x7f93ac100560 msgr2=0x7f93ac102980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 -- 172.21.15.28:0/2869138649 shutdown_connections 2024-05-20T20:05:57.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.744+0000 7f93abe00640 1 -- 172.21.15.28:0/2869138649 wait complete. 2024-05-20T20:05:57.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.745+0000 7f93abe00640 1 Processor -- start 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.745+0000 7f93abe00640 1 -- start start 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93abe00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac076f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93abe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93ac105450 0x7f93ac0774a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93abe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93ac10aaa0 0x7f93ac0779e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93abe00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f93ac114d30 con 0x7f93ac10aaa0 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93abe00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f93ac114bb0 con 0x7f93ac104a90 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93abe00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f93ac114eb0 con 0x7f93ac105450 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93aaa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac076f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93aaa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac076f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53790/0 (socket says 172.21.15.28:53790) 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93aaa00640 1 -- 172.21.15.28:0/2122162261 learned_addr learned my addr 172.21.15.28:0/2122162261 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93ab400640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93ac10aaa0 0x7f93ac0779e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.746+0000 7f93a2000640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93ac105450 0x7f93ac0774a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93aaa00640 1 -- 172.21.15.28:0/2122162261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93ac105450 msgr2=0x7f93ac0774a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93aaa00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93ac105450 0x7f93ac0774a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93aaa00640 1 -- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93ac10aaa0 msgr2=0x7f93ac0779e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93aaa00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93ac10aaa0 0x7f93ac0779e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93aaa00640 1 -- 172.21.15.28:0/2122162261 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f93ac1ab8a0 con 0x7f93ac104a90 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93aaa00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac076f60 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f939c00e9f0 tx=0x7f939c00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93a3e00640 1 -- 172.21.15.28:0/2122162261 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f939c00cde0 con 0x7f93ac104a90 2024-05-20T20:05:57.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f93ac1abb90 con 0x7f93ac104a90 2024-05-20T20:05:57.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93a3e00640 1 -- 172.21.15.28:0/2122162261 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f939c0035f0 con 0x7f93ac104a90 2024-05-20T20:05:57.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93a3e00640 1 -- 172.21.15.28:0/2122162261 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f939c010690 con 0x7f93ac104a90 2024-05-20T20:05:57.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.747+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f93ac1ac0b0 con 0x7f93ac104a90 2024-05-20T20:05:57.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.748+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f93ac1061d0 con 0x7f93ac104a90 2024-05-20T20:05:57.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.751+0000 7f93a3e00640 1 -- 172.21.15.28:0/2122162261 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 25) ==== 149879+0+0 (secure 0 0 0) 0x7f939c047a40 con 0x7f93ac104a90 2024-05-20T20:05:57.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.753+0000 7f93a3e00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93900b2a70 0x7f93900b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:05:57.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.753+0000 7f93a3e00640 1 -- 172.21.15.28:0/2122162261 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f939c0e2b30 con 0x7f93ac104a90 2024-05-20T20:05:57.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.753+0000 7f93a2000640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93900b2a70 0x7f93900b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:05:57.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.754+0000 7f93a2000640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93900b2a70 0x7f93900b4f30 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f939402a930 tx=0x7f9394030340 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:05:57.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.755+0000 7f93a3e00640 1 -- 172.21.15.28:0/2122162261 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f939c0aa270 con 0x7f93ac104a90 2024-05-20T20:05:57.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:57.997+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) -- 0x7f93ac062e90 con 0x7f93900b2a70 2024-05-20T20:05:58.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.008+0000 7f93a3e00640 1 -- 172.21.15.28:0/2122162261 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+10 (secure 0 0 0) 0x7f93ac062e90 con 0x7f93900b2a70 2024-05-20T20:05:58.008 INFO:teuthology.orchestra.run.smithi028.stdout:/ 1m 6m3h 2024-05-20T20:05:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93900b2a70 msgr2=0x7f93900b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93900b2a70 0x7f93900b4f30 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f939402a930 tx=0x7f9394030340 comp rx=0 tx=0).stop 2024-05-20T20:05:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 msgr2=0x7f93ac076f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:05:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac076f60 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f939c00e9f0 tx=0x7f939c00eeb0 comp rx=0 tx=0).stop 2024-05-20T20:05:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 shutdown_connections 2024-05-20T20:05:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93900b2a70 0x7f93900b4f30 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93ac10aaa0 0x7f93ac0779e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f93ac105450 0x7f93ac0774a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 --2- 172.21.15.28:0/2122162261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93ac104a90 0x7f93ac076f60 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:05:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 >> 172.21.15.28:0/2122162261 conn(0x7f93ac100560 msgr2=0x7f93ac100c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:05:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 shutdown_connections 2024-05-20T20:05:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:05:58.011+0000 7f93abe00640 1 -- 172.21.15.28:0/2122162261 wait complete. 2024-05-20T20:05:58.031 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > START_TIME' 2024-05-20T20:05:58.118 INFO:teuthology.run_tasks:Running task full_sequential_finally... 2024-05-20T20:05:58.128 INFO:teuthology.run_tasks:Running task background_exec... 2024-05-20T20:05:58.179 INFO:teuthology.task.background_exec:Running background command on role mon.a host ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:05:58.179 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest daemon-helper kill --kill-group bash -c 'while sleep 13; do ceph tell mds.cephfs:0 flush journal; done' 2024-05-20T20:05:58.181 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:05:58.230 INFO:teuthology.run_tasks:Running task quiescer... 2024-05-20T20:05:58.243 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-20T20:05:58.244 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T20:05:58.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:58 smithi028 ceph-mon[31178]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:58.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:58 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:05:58.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:58 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:05:58.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:58 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:05:58.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:58 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:05:58.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:58 smithi028 ceph-mon[31178]: from='client.25292 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:58.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:58 smithi028 ceph-mon[31178]: pgmap v13: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:05:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:58 smithi154 ceph-mon[36314]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:58 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:05:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:58 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:05:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:58 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:05:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:58 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:05:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:58 smithi154 ceph-mon[36314]: from='client.25292 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:58.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:58 smithi154 ceph-mon[36314]: pgmap v13: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:05:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:58 smithi033 ceph-mon[36179]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:58 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:05:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:58 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:05:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:58 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:05:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:58 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:05:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:58 smithi033 ceph-mon[36179]: from='client.25292 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:58.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:58 smithi033 ceph-mon[36179]: pgmap v13: 97 pgs: 97 active+clean; 1.1 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:05:58.702 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:58.702 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:05:59.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:05:59 smithi154 ceph-mon[36314]: from='client.25316 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:59.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:05:59 smithi033 ceph-mon[36179]: from='client.25316 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:05:59.719 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:05:59 smithi028 ceph-mon[31178]: from='client.25316 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:06:00.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:00 smithi028 ceph-mon[31178]: pgmap v14: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 76 KiB/s wr, 44 op/s 2024-05-20T20:06:00.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:00 smithi154 ceph-mon[36314]: pgmap v14: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 76 KiB/s wr, 44 op/s 2024-05-20T20:06:00.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:00 smithi033 ceph-mon[36179]: pgmap v14: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 76 KiB/s wr, 44 op/s 2024-05-20T20:06:01.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:06:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:06:01.018+0000 7f7d12800700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:06:01.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:06:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:06:01.018+0000 7f7d12800700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:06:01.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:06:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:06:01.018+0000 7f7d12800700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:06:01.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:06:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:06:01.018+0000 7f7d12800700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:06:01.055 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:06:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:06:01.018+0000 7f7d12800700 -1 client.0 error registering admin socket command: (17) File exists 2024-05-20T20:06:01.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 -- 172.21.15.28:0/1213944762 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760109540 msgr2=0x7f7760109920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 --2- 172.21.15.28:0/1213944762 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760109540 0x7f7760109920 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f7754009920 tx=0x7f7754009c30 comp rx=0 tx=0).stop 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 -- 172.21.15.28:0/1213944762 shutdown_connections 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 --2- 172.21.15.28:0/1213944762 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7760103e50 0x7f7760110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 --2- 172.21.15.28:0/1213944762 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760109540 0x7f7760109920 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 --2- 172.21.15.28:0/1213944762 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7760103490 0x7f7760103910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 -- 172.21.15.28:0/1213944762 >> 172.21.15.28:0/1213944762 conn(0x7f77600fefc0 msgr2=0x7f77601013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 -- 172.21.15.28:0/1213944762 shutdown_connections 2024-05-20T20:06:01.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.857+0000 7f7766a00700 1 -- 172.21.15.28:0/1213944762 wait complete. 2024-05-20T20:06:01.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.858+0000 7f7766a00700 1 Processor -- start 2024-05-20T20:06:01.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.858+0000 7f7766a00700 1 -- start start 2024-05-20T20:06:01.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7760103490 0x7f77601a1a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:01.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760103e50 0x7f77601a1fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:01.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7760109540 0x7f776019bb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:01.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7760113620 con 0x7f7760103490 2024-05-20T20:06:01.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f77601134a0 con 0x7f7760103e50 2024-05-20T20:06:01.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f77601137a0 con 0x7f7760109540 2024-05-20T20:06:01.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7765600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7760103490 0x7f77601a1a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:01.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7764c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760103e50 0x7f77601a1fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:01.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7760109540 0x7f776019bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:01.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7764c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760103e50 0x7f77601a1fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53834/0 (socket says 172.21.15.28:53834) 2024-05-20T20:06:01.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7765600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7760103490 0x7f77601a1a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50828/0 (socket says 172.21.15.28:50828) 2024-05-20T20:06:01.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7764c00700 1 -- 172.21.15.28:0/991986501 learned_addr learned my addr 172.21.15.28:0/991986501 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766000700 1 -- 172.21.15.28:0/991986501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760103e50 msgr2=0x7f77601a1fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766000700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760103e50 0x7f77601a1fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766000700 1 -- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7760103490 msgr2=0x7f77601a1a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.859+0000 7f7766000700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7760103490 0x7f77601a1a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.860+0000 7f7766000700 1 -- 172.21.15.28:0/991986501 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f776019c410 con 0x7f7760109540 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.860+0000 7f7766000700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7760109540 0x7f776019bb30 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f775800efc0 tx=0x7f775800c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.860+0000 7f775e000700 1 -- 172.21.15.28:0/991986501 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77580092e0 con 0x7f7760109540 2024-05-20T20:06:01.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.860+0000 7f775e000700 1 -- 172.21.15.28:0/991986501 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7758004bb0 con 0x7f7760109540 2024-05-20T20:06:01.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.860+0000 7f775e000700 1 -- 172.21.15.28:0/991986501 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7758004020 con 0x7f7760109540 2024-05-20T20:06:01.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.860+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f776019c6a0 con 0x7f7760109540 2024-05-20T20:06:01.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.860+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f77601a8810 con 0x7f7760109540 2024-05-20T20:06:01.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.861+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7760104bb0 con 0x7f7760109540 2024-05-20T20:06:01.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.863+0000 7f775e000700 1 -- 172.21.15.28:0/991986501 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f775801b030 con 0x7f7760109540 2024-05-20T20:06:01.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.865+0000 7f775e000700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f77480b2c20 0x7f77480b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:01.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.865+0000 7f775e000700 1 -- 172.21.15.28:0/991986501 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f77580e2290 con 0x7f7760109540 2024-05-20T20:06:01.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.865+0000 7f7765600700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f77480b2c20 0x7f77480b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:01.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.866+0000 7f7765600700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f77480b2c20 0x7f77480b50e0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f7754009680 tx=0x7f7754019040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:01.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:01.869+0000 7f775e000700 1 -- 172.21.15.28:0/991986501 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7758022020 con 0x7f7760109540 2024-05-20T20:06:02.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.114+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f776010f6f0 con 0x7f77480b2c20 2024-05-20T20:06:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.137+0000 7f775e000700 1 -- 172.21.15.28:0/991986501 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258087 (secure 0 0 0) 0x7f776010f6f0 con 0x7f77480b2c20 2024-05-20T20:06:02.143 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:02.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f77480b2c20 msgr2=0x7f77480b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f77480b2c20 0x7f77480b50e0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f7754009680 tx=0x7f7754019040 comp rx=0 tx=0).stop 2024-05-20T20:06:02.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7760109540 msgr2=0x7f776019bb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7760109540 0x7f776019bb30 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f775800efc0 tx=0x7f775800c3a0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 shutdown_connections 2024-05-20T20:06:02.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7760109540 0x7f776019bb30 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7760103490 0x7f77601a1a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f77480b2c20 0x7f77480b50e0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 --2- 172.21.15.28:0/991986501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7760103e50 0x7f77601a1fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 >> 172.21.15.28:0/991986501 conn(0x7f77600fefc0 msgr2=0x7f776010efe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:02.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.144+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 shutdown_connections 2024-05-20T20:06:02.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.145+0000 7f7766a00700 1 -- 172.21.15.28:0/991986501 wait complete. 2024-05-20T20:06:02.146 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 -- 172.21.15.28:0/2917977319 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c4076090 msgr2=0x7f33c410fa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 --2- 172.21.15.28:0/2917977319 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c4076090 0x7f33c410fa00 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f33ac009990 tx=0x7f33ac009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 -- 172.21.15.28:0/2917977319 shutdown_connections 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 --2- 172.21.15.28:0/2917977319 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f33c410ff40 0x7f33c4112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 --2- 172.21.15.28:0/2917977319 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33c4075770 0x7f33c4075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 --2- 172.21.15.28:0/2917977319 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c4076090 0x7f33c410fa00 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 -- 172.21.15.28:0/2917977319 >> 172.21.15.28:0/2917977319 conn(0x7f33c407a6a0 msgr2=0x7f33c407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 -- 172.21.15.28:0/2917977319 shutdown_connections 2024-05-20T20:06:02.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.311+0000 7f33ca800700 1 -- 172.21.15.28:0/2917977319 wait complete. 2024-05-20T20:06:02.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.313+0000 7f33ca800700 1 Processor -- start 2024-05-20T20:06:02.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.313+0000 7f33ca800700 1 -- start start 2024-05-20T20:06:02.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33ca800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33c4075770 0x7f33c419bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33ca800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f33c4076090 0x7f33c419c330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33ca800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c410ff40 0x7f33c41a0670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33ca800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f33c4115920 con 0x7f33c410ff40 2024-05-20T20:06:02.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33ca800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f33c41157a0 con 0x7f33c4075770 2024-05-20T20:06:02.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33ca800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f33c4115aa0 con 0x7f33c4076090 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33c3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c410ff40 0x7f33c41a0670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33c3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33c4075770 0x7f33c419bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33c3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33c4075770 0x7f33c419bdf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53852/0 (socket says 172.21.15.28:53852) 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33baa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f33c4076090 0x7f33c419c330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33c3400700 1 -- 172.21.15.28:0/2129352440 learned_addr learned my addr 172.21.15.28:0/2129352440 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.314+0000 7f33c3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c410ff40 0x7f33c41a0670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50832/0 (socket says 172.21.15.28:50832) 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33baa00700 1 -- 172.21.15.28:0/2129352440 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33c4075770 msgr2=0x7f33c419bdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33baa00700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33c4075770 0x7f33c419bdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33baa00700 1 -- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c410ff40 msgr2=0x7f33c41a0670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33baa00700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c410ff40 0x7f33c41a0670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33baa00700 1 -- 172.21.15.28:0/2129352440 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f33c41a0db0 con 0x7f33c4076090 2024-05-20T20:06:02.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33baa00700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f33c4076090 0x7f33c419c330 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f33ac0059d0 tx=0x7f33ac0057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:02.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33c0c00700 1 -- 172.21.15.28:0/2129352440 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac027070 con 0x7f33c4076090 2024-05-20T20:06:02.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f33c41a1040 con 0x7f33c4076090 2024-05-20T20:06:02.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33c0c00700 1 -- 172.21.15.28:0/2129352440 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f33ac022470 con 0x7f33c4076090 2024-05-20T20:06:02.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33c0c00700 1 -- 172.21.15.28:0/2129352440 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac017650 con 0x7f33c4076090 2024-05-20T20:06:02.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.315+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f33c41a8920 con 0x7f33c4076090 2024-05-20T20:06:02.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.319+0000 7f33c0c00700 1 -- 172.21.15.28:0/2129352440 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f33ac02a070 con 0x7f33c4076090 2024-05-20T20:06:02.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.320+0000 7f33d510ab80 1 Processor -- start 2024-05-20T20:06:02.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.320+0000 7f33d510ab80 1 -- start start 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33c0c00700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33a80b2ba0 0x7f33a80b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03229f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322f30 0x55d4b031cb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b031d5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33d510ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d4b027ae40 con 0x55d4b0322610 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33d510ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d4b026a9e0 con 0x55d4b0322f30 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33d510ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d4b026ea20 con 0x55d4b031d140 2024-05-20T20:06:02.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33c0c00700 1 -- 172.21.15.28:0/2129352440 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f33ac0e35d0 con 0x7f33c4076090 2024-05-20T20:06:02.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33c3e00700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33a80b2ba0 0x7f33a80b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33c3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03229f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33baa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322f30 0x55d4b031cb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33baa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322f30 0x55d4b031cb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53856/0 (socket says 172.21.15.28:53856) 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.321+0000 7f33baa00700 1 -- 172.21.15.28:0/3264679323 learned_addr learned my addr 172.21.15.28:0/3264679323 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33c3e00700 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b031d5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33baa00700 1 -- 172.21.15.28:0/3264679323 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 msgr2=0x55d4b031d5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33baa00700 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b031d5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33baa00700 1 -- 172.21.15.28:0/3264679323 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 msgr2=0x55d4b03229f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33baa00700 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03229f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33baa00700 1 -- 172.21.15.28:0/3264679323 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d4b0233420 con 0x55d4b0322f30 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33baa00700 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322f30 0x55d4b031cb20 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f33ac0aa6a0 tx=0x7f33ac0b1050 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b67630863901a2bb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33c3e00700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33a80b2ba0 0x7f33a80b5060 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x55d4b031db00 tx=0x7f33bc009920 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.322+0000 7f33b8c00700 1 -- 172.21.15.28:0/3264679323 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac0a9e80 con 0x55d4b0322f30 2024-05-20T20:06:02.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.323+0000 7f33b8c00700 1 -- 172.21.15.28:0/3264679323 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f33ac0aa220 con 0x55d4b0322f30 2024-05-20T20:06:02.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.323+0000 7f33b8c00700 1 -- 172.21.15.28:0/3264679323 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac013cc0 con 0x55d4b0322f30 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.323+0000 7f33d510ab80 1 -- 172.21.15.28:0/3264679323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322f30 msgr2=0x55d4b031cb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.323+0000 7f33d510ab80 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322f30 0x55d4b031cb20 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f33ac0aa6a0 tx=0x7f33ac0b1050 comp rx=0 tx=0).stop 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.323+0000 7f33d510ab80 1 -- 172.21.15.28:0/3264679323 shutdown_connections 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.323+0000 7f33d510ab80 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b031d5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.323+0000 7f33d510ab80 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03229f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.324+0000 7f33d510ab80 1 --2- 172.21.15.28:0/3264679323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322f30 0x55d4b031cb20 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.324+0000 7f33d510ab80 1 -- 172.21.15.28:0/3264679323 >> 172.21.15.28:0/3264679323 conn(0x55d4b024abe0 msgr2=0x55d4b0317690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.324+0000 7f33d510ab80 1 -- 172.21.15.28:0/3264679323 shutdown_connections 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.324+0000 7f33d510ab80 1 -- 172.21.15.28:0/3264679323 wait complete. 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.324+0000 7f33d510ab80 1 -- start start 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.324+0000 7f33d510ab80 1 Processor -- start 2024-05-20T20:06:02.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b0328cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b03260b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322f30 0x55d4b03265f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33d510ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d4b0170fa0 con 0x55d4b0322f30 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33d510ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d4b020b760 con 0x55d4b0322610 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33d510ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d4b03297a0 con 0x55d4b031d140 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33d510ab80 10 client.0 _ll_register_callbacks cb 0x55d4b0214c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33c3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322f30 0x55d4b03265f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33baa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b03260b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33baa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b03260b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53870/0 (socket says 172.21.15.28:53870) 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33baa00700 1 -- 172.21.15.28:0/2973831321 learned_addr learned my addr 172.21.15.28:0/2973831321 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:02.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.325+0000 7f33c3400700 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b0328cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33baa00700 1 -- 172.21.15.28:0/2973831321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 msgr2=0x55d4b0328cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33baa00700 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b0328cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33baa00700 1 -- 172.21.15.28:0/2973831321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322f30 msgr2=0x55d4b03265f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33baa00700 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322f30 0x55d4b03265f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33baa00700 1 -- 172.21.15.28:0/2973831321 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d4b01ba8d0 con 0x55d4b0322610 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33baa00700 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b03260b0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f33ac005f50 tx=0x7f33ac0922d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a2de7d366e6587dc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33b1600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33b1600700 1 -- 172.21.15.28:0/2973831321 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac02b3b0 con 0x55d4b0322610 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33b1600700 1 -- 172.21.15.28:0/2973831321 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f33ac027420 con 0x55d4b0322610 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.326+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f33ac0aa220 con 0x55d4b0322610 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33b1600700 1 -- 172.21.15.28:0/2973831321 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f33ac00e3e0 con 0x55d4b0322610 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 10 client.25340 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:02.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d4b023b060 con 0x55d4b0322610 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33b1600700 1 -- 172.21.15.28:0/2973831321 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f33ac0aa220 con 0x55d4b0322610 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33b1600700 1 -- 172.21.15.28:0/2973831321 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f33ac0aa970 con 0x55d4b0322610 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 10 client.25340 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 10 client.25340 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 20 client.25340 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b0336110 0x55d4b0338500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 4 client.25340 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.327+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d4b027cd90 con 0x55d4b0336110 2024-05-20T20:06:02.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.328+0000 7f33c3400700 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b0336110 0x55d4b0338500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.328+0000 7f33c3400700 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b0336110 0x55d4b0338500 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f0bf919c1ee6b42d server_cookie=be454a08f4ecd1f6 in_seq=0 out_seq=0 2024-05-20T20:06:02.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.328+0000 7f33b1600700 10 client.25340 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:02.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33b1600700 1 -- 172.21.15.28:0/2973831321 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55d4b027cd90 con 0x55d4b0336110 2024-05-20T20:06:02.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33b1600700 10 client.25340 handle_command_reply: tid=0 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 client.25340 shutdown 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 msgr2=0x55d4b03260b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b03260b0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f33ac005f50 tx=0x7f33ac0922d0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 shutdown_connections 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b0336110 0x55d4b0338500 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322f30 0x55d4b03265f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b031d140 0x55d4b0328cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 --2- 172.21.15.28:0/2973831321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b03260b0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33b1600700 0 client.25340 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.329+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 >> 172.21.15.28:0/2973831321 conn(0x55d4b024abe0 msgr2=0x55d4b0316d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.330+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 shutdown_connections 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.330+0000 7f33d510ab80 1 -- 172.21.15.28:0/2973831321 wait complete. 2024-05-20T20:06:02.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.330+0000 7f33d510ab80 20 client.25340 trim_cache size 0 max 16384 2024-05-20T20:06:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 Processor -- start 2024-05-20T20:06:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 -- start start 2024-05-20T20:06:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b031d140 0x55d4b031cb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03174f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b0324f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f33c41afe70 con 0x55d4b0322610 2024-05-20T20:06:02.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f33c41b19b0 con 0x55d4b031d140 2024-05-20T20:06:02.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.353+0000 7f33d510ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f33c41b0540 con 0x55d4b0322f30 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03174f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33c3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b031d140 0x55d4b031cb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03174f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50846/0 (socket says 172.21.15.28:50846) 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 -- 172.21.15.28:0/1936671750 learned_addr learned my addr 172.21.15.28:0/1936671750 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33c3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b031d140 0x55d4b031cb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53872/0 (socket says 172.21.15.28:53872) 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33c3e00700 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b0324f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 -- 172.21.15.28:0/1936671750 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 msgr2=0x55d4b0324f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b0324f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 -- 172.21.15.28:0/1936671750 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b031d140 msgr2=0x55d4b031cb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b031d140 0x55d4b031cb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.354+0000 7f33baa00700 1 -- 172.21.15.28:0/1936671750 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d4b03184d0 con 0x55d4b0322610 2024-05-20T20:06:02.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.355+0000 7f33baa00700 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03174f0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f33ac0aa410 tx=0x7f33ac0b1050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da9a346b9969dbf1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:02.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.355+0000 7f33b3e00700 1 -- 172.21.15.28:0/1936671750 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac00e5d0 con 0x55d4b0322610 2024-05-20T20:06:02.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.355+0000 7f33b3e00700 1 -- 172.21.15.28:0/1936671750 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f33ac0aa220 con 0x55d4b0322610 2024-05-20T20:06:02.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.355+0000 7f33b3e00700 1 -- 172.21.15.28:0/1936671750 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac0aa970 con 0x55d4b0322610 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 -- 172.21.15.28:0/1936671750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 msgr2=0x55d4b03174f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03174f0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f33ac0aa410 tx=0x7f33ac0b1050 comp rx=0 tx=0).stop 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 -- 172.21.15.28:0/1936671750 shutdown_connections 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b0324f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b031d140 0x55d4b031cb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 --2- 172.21.15.28:0/1936671750 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b0322610 0x55d4b03174f0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 -- 172.21.15.28:0/1936671750 >> 172.21.15.28:0/1936671750 conn(0x55d4b024abe0 msgr2=0x55d4b0324870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 -- 172.21.15.28:0/1936671750 shutdown_connections 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.356+0000 7f33d510ab80 1 -- 172.21.15.28:0/1936671750 wait complete. 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 -- start start 2024-05-20T20:06:02.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 Processor -- start 2024-05-20T20:06:02.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b031d140 0x55d4b03399e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b0339f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b039f950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f33c41b12e0 con 0x55d4b031d140 2024-05-20T20:06:02.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f33c41b0c10 con 0x55d4b0322610 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.357+0000 7f33d510ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f33c41ade10 con 0x55d4b0322f30 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33d510ab80 10 client.0 _ll_register_callbacks cb 0x55d4b0214c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33c3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b031d140 0x55d4b03399e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33c3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b039f950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33baa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b0339f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33c3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b031d140 0x55d4b03399e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50848/0 (socket says 172.21.15.28:50848) 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33c3400700 1 -- 172.21.15.28:0/286814053 learned_addr learned my addr 172.21.15.28:0/286814053 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33baa00700 1 -- 172.21.15.28:0/286814053 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 msgr2=0x55d4b039f950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33baa00700 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b039f950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33baa00700 1 -- 172.21.15.28:0/286814053 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b031d140 msgr2=0x55d4b03399e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33baa00700 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b031d140 0x55d4b03399e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.358+0000 7f33baa00700 1 -- 172.21.15.28:0/286814053 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d4b039fee0 con 0x55d4b0322610 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33baa00700 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b0339f20 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x55d4b03275e0 tx=0x7f33ac013f60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c453c5471e2db565 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33b1600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33b1600700 1 -- 172.21.15.28:0/286814053 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33ac0aae10 con 0x55d4b0322610 2024-05-20T20:06:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f33ac0aa220 con 0x55d4b0322610 2024-05-20T20:06:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33b1600700 1 -- 172.21.15.28:0/286814053 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f33ac0050f0 con 0x55d4b0322610 2024-05-20T20:06:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33b1600700 1 -- 172.21.15.28:0/286814053 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f33ac0aa710 con 0x55d4b0322610 2024-05-20T20:06:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33d510ab80 10 client.25346 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d4b039fee0 con 0x55d4b0322610 2024-05-20T20:06:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.359+0000 7f33b1600700 1 -- 172.21.15.28:0/286814053 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f33ac0aa220 con 0x55d4b0322610 2024-05-20T20:06:02.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33b1600700 1 -- 172.21.15.28:0/286814053 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f33ac0a6080 con 0x55d4b0322610 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33d510ab80 10 client.25346 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33d510ab80 10 client.25346 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33d510ab80 20 client.25346 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33d510ab80 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b03a1430 0x55d4b03a3820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33d510ab80 4 client.25346 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55d4b0276570 con 0x55d4b03a1430 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.360+0000 7f33c3400700 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b03a1430 0x55d4b03a3820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33c3400700 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b03a1430 0x55d4b03a3820 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f9eb318b185a8751 server_cookie=b58c8089e0668d4e in_seq=0 out_seq=0 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33b1600700 10 client.25346 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33b1600700 1 -- 172.21.15.28:0/286814053 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+259 (crc 0 0 0) 0x55d4b0276570 con 0x55d4b03a1430 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33b1600700 10 client.25346 handle_command_reply: tid=0 2024-05-20T20:06:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 client.25346 shutdown 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 msgr2=0x55d4b0339f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b0339f20 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x55d4b03275e0 tx=0x7f33ac013f60 comp rx=0 tx=0).stop 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 shutdown_connections 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d4b03a1430 0x55d4b03a3820 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d4b0322f30 0x55d4b039f950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d4b031d140 0x55d4b03399e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 --2- 172.21.15.28:0/286814053 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d4b0322610 0x55d4b0339f20 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33b1600700 0 client.25346 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 >> 172.21.15.28:0/286814053 conn(0x55d4b024abe0 msgr2=0x55d4b0321a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 shutdown_connections 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.361+0000 7f33d510ab80 1 -- 172.21.15.28:0/286814053 wait complete. 2024-05-20T20:06:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33d510ab80 20 client.25346 trim_cache size 0 max 16384 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "IDLE (waiting for more scrubs)", 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": { 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: "9f22e454-6796-47fe-b349-05531a40a16c": { 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: "path": "/", 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: "tag": "9f22e454-6796-47fe-b349-05531a40a16c", 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: "options": "recursive,force" 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:06:02.365 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:06:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33a80b2ba0 msgr2=0x7f33a80b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33a80b2ba0 0x7f33a80b5060 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x55d4b031db00 tx=0x7f33bc009920 comp rx=0 tx=0).stop 2024-05-20T20:06:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f33c4076090 msgr2=0x7f33c419c330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f33c4076090 0x7f33c419c330 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f33ac0059d0 tx=0x7f33ac0057c0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 shutdown_connections 2024-05-20T20:06:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33c410ff40 0x7f33c41a0670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33a80b2ba0 0x7f33a80b5060 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33c4075770 0x7f33c419bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 --2- 172.21.15.28:0/2129352440 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f33c4076090 0x7f33c419c330 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.362+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 >> 172.21.15.28:0/2129352440 conn(0x7f33c407a6a0 msgr2=0x7f33c4100750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.363+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 shutdown_connections 2024-05-20T20:06:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:02.363+0000 7f33ca800700 1 -- 172.21.15.28:0/2129352440 wait complete. 2024-05-20T20:06:02.960 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:02 smithi028 ceph-mon[31178]: pgmap v15: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 76 KiB/s wr, 40 op/s 2024-05-20T20:06:02.960 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:02 smithi028 ceph-mon[31178]: mgrmap e26: x(active, since 24s), standbys: z, y 2024-05-20T20:06:03.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:02 smithi154 ceph-mon[36314]: pgmap v15: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 76 KiB/s wr, 40 op/s 2024-05-20T20:06:03.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:02 smithi154 ceph-mon[36314]: mgrmap e26: x(active, since 24s), standbys: z, y 2024-05-20T20:06:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:02 smithi033 ceph-mon[36179]: pgmap v15: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 76 KiB/s wr, 40 op/s 2024-05-20T20:06:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:02 smithi033 ceph-mon[36179]: mgrmap e26: x(active, since 24s), standbys: z, y 2024-05-20T20:06:03.494 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2024-05-20T20:06:01.636134+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1391779,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":3063,"num_read_kb":1933,"num_write":649,"num_write_kb":4540,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":988,"ondisk_log_size":988,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347896,"kb_used_data":11184,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124593928,"statfs":{"total":1151940427776,"available":1151584182272,"internally_reserved":0,"allocated":11452416,"data_stored":9378429,"data_compressed":316014,"data_compressed_allocated":417792,"data_compressed_original":4165632,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":196632,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":355,"num_read_kb":282,"num_write":126,"num_write_kb":914,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006531"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603392+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:05:37.603392+0000","last_peered":"2024-05-20T20:05:37.603392+0000","last_clean":"2024-05-20T20:05:37.603392+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:05:37.603392+0000","last_undegraded":"2024-05-20T20:05:37.603392+0000","last_fullsized":"2024-05-20T20:05:37.603392+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750621+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:05:37.750621+0000","last_peered":"2024-05-20T20:05:37.750621+0000","last_clean":"2024-05-20T20:05:37.750621+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:05:37.750621+0000","last_undegraded":"2024-05-20T20:05:37.750621+0000","last_fullsized":"2024-05-20T20:05:37.750621+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750419+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:05:37.750419+0000","last_peered":"2024-05-20T20:05:37.750419+0000","last_clean":"2024-05-20T20:05:37.750419+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:05:37.750419+0000","last_undegraded":"2024-05-20T20:05:37.750419+0000","last_fullsized":"2024-05-20T20:05:37.750419+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946345+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:05:37.946345+0000","last_peered":"2024-05-20T20:05:37.946345+0000","last_clean":"2024-05-20T20:05:37.946345+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:05:37.946345+0000","last_undegraded":"2024-05-20T20:05:37.946345+0000","last_fullsized":"2024-05-20T20:05:37.946345+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":27,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950341+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:05:37.950341+0000","last_peered":"2024-05-20T20:05:37.950341+0000","last_clean":"2024-05-20T20:05:37.950341+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:05:37.950341+0000","last_undegraded":"2024-05-20T20:05:37.950341+0000","last_fullsized":"2024-05-20T20:05:37.950341+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601481+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:05:37.601481+0000","last_peered":"2024-05-20T20:05:37.601481+0000","last_clean":"2024-05-20T20:05:37.601481+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:05:37.601481+0000","last_undegraded":"2024-05-20T20:05:37.601481+0000","last_fullsized":"2024-05-20T20:05:37.601481+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946714+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:05:37.946714+0000","last_peered":"2024-05-20T20:05:37.946714+0000","last_clean":"2024-05-20T20:05:37.946714+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:05:37.946714+0000","last_undegraded":"2024-05-20T20:05:37.946714+0000","last_fullsized":"2024-05-20T20:05:37.946714+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740698+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:05:37.740698+0000","last_peered":"2024-05-20T20:05:37.740698+0000","last_clean":"2024-05-20T20:05:37.740698+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:05:37.740698+0000","last_undegraded":"2024-05-20T20:05:37.740698+0000","last_fullsized":"2024-05-20T20:05:37.740698+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.942016+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:05:37.942016+0000","last_peered":"2024-05-20T20:05:37.942016+0000","last_clean":"2024-05-20T20:05:37.942016+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:05:37.942016+0000","last_undegraded":"2024-05-20T20:05:37.942016+0000","last_fullsized":"2024-05-20T20:05:37.942016+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":45,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601908+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:05:37.601908+0000","last_peered":"2024-05-20T20:05:37.601908+0000","last_clean":"2024-05-20T20:05:37.601908+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:05:37.601908+0000","last_undegraded":"2024-05-20T20:05:37.601908+0000","last_fullsized":"2024-05-20T20:05:37.601908+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947610+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:05:37.947610+0000","last_peered":"2024-05-20T20:05:37.947610+0000","last_clean":"2024-05-20T20:05:37.947610+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:05:37.947610+0000","last_undegraded":"2024-05-20T20:05:37.947610+0000","last_fullsized":"2024-05-20T20:05:37.947610+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946190+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:05:37.946190+0000","last_peered":"2024-05-20T20:05:37.946190+0000","last_clean":"2024-05-20T20:05:37.946190+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:05:37.946190+0000","last_undegraded":"2024-05-20T20:05:37.946190+0000","last_fullsized":"2024-05-20T20:05:37.946190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951933+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:05:37.951933+0000","last_peered":"2024-05-20T20:05:37.951933+0000","last_clean":"2024-05-20T20:05:37.951933+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:05:37.951933+0000","last_undegraded":"2024-05-20T20:05:37.951933+0000","last_fullsized":"2024-05-20T20:05:37.951933+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749940+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:05:37.749940+0000","last_peered":"2024-05-20T20:05:37.749940+0000","last_clean":"2024-05-20T20:05:37.749940+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:05:37.749940+0000","last_undegraded":"2024-05-20T20:05:37.749940+0000","last_fullsized":"2024-05-20T20:05:37.749940+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749763+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:05:37.749763+0000","last_peered":"2024-05-20T20:05:37.749763+0000","last_clean":"2024-05-20T20:05:37.749763+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:05:37.749763+0000","last_undegraded":"2024-05-20T20:05:37.749763+0000","last_fullsized":"2024-05-20T20:05:37.749763+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603526+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:05:37.603526+0000","last_peered":"2024-05-20T20:05:37.603526+0000","last_clean":"2024-05-20T20:05:37.603526+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:05:37.603526+0000","last_undegraded":"2024-05-20T20:05:37.603526+0000","last_fullsized":"2024-05-20T20:05:37.603526+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602253+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:05:37.602253+0000","last_peered":"2024-05-20T20:05:37.602253+0000","last_clean":"2024-05-20T20:05:37.602253+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:05:37.602253+0000","last_undegraded":"2024-05-20T20:05:37.602253+0000","last_fullsized":"2024-05-20T20:05:37.602253+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940697+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:05:37.940697+0000","last_peered":"2024-05-20T20:05:37.940697+0000","last_clean":"2024-05-20T20:05:37.940697+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:05:37.940697+0000","last_undegraded":"2024-05-20T20:05:37.940697+0000","last_fullsized":"2024-05-20T20:05:37.940697+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'14","reported_seq":298,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:57.312783+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:05:57.312783+0000","last_peered":"2024-05-20T20:05:57.312783+0000","last_clean":"2024-05-20T20:05:57.312783+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:05:57.312783+0000","last_undegraded":"2024-05-20T20:05:57.312783+0000","last_fullsized":"2024-05-20T20:05:57.312783+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":254,"num_read_kb":254,"num_write":15,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741585+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:05:37.741585+0000","last_peered":"2024-05-20T20:05:37.741585+0000","last_clean":"2024-05-20T20:05:37.741585+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:05:37.741585+0000","last_undegraded":"2024-05-20T20:05:37.741585+0000","last_fullsized":"2024-05-20T20:05:37.741585+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741764+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:05:37.741764+0000","last_peered":"2024-05-20T20:05:37.741764+0000","last_clean":"2024-05-20T20:05:37.741764+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:05:37.741764+0000","last_undegraded":"2024-05-20T20:05:37.741764+0000","last_fullsized":"2024-05-20T20:05:37.741764+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750654+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:05:37.750654+0000","last_peered":"2024-05-20T20:05:37.750654+0000","last_clean":"2024-05-20T20:05:37.750654+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:05:37.750654+0000","last_undegraded":"2024-05-20T20:05:37.750654+0000","last_fullsized":"2024-05-20T20:05:37.750654+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'13","reported_seq":43,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950989+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:05:37.950989+0000","last_peered":"2024-05-20T20:05:37.950989+0000","last_clean":"2024-05-20T20:05:37.950989+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:05:37.950989+0000","last_undegraded":"2024-05-20T20:05:37.950989+0000","last_fullsized":"2024-05-20T20:05:37.950989+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":18,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950749+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:05:37.950749+0000","last_peered":"2024-05-20T20:05:37.950749+0000","last_clean":"2024-05-20T20:05:37.950749+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:05:37.950749+0000","last_undegraded":"2024-05-20T20:05:37.950749+0000","last_fullsized":"2024-05-20T20:05:37.950749+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602506+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:05:37.602506+0000","last_peered":"2024-05-20T20:05:37.602506+0000","last_clean":"2024-05-20T20:05:37.602506+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:05:37.602506+0000","last_undegraded":"2024-05-20T20:05:37.602506+0000","last_fullsized":"2024-05-20T20:05:37.602506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"81'8","reported_seq":294,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:57.060147+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:05:57.060147+0000","last_peered":"2024-05-20T20:05:57.060147+0000","last_clean":"2024-05-20T20:05:57.060147+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:05:57.060147+0000","last_undegraded":"2024-05-20T20:05:57.060147+0000","last_fullsized":"2024-05-20T20:05:57.060147+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1545,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":257,"num_read_kb":5,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950345+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:05:37.950345+0000","last_peered":"2024-05-20T20:05:37.950345+0000","last_clean":"2024-05-20T20:05:37.950345+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:05:37.950345+0000","last_undegraded":"2024-05-20T20:05:37.950345+0000","last_fullsized":"2024-05-20T20:05:37.950345+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601320+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:05:37.601320+0000","last_peered":"2024-05-20T20:05:37.601320+0000","last_clean":"2024-05-20T20:05:37.601320+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:05:37.601320+0000","last_undegraded":"2024-05-20T20:05:37.601320+0000","last_fullsized":"2024-05-20T20:05:37.601320+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751407+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:05:37.751407+0000","last_peered":"2024-05-20T20:05:37.751407+0000","last_clean":"2024-05-20T20:05:37.751407+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:05:37.751407+0000","last_undegraded":"2024-05-20T20:05:37.751407+0000","last_fullsized":"2024-05-20T20:05:37.751407+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950714+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:05:37.950714+0000","last_peered":"2024-05-20T20:05:37.950714+0000","last_clean":"2024-05-20T20:05:37.950714+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:05:37.950714+0000","last_undegraded":"2024-05-20T20:05:37.950714+0000","last_fullsized":"2024-05-20T20:05:37.950714+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740756+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:05:37.740756+0000","last_peered":"2024-05-20T20:05:37.740756+0000","last_clean":"2024-05-20T20:05:37.740756+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:05:37.740756+0000","last_undegraded":"2024-05-20T20:05:37.740756+0000","last_fullsized":"2024-05-20T20:05:37.740756+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751178+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:05:37.751178+0000","last_peered":"2024-05-20T20:05:37.751178+0000","last_clean":"2024-05-20T20:05:37.751178+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:05:37.751178+0000","last_undegraded":"2024-05-20T20:05:37.751178+0000","last_fullsized":"2024-05-20T20:05:37.751178+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601398+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:05:37.601398+0000","last_peered":"2024-05-20T20:05:37.601398+0000","last_clean":"2024-05-20T20:05:37.601398+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:05:37.601398+0000","last_undegraded":"2024-05-20T20:05:37.601398+0000","last_fullsized":"2024-05-20T20:05:37.601398+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'7","reported_seq":540,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:58.061121+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:05:58.061121+0000","last_peered":"2024-05-20T20:05:58.061121+0000","last_clean":"2024-05-20T20:05:58.061121+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:05:58.061121+0000","last_undegraded":"2024-05-20T20:05:58.061121+0000","last_fullsized":"2024-05-20T20:05:58.061121+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":504,"num_read_kb":504,"num_write":8,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941795+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:05:37.941795+0000","last_peered":"2024-05-20T20:05:37.941795+0000","last_clean":"2024-05-20T20:05:37.941795+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:05:37.941795+0000","last_undegraded":"2024-05-20T20:05:37.941795+0000","last_fullsized":"2024-05-20T20:05:37.941795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947471+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:05:37.947471+0000","last_peered":"2024-05-20T20:05:37.947471+0000","last_clean":"2024-05-20T20:05:37.947471+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:05:37.947471+0000","last_undegraded":"2024-05-20T20:05:37.947471+0000","last_fullsized":"2024-05-20T20:05:37.947471+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750655+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:05:37.750655+0000","last_peered":"2024-05-20T20:05:37.750655+0000","last_clean":"2024-05-20T20:05:37.750655+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:05:37.750655+0000","last_undegraded":"2024-05-20T20:05:37.750655+0000","last_fullsized":"2024-05-20T20:05:37.750655+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751776+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:05:37.751776+0000","last_peered":"2024-05-20T20:05:37.751776+0000","last_clean":"2024-05-20T20:05:37.751776+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:05:37.751776+0000","last_undegraded":"2024-05-20T20:05:37.751776+0000","last_fullsized":"2024-05-20T20:05:37.751776+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941312+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:05:37.941312+0000","last_peered":"2024-05-20T20:05:37.941312+0000","last_clean":"2024-05-20T20:05:37.941312+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:05:37.941312+0000","last_undegraded":"2024-05-20T20:05:37.941312+0000","last_fullsized":"2024-05-20T20:05:37.941312+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'39","reported_seq":587,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:58.061123+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:05:58.061123+0000","last_peered":"2024-05-20T20:05:58.061123+0000","last_clean":"2024-05-20T20:05:58.061123+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:05:58.061123+0000","last_undegraded":"2024-05-20T20:05:58.061123+0000","last_fullsized":"2024-05-20T20:05:58.061123+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70510,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":519,"num_read_kb":329,"num_write":41,"num_write_kb":85,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950750+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:05:37.950750+0000","last_peered":"2024-05-20T20:05:37.950750+0000","last_clean":"2024-05-20T20:05:37.950750+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:05:37.950750+0000","last_undegraded":"2024-05-20T20:05:37.950750+0000","last_fullsized":"2024-05-20T20:05:37.950750+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602898+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:05:37.602898+0000","last_peered":"2024-05-20T20:05:37.602898+0000","last_clean":"2024-05-20T20:05:37.602898+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:05:37.602898+0000","last_undegraded":"2024-05-20T20:05:37.602898+0000","last_fullsized":"2024-05-20T20:05:37.602898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950828+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:05:37.950828+0000","last_peered":"2024-05-20T20:05:37.950828+0000","last_clean":"2024-05-20T20:05:37.950828+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:05:37.950828+0000","last_undegraded":"2024-05-20T20:05:37.950828+0000","last_fullsized":"2024-05-20T20:05:37.950828+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751260+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:05:37.751260+0000","last_peered":"2024-05-20T20:05:37.751260+0000","last_clean":"2024-05-20T20:05:37.751260+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:05:37.751260+0000","last_undegraded":"2024-05-20T20:05:37.751260+0000","last_fullsized":"2024-05-20T20:05:37.751260+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947376+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:05:37.947376+0000","last_peered":"2024-05-20T20:05:37.947376+0000","last_clean":"2024-05-20T20:05:37.947376+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:05:37.947376+0000","last_undegraded":"2024-05-20T20:05:37.947376+0000","last_fullsized":"2024-05-20T20:05:37.947376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'8","reported_seq":294,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:57.262936+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:05:57.262936+0000","last_peered":"2024-05-20T20:05:57.262936+0000","last_clean":"2024-05-20T20:05:57.262936+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:05:57.262936+0000","last_undegraded":"2024-05-20T20:05:57.262936+0000","last_fullsized":"2024-05-20T20:05:57.262936+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1680,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":258,"num_read_kb":6,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946134+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:05:37.946134+0000","last_peered":"2024-05-20T20:05:37.946134+0000","last_clean":"2024-05-20T20:05:37.946134+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:05:37.946134+0000","last_undegraded":"2024-05-20T20:05:37.946134+0000","last_fullsized":"2024-05-20T20:05:37.946134+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":72,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950363+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:05:37.950363+0000","last_peered":"2024-05-20T20:05:37.950363+0000","last_clean":"2024-05-20T20:05:37.950363+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:05:37.950363+0000","last_undegraded":"2024-05-20T20:05:37.950363+0000","last_fullsized":"2024-05-20T20:05:37.950363+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751376+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:05:37.751376+0000","last_peered":"2024-05-20T20:05:37.751376+0000","last_clean":"2024-05-20T20:05:37.751376+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:05:37.751376+0000","last_undegraded":"2024-05-20T20:05:37.751376+0000","last_fullsized":"2024-05-20T20:05:37.751376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'5","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:39.658956+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:05:39.658956+0000","last_peered":"2024-05-20T20:05:39.658956+0000","last_clean":"2024-05-20T20:05:39.658956+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:05:39.658956+0000","last_undegraded":"2024-05-20T20:05:39.658956+0000","last_fullsized":"2024-05-20T20:05:39.658956+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603167+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:05:37.603167+0000","last_peered":"2024-05-20T20:05:37.603167+0000","last_clean":"2024-05-20T20:05:37.603167+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:05:37.603167+0000","last_undegraded":"2024-05-20T20:05:37.603167+0000","last_fullsized":"2024-05-20T20:05:37.603167+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602390+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:05:37.602390+0000","last_peered":"2024-05-20T20:05:37.602390+0000","last_clean":"2024-05-20T20:05:37.602390+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:05:37.602390+0000","last_undegraded":"2024-05-20T20:05:37.602390+0000","last_fullsized":"2024-05-20T20:05:37.602390+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601741+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:05:37.601741+0000","last_peered":"2024-05-20T20:05:37.601741+0000","last_clean":"2024-05-20T20:05:37.601741+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:05:37.601741+0000","last_undegraded":"2024-05-20T20:05:37.601741+0000","last_fullsized":"2024-05-20T20:05:37.601741+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'6","reported_seq":290,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:57.262970+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:05:57.262970+0000","last_peered":"2024-05-20T20:05:57.262970+0000","last_clean":"2024-05-20T20:05:57.262970+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:05:57.262970+0000","last_undegraded":"2024-05-20T20:05:57.262970+0000","last_fullsized":"2024-05-20T20:05:57.262970+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":254,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601417+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:05:37.601417+0000","last_peered":"2024-05-20T20:05:37.601417+0000","last_clean":"2024-05-20T20:05:37.601417+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:05:37.601417+0000","last_undegraded":"2024-05-20T20:05:37.601417+0000","last_fullsized":"2024-05-20T20:05:37.601417+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751541+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:05:37.751541+0000","last_peered":"2024-05-20T20:05:37.751541+0000","last_clean":"2024-05-20T20:05:37.751541+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:05:37.751541+0000","last_undegraded":"2024-05-20T20:05:37.751541+0000","last_fullsized":"2024-05-20T20:05:37.751541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"80'6","reported_seq":289,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:57.262280+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:05:57.262280+0000","last_peered":"2024-05-20T20:05:57.262280+0000","last_clean":"2024-05-20T20:05:57.262280+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:05:57.262280+0000","last_undegraded":"2024-05-20T20:05:57.262280+0000","last_fullsized":"2024-05-20T20:05:57.262280+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":253,"num_read_kb":253,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741186+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:05:37.741186+0000","last_peered":"2024-05-20T20:05:37.741186+0000","last_clean":"2024-05-20T20:05:37.741186+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:05:37.741186+0000","last_undegraded":"2024-05-20T20:05:37.741186+0000","last_fullsized":"2024-05-20T20:05:37.741186+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.945964+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:05:37.945964+0000","last_peered":"2024-05-20T20:05:37.945964+0000","last_clean":"2024-05-20T20:05:37.945964+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:05:37.945964+0000","last_undegraded":"2024-05-20T20:05:37.945964+0000","last_fullsized":"2024-05-20T20:05:37.945964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"81'99","reported_seq":131,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:39.743703+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:05:39.743703+0000","last_peered":"2024-05-20T20:05:39.743703+0000","last_clean":"2024-05-20T20:05:39.743703+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:05:39.743703+0000","last_undegraded":"2024-05-20T20:05:39.743703+0000","last_fullsized":"2024-05-20T20:05:39.743703+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950636+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:05:37.950636+0000","last_peered":"2024-05-20T20:05:37.950636+0000","last_clean":"2024-05-20T20:05:37.950636+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:05:37.950636+0000","last_undegraded":"2024-05-20T20:05:37.950636+0000","last_fullsized":"2024-05-20T20:05:37.950636+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946570+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:05:37.946570+0000","last_peered":"2024-05-20T20:05:37.946570+0000","last_clean":"2024-05-20T20:05:37.946570+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:05:37.946570+0000","last_undegraded":"2024-05-20T20:05:37.946570+0000","last_fullsized":"2024-05-20T20:05:37.946570+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950840+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:05:37.950840+0000","last_peered":"2024-05-20T20:05:37.950840+0000","last_clean":"2024-05-20T20:05:37.950840+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:05:37.950840+0000","last_undegraded":"2024-05-20T20:05:37.950840+0000","last_fullsized":"2024-05-20T20:05:37.950840+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952108+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:05:37.952108+0000","last_peered":"2024-05-20T20:05:37.952108+0000","last_clean":"2024-05-20T20:05:37.952108+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:05:37.952108+0000","last_undegraded":"2024-05-20T20:05:37.952108+0000","last_fullsized":"2024-05-20T20:05:37.952108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'105","reported_seq":243,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:58.005339+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:05:58.005339+0000","last_peered":"2024-05-20T20:05:58.005339+0000","last_clean":"2024-05-20T20:05:58.005339+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:05:58.005339+0000","last_undegraded":"2024-05-20T20:05:58.005339+0000","last_fullsized":"2024-05-20T20:05:58.005339+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":312,"num_read_kb":277,"num_write":192,"num_write_kb":1287,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741294+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:05:37.741294+0000","last_peered":"2024-05-20T20:05:37.741294+0000","last_clean":"2024-05-20T20:05:37.741294+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:05:37.741294+0000","last_undegraded":"2024-05-20T20:05:37.741294+0000","last_fullsized":"2024-05-20T20:05:37.741294+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946101+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:05:37.946101+0000","last_peered":"2024-05-20T20:05:37.946101+0000","last_clean":"2024-05-20T20:05:37.946101+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:05:37.946101+0000","last_undegraded":"2024-05-20T20:05:37.946101+0000","last_fullsized":"2024-05-20T20:05:37.946101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750609+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:05:37.750609+0000","last_peered":"2024-05-20T20:05:37.750609+0000","last_clean":"2024-05-20T20:05:37.750609+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:05:37.750609+0000","last_undegraded":"2024-05-20T20:05:37.750609+0000","last_fullsized":"2024-05-20T20:05:37.750609+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941387+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:05:37.941387+0000","last_peered":"2024-05-20T20:05:37.941387+0000","last_clean":"2024-05-20T20:05:37.941387+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:05:37.941387+0000","last_undegraded":"2024-05-20T20:05:37.941387+0000","last_fullsized":"2024-05-20T20:05:37.941387+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":26,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950366+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:05:37.950366+0000","last_peered":"2024-05-20T20:05:37.950366+0000","last_clean":"2024-05-20T20:05:37.950366+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:05:37.950366+0000","last_undegraded":"2024-05-20T20:05:37.950366+0000","last_fullsized":"2024-05-20T20:05:37.950366+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741016+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:05:37.741016+0000","last_peered":"2024-05-20T20:05:37.741016+0000","last_clean":"2024-05-20T20:05:37.741016+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:05:37.741016+0000","last_undegraded":"2024-05-20T20:05:37.741016+0000","last_fullsized":"2024-05-20T20:05:37.741016+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602899+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:05:37.602899+0000","last_peered":"2024-05-20T20:05:37.602899+0000","last_clean":"2024-05-20T20:05:37.602899+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:05:37.602899+0000","last_undegraded":"2024-05-20T20:05:37.602899+0000","last_fullsized":"2024-05-20T20:05:37.602899+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940518+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:05:37.940518+0000","last_peered":"2024-05-20T20:05:37.940518+0000","last_clean":"2024-05-20T20:05:37.940518+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:05:37.940518+0000","last_undegraded":"2024-05-20T20:05:37.940518+0000","last_fullsized":"2024-05-20T20:05:37.940518+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741426+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:05:37.741426+0000","last_peered":"2024-05-20T20:05:37.741426+0000","last_clean":"2024-05-20T20:05:37.741426+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:05:37.741426+0000","last_undegraded":"2024-05-20T20:05:37.741426+0000","last_fullsized":"2024-05-20T20:05:37.741426+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950516+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:05:37.950516+0000","last_peered":"2024-05-20T20:05:37.950516+0000","last_clean":"2024-05-20T20:05:37.950516+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:05:37.950516+0000","last_undegraded":"2024-05-20T20:05:37.950516+0000","last_fullsized":"2024-05-20T20:05:37.950516+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947502+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:05:37.947502+0000","last_peered":"2024-05-20T20:05:37.947502+0000","last_clean":"2024-05-20T20:05:37.947502+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:05:37.947502+0000","last_undegraded":"2024-05-20T20:05:37.947502+0000","last_fullsized":"2024-05-20T20:05:37.947502+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'6","reported_seq":291,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:57.060416+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:05:57.060416+0000","last_peered":"2024-05-20T20:05:57.060416+0000","last_clean":"2024-05-20T20:05:57.060416+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:05:57.060416+0000","last_undegraded":"2024-05-20T20:05:57.060416+0000","last_fullsized":"2024-05-20T20:05:57.060416+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1561,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":255,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947819+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:05:37.947819+0000","last_peered":"2024-05-20T20:05:37.947819+0000","last_clean":"2024-05-20T20:05:37.947819+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:05:37.947819+0000","last_undegraded":"2024-05-20T20:05:37.947819+0000","last_fullsized":"2024-05-20T20:05:37.947819+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"81'67","reported_seq":128,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:57.513512+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:05:57.513512+0000","last_peered":"2024-05-20T20:05:57.513512+0000","last_clean":"2024-05-20T20:05:57.513512+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:05:57.513512+0000","last_undegraded":"2024-05-20T20:05:57.513512+0000","last_fullsized":"2024-05-20T20:05:57.513512+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328302,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":34,"num_read_kb":33,"num_write":76,"num_write_kb":941,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947490+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:05:37.947490+0000","last_peered":"2024-05-20T20:05:37.947490+0000","last_clean":"2024-05-20T20:05:37.947490+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:05:37.947490+0000","last_undegraded":"2024-05-20T20:05:37.947490+0000","last_fullsized":"2024-05-20T20:05:37.947490+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952372+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:05:37.952372+0000","last_peered":"2024-05-20T20:05:37.952372+0000","last_clean":"2024-05-20T20:05:37.952372+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:05:37.952372+0000","last_undegraded":"2024-05-20T20:05:37.952372+0000","last_fullsized":"2024-05-20T20:05:37.952372+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947860+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:05:37.947860+0000","last_peered":"2024-05-20T20:05:37.947860+0000","last_clean":"2024-05-20T20:05:37.947860+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:05:37.947860+0000","last_undegraded":"2024-05-20T20:05:37.947860+0000","last_fullsized":"2024-05-20T20:05:37.947860+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603108+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:05:37.603108+0000","last_peered":"2024-05-20T20:05:37.603108+0000","last_clean":"2024-05-20T20:05:37.603108+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:05:37.603108+0000","last_undegraded":"2024-05-20T20:05:37.603108+0000","last_fullsized":"2024-05-20T20:05:37.603108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940537+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:05:37.940537+0000","last_peered":"2024-05-20T20:05:37.940537+0000","last_clean":"2024-05-20T20:05:37.940537+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:05:37.940537+0000","last_undegraded":"2024-05-20T20:05:37.940537+0000","last_fullsized":"2024-05-20T20:05:37.940537+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940554+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:05:37.940554+0000","last_peered":"2024-05-20T20:05:37.940554+0000","last_clean":"2024-05-20T20:05:37.940554+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:05:37.940554+0000","last_undegraded":"2024-05-20T20:05:37.940554+0000","last_fullsized":"2024-05-20T20:05:37.940554+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750191+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:05:37.750191+0000","last_peered":"2024-05-20T20:05:37.750191+0000","last_clean":"2024-05-20T20:05:37.750191+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:05:37.750191+0000","last_undegraded":"2024-05-20T20:05:37.750191+0000","last_fullsized":"2024-05-20T20:05:37.750191+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952180+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:05:37.952180+0000","last_peered":"2024-05-20T20:05:37.952180+0000","last_clean":"2024-05-20T20:05:37.952180+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:05:37.952180+0000","last_undegraded":"2024-05-20T20:05:37.952180+0000","last_fullsized":"2024-05-20T20:05:37.952180+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603224+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:05:37.603224+0000","last_peered":"2024-05-20T20:05:37.603224+0000","last_clean":"2024-05-20T20:05:37.603224+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:05:37.603224+0000","last_undegraded":"2024-05-20T20:05:37.603224+0000","last_fullsized":"2024-05-20T20:05:37.603224+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940832+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:05:37.940832+0000","last_peered":"2024-05-20T20:05:37.940832+0000","last_clean":"2024-05-20T20:05:37.940832+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:05:37.940832+0000","last_undegraded":"2024-05-20T20:05:37.940832+0000","last_fullsized":"2024-05-20T20:05:37.940832+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":26,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741159+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:05:37.741159+0000","last_peered":"2024-05-20T20:05:37.741159+0000","last_clean":"2024-05-20T20:05:37.741159+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:05:37.741159+0000","last_undegraded":"2024-05-20T20:05:37.741159+0000","last_fullsized":"2024-05-20T20:05:37.741159+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751338+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:05:37.751338+0000","last_peered":"2024-05-20T20:05:37.751338+0000","last_clean":"2024-05-20T20:05:37.751338+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:05:37.751338+0000","last_undegraded":"2024-05-20T20:05:37.751338+0000","last_fullsized":"2024-05-20T20:05:37.751338+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740731+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:05:37.740731+0000","last_peered":"2024-05-20T20:05:37.740731+0000","last_clean":"2024-05-20T20:05:37.740731+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:05:37.740731+0000","last_undegraded":"2024-05-20T20:05:37.740731+0000","last_fullsized":"2024-05-20T20:05:37.740731+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941843+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:05:37.941843+0000","last_peered":"2024-05-20T20:05:37.941843+0000","last_clean":"2024-05-20T20:05:37.941843+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:05:37.941843+0000","last_undegraded":"2024-05-20T20:05:37.941843+0000","last_fullsized":"2024-05-20T20:05:37.941843+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951652+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:05:37.951652+0000","last_peered":"2024-05-20T20:05:37.951652+0000","last_clean":"2024-05-20T20:05:37.951652+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:05:37.951652+0000","last_undegraded":"2024-05-20T20:05:37.951652+0000","last_fullsized":"2024-05-20T20:05:37.951652+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602990+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:05:37.602990+0000","last_peered":"2024-05-20T20:05:37.602990+0000","last_clean":"2024-05-20T20:05:37.602990+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:05:37.602990+0000","last_undegraded":"2024-05-20T20:05:37.602990+0000","last_fullsized":"2024-05-20T20:05:37.602990+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":108381,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":801177,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":2919,"num_read_kb":1685,"num_write":512,"num_write_kb":2388,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":811008,"data_stored":2403531,"data_compressed":103473,"data_compressed_allocated":135168,"data_compressed_original":2101248,"omap_allocated":0,"internal_metadata":0},"log_size":387,"ondisk_log_size":387,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710792,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":917504,"data_stored":828458,"data_compressed":27969,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877907021,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28096,"kb_used_data":1072,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717056,"statfs":{"total":95995035648,"available":95966265344,"internally_reserved":0,"allocated":1097728,"data_stored":1425646,"data_compressed":60009,"data_compressed_allocated":77824,"data_compressed_original":921600,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070546,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32088,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713064,"statfs":{"total":95995035648,"available":95962177536,"internally_reserved":0,"allocated":933888,"data_stored":827929,"data_compressed":27969,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266776,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":905216,"data_stored":435091,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863463006,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32112,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713040,"statfs":{"total":95995035648,"available":95962152960,"internally_reserved":0,"allocated":958464,"data_stored":1029624,"data_compressed":44807,"data_compressed_allocated":57344,"data_compressed_original":614400,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626532,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":892928,"data_stored":434821,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574976,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27960,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717192,"statfs":{"total":95995035648,"available":95966404608,"internally_reserved":0,"allocated":966656,"data_stored":506244,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738502,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":892,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":913408,"data_stored":436160,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411451,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":970752,"data_stored":507207,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182517,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27980,"kb_used_data":956,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717172,"statfs":{"total":95995035648,"available":95966384128,"internally_reserved":0,"allocated":978944,"data_stored":1155703,"data_compressed":43171,"data_compressed_allocated":57344,"data_compressed_original":724992,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986288,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27900,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717252,"statfs":{"total":95995035648,"available":95966466048,"internally_reserved":0,"allocated":901120,"data_stored":435495,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822762,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32176,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712976,"statfs":{"total":95995035648,"available":95962087424,"internally_reserved":0,"allocated":1015808,"data_stored":1356051,"data_compressed":60009,"data_compressed_allocated":77824,"data_compressed_original":921600,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":72653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":73616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":721878,"data_compressed":34491,"data_compressed_allocated":45056,"data_compressed_original":700416,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":331975,"data_compressed":15202,"data_compressed_allocated":20480,"data_compressed_original":307200,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":394338,"data_compressed":19289,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":172032,"data_stored":401570,"data_compressed":15202,"data_compressed_allocated":20480,"data_compressed_original":307200,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394867,"data_compressed":19289,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:06:03.499 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T20:06:03.975 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:03 smithi028 ceph-mon[31178]: from='client.25306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:04.053 INFO:tasks.cephfs.filesystem:scrub status for tag:9f22e454-6796-47fe-b349-05531a40a16c - {'path': '/', 'tag': '9f22e454-6796-47fe-b349-05531a40a16c', 'options': 'recursive,force'} 2024-05-20T20:06:04.106 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:04.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:03 smithi154 ceph-mon[36314]: from='client.25306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:04.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:03 smithi033 ceph-mon[36179]: from='client.25306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:05.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:04 smithi028 ceph-mon[31178]: pgmap v16: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 88 KiB/s wr, 45 op/s 2024-05-20T20:06:05.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:04 smithi154 ceph-mon[36314]: pgmap v16: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 88 KiB/s wr, 45 op/s 2024-05-20T20:06:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:04 smithi033 ceph-mon[36179]: pgmap v16: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 88 KiB/s wr, 45 op/s 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 -- 172.21.15.28:0/4220564430 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 msgr2=0x7f83d4116b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 --2- 172.21.15.28:0/4220564430 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 0x7f83d4116b00 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f83cc00b0a0 tx=0x7f83cc00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 -- 172.21.15.28:0/4220564430 shutdown_connections 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 --2- 172.21.15.28:0/4220564430 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 0x7f83d4116b00 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 --2- 172.21.15.28:0/4220564430 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4075770 0x7f83d4075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 --2- 172.21.15.28:0/4220564430 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4076090 0x7f83d41141d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 -- 172.21.15.28:0/4220564430 >> 172.21.15.28:0/4220564430 conn(0x7f83d407a6a0 msgr2=0x7f83d407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 -- 172.21.15.28:0/4220564430 shutdown_connections 2024-05-20T20:06:06.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.396+0000 7f83da400700 1 -- 172.21.15.28:0/4220564430 wait complete. 2024-05-20T20:06:06.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.397+0000 7f83da400700 1 Processor -- start 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83da400700 1 -- start start 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83da400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4075770 0x7f83d41a01f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83da400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4076090 0x7f83d41a0730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83da400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 0x7f83d41a4ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83da400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f83d4119d40 con 0x7f83d4114710 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83da400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f83d4119bc0 con 0x7f83d4076090 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83da400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f83d4119ec0 con 0x7f83d4075770 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83d2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4076090 0x7f83d41a0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.398+0000 7f83d3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4075770 0x7f83d41a01f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4075770 0x7f83d41a01f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55332/0 (socket says 172.21.15.28:55332) 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 -- 172.21.15.28:0/1376466008 learned_addr learned my addr 172.21.15.28:0/1376466008 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:06.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4076090 0x7f83d41a0730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53892/0 (socket says 172.21.15.28:53892) 2024-05-20T20:06:06.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3e00700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 0x7f83d41a4ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:06.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 -- 172.21.15.28:0/1376466008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4076090 msgr2=0x7f83d41a0730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4076090 0x7f83d41a0730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 -- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 msgr2=0x7f83d41a4ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 0x7f83d41a4ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 -- 172.21.15.28:0/1376466008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f83d41a5200 con 0x7f83d4075770 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d2a00700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4076090 0x7f83d41a0730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.399+0000 7f83d3400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4075770 0x7f83d41a01f0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f83c400e9a0 tx=0x7f83c400ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.400+0000 7f83cbe00700 1 -- 172.21.15.28:0/1376466008 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83c400ca10 con 0x7f83d4075770 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.400+0000 7f83cbe00700 1 -- 172.21.15.28:0/1376466008 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f83c400cbb0 con 0x7f83d4075770 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.400+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f83d41a54f0 con 0x7f83d4075770 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.400+0000 7f83cbe00700 1 -- 172.21.15.28:0/1376466008 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83c4018680 con 0x7f83d4075770 2024-05-20T20:06:06.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.400+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f83d41acd70 con 0x7f83d4075770 2024-05-20T20:06:06.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.401+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f83d406a230 con 0x7f83d4075770 2024-05-20T20:06:06.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.403+0000 7f83cbe00700 1 -- 172.21.15.28:0/1376466008 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f83c404abc0 con 0x7f83d4075770 2024-05-20T20:06:06.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.405+0000 7f83cbe00700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83b80b2ba0 0x7f83b80b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:06.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.405+0000 7f83cbe00700 1 -- 172.21.15.28:0/1376466008 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f83c40e2b30 con 0x7f83d4075770 2024-05-20T20:06:06.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.405+0000 7f83d2a00700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83b80b2ba0 0x7f83b80b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:06.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.406+0000 7f83d2a00700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83b80b2ba0 0x7f83b80b5060 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f83d4113bc0 tx=0x7f83c0009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:06.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.407+0000 7f83cbe00700 1 -- 172.21.15.28:0/1376466008 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f83c40aa2e0 con 0x7f83d4075770 2024-05-20T20:06:06.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.648+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f83d41108c0 con 0x7f83b80b2ba0 2024-05-20T20:06:06.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.668+0000 7f83cbe00700 1 -- 172.21.15.28:0/1376466008 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258091 (secure 0 0 0) 0x7f83d41108c0 con 0x7f83b80b2ba0 2024-05-20T20:06:06.677 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:06.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.675+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83b80b2ba0 msgr2=0x7f83b80b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:06.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.675+0000 7f83da400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83b80b2ba0 0x7f83b80b5060 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f83d4113bc0 tx=0x7f83c0009260 comp rx=0 tx=0).stop 2024-05-20T20:06:06.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.675+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4075770 msgr2=0x7f83d41a01f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.675+0000 7f83da400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4075770 0x7f83d41a01f0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f83c400e9a0 tx=0x7f83c400ecb0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 shutdown_connections 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83d4114710 0x7f83d41a4ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83d4075770 0x7f83d41a01f0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83b80b2ba0 0x7f83b80b5060 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 --2- 172.21.15.28:0/1376466008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83d4076090 0x7f83d41a0730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 >> 172.21.15.28:0/1376466008 conn(0x7f83d407a6a0 msgr2=0x7f83d41101b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 shutdown_connections 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:06.676+0000 7f83da400700 1 -- 172.21.15.28:0/1376466008 wait complete. 2024-05-20T20:06:06.680 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:06:07.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:06 smithi028 ceph-mon[31178]: pgmap v17: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 99 KiB/s wr, 57 op/s 2024-05-20T20:06:07.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:06 smithi154 ceph-mon[36314]: pgmap v17: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 99 KiB/s wr, 57 op/s 2024-05-20T20:06:07.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:06 smithi033 ceph-mon[36179]: pgmap v17: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 99 KiB/s wr, 57 op/s 2024-05-20T20:06:07.689 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2024-05-20T20:06:05.637530+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1393692,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":3237,"num_read_kb":2072,"num_write":710,"num_write_kb":4809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1027,"ondisk_log_size":1027,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347892,"kb_used_data":11184,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124593932,"statfs":{"total":1151940427776,"available":1151584186368,"internally_reserved":0,"allocated":11452416,"data_stored":9383916,"data_compressed":316038,"data_compressed_allocated":417792,"data_compressed_original":4165632,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":198545,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":499,"num_read_kb":406,"num_write":187,"num_write_kb":1183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005840"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603392+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:05:37.603392+0000","last_peered":"2024-05-20T20:05:37.603392+0000","last_clean":"2024-05-20T20:05:37.603392+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:05:37.603392+0000","last_undegraded":"2024-05-20T20:05:37.603392+0000","last_fullsized":"2024-05-20T20:05:37.603392+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750621+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:05:37.750621+0000","last_peered":"2024-05-20T20:05:37.750621+0000","last_clean":"2024-05-20T20:05:37.750621+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:05:37.750621+0000","last_undegraded":"2024-05-20T20:05:37.750621+0000","last_fullsized":"2024-05-20T20:05:37.750621+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750419+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:05:37.750419+0000","last_peered":"2024-05-20T20:05:37.750419+0000","last_clean":"2024-05-20T20:05:37.750419+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:05:37.750419+0000","last_undegraded":"2024-05-20T20:05:37.750419+0000","last_fullsized":"2024-05-20T20:05:37.750419+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946345+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:05:37.946345+0000","last_peered":"2024-05-20T20:05:37.946345+0000","last_clean":"2024-05-20T20:05:37.946345+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:05:37.946345+0000","last_undegraded":"2024-05-20T20:05:37.946345+0000","last_fullsized":"2024-05-20T20:05:37.946345+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":27,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950341+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:05:37.950341+0000","last_peered":"2024-05-20T20:05:37.950341+0000","last_clean":"2024-05-20T20:05:37.950341+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:05:37.950341+0000","last_undegraded":"2024-05-20T20:05:37.950341+0000","last_fullsized":"2024-05-20T20:05:37.950341+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601481+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:05:37.601481+0000","last_peered":"2024-05-20T20:05:37.601481+0000","last_clean":"2024-05-20T20:05:37.601481+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:05:37.601481+0000","last_undegraded":"2024-05-20T20:05:37.601481+0000","last_fullsized":"2024-05-20T20:05:37.601481+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946714+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:05:37.946714+0000","last_peered":"2024-05-20T20:05:37.946714+0000","last_clean":"2024-05-20T20:05:37.946714+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:05:37.946714+0000","last_undegraded":"2024-05-20T20:05:37.946714+0000","last_fullsized":"2024-05-20T20:05:37.946714+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740698+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:05:37.740698+0000","last_peered":"2024-05-20T20:05:37.740698+0000","last_clean":"2024-05-20T20:05:37.740698+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:05:37.740698+0000","last_undegraded":"2024-05-20T20:05:37.740698+0000","last_fullsized":"2024-05-20T20:05:37.740698+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.942016+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:05:37.942016+0000","last_peered":"2024-05-20T20:05:37.942016+0000","last_clean":"2024-05-20T20:05:37.942016+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:05:37.942016+0000","last_undegraded":"2024-05-20T20:05:37.942016+0000","last_fullsized":"2024-05-20T20:05:37.942016+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":45,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601908+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:05:37.601908+0000","last_peered":"2024-05-20T20:05:37.601908+0000","last_clean":"2024-05-20T20:05:37.601908+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:05:37.601908+0000","last_undegraded":"2024-05-20T20:05:37.601908+0000","last_fullsized":"2024-05-20T20:05:37.601908+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947610+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:05:37.947610+0000","last_peered":"2024-05-20T20:05:37.947610+0000","last_clean":"2024-05-20T20:05:37.947610+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:05:37.947610+0000","last_undegraded":"2024-05-20T20:05:37.947610+0000","last_fullsized":"2024-05-20T20:05:37.947610+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946190+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:05:37.946190+0000","last_peered":"2024-05-20T20:05:37.946190+0000","last_clean":"2024-05-20T20:05:37.946190+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:05:37.946190+0000","last_undegraded":"2024-05-20T20:05:37.946190+0000","last_fullsized":"2024-05-20T20:05:37.946190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951933+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:05:37.951933+0000","last_peered":"2024-05-20T20:05:37.951933+0000","last_clean":"2024-05-20T20:05:37.951933+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:05:37.951933+0000","last_undegraded":"2024-05-20T20:05:37.951933+0000","last_fullsized":"2024-05-20T20:05:37.951933+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749970+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:05:37.749970+0000","last_peered":"2024-05-20T20:05:37.749970+0000","last_clean":"2024-05-20T20:05:37.749970+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:05:37.749970+0000","last_undegraded":"2024-05-20T20:05:37.749970+0000","last_fullsized":"2024-05-20T20:05:37.749970+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749804+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:05:37.749804+0000","last_peered":"2024-05-20T20:05:37.749804+0000","last_clean":"2024-05-20T20:05:37.749804+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:05:37.749804+0000","last_undegraded":"2024-05-20T20:05:37.749804+0000","last_fullsized":"2024-05-20T20:05:37.749804+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941202+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:05:37.941202+0000","last_peered":"2024-05-20T20:05:37.941202+0000","last_clean":"2024-05-20T20:05:37.941202+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:05:37.941202+0000","last_undegraded":"2024-05-20T20:05:37.941202+0000","last_fullsized":"2024-05-20T20:05:37.941202+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603526+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:05:37.603526+0000","last_peered":"2024-05-20T20:05:37.603526+0000","last_clean":"2024-05-20T20:05:37.603526+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:05:37.603526+0000","last_undegraded":"2024-05-20T20:05:37.603526+0000","last_fullsized":"2024-05-20T20:05:37.603526+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602307+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:05:37.602307+0000","last_peered":"2024-05-20T20:05:37.602307+0000","last_clean":"2024-05-20T20:05:37.602307+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:05:37.602307+0000","last_undegraded":"2024-05-20T20:05:37.602307+0000","last_fullsized":"2024-05-20T20:05:37.602307+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940697+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:05:37.940697+0000","last_peered":"2024-05-20T20:05:37.940697+0000","last_clean":"2024-05-20T20:05:37.940697+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:05:37.940697+0000","last_undegraded":"2024-05-20T20:05:37.940697+0000","last_fullsized":"2024-05-20T20:05:37.940697+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'14","reported_seq":303,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:02.273649+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:06:02.273649+0000","last_peered":"2024-05-20T20:06:02.273649+0000","last_clean":"2024-05-20T20:06:02.273649+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:06:02.273649+0000","last_undegraded":"2024-05-20T20:06:02.273649+0000","last_fullsized":"2024-05-20T20:06:02.273649+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":259,"num_read_kb":259,"num_write":15,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741585+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:05:37.741585+0000","last_peered":"2024-05-20T20:05:37.741585+0000","last_clean":"2024-05-20T20:05:37.741585+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:05:37.741585+0000","last_undegraded":"2024-05-20T20:05:37.741585+0000","last_fullsized":"2024-05-20T20:05:37.741585+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741764+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:05:37.741764+0000","last_peered":"2024-05-20T20:05:37.741764+0000","last_clean":"2024-05-20T20:05:37.741764+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:05:37.741764+0000","last_undegraded":"2024-05-20T20:05:37.741764+0000","last_fullsized":"2024-05-20T20:05:37.741764+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750654+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:05:37.750654+0000","last_peered":"2024-05-20T20:05:37.750654+0000","last_clean":"2024-05-20T20:05:37.750654+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:05:37.750654+0000","last_undegraded":"2024-05-20T20:05:37.750654+0000","last_fullsized":"2024-05-20T20:05:37.750654+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'13","reported_seq":43,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950989+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:05:37.950989+0000","last_peered":"2024-05-20T20:05:37.950989+0000","last_clean":"2024-05-20T20:05:37.950989+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:05:37.950989+0000","last_undegraded":"2024-05-20T20:05:37.950989+0000","last_fullsized":"2024-05-20T20:05:37.950989+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":18,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950749+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:05:37.950749+0000","last_peered":"2024-05-20T20:05:37.950749+0000","last_clean":"2024-05-20T20:05:37.950749+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:05:37.950749+0000","last_undegraded":"2024-05-20T20:05:37.950749+0000","last_fullsized":"2024-05-20T20:05:37.950749+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602506+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:05:37.602506+0000","last_peered":"2024-05-20T20:05:37.602506+0000","last_clean":"2024-05-20T20:05:37.602506+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:05:37.602506+0000","last_undegraded":"2024-05-20T20:05:37.602506+0000","last_fullsized":"2024-05-20T20:05:37.602506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"81'8","reported_seq":299,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:02.068094+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:06:02.068094+0000","last_peered":"2024-05-20T20:06:02.068094+0000","last_clean":"2024-05-20T20:06:02.068094+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:06:02.068094+0000","last_undegraded":"2024-05-20T20:06:02.068094+0000","last_fullsized":"2024-05-20T20:06:02.068094+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1545,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":262,"num_read_kb":5,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950345+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:05:37.950345+0000","last_peered":"2024-05-20T20:05:37.950345+0000","last_clean":"2024-05-20T20:05:37.950345+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:05:37.950345+0000","last_undegraded":"2024-05-20T20:05:37.950345+0000","last_fullsized":"2024-05-20T20:05:37.950345+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601320+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:05:37.601320+0000","last_peered":"2024-05-20T20:05:37.601320+0000","last_clean":"2024-05-20T20:05:37.601320+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:05:37.601320+0000","last_undegraded":"2024-05-20T20:05:37.601320+0000","last_fullsized":"2024-05-20T20:05:37.601320+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751407+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:05:37.751407+0000","last_peered":"2024-05-20T20:05:37.751407+0000","last_clean":"2024-05-20T20:05:37.751407+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:05:37.751407+0000","last_undegraded":"2024-05-20T20:05:37.751407+0000","last_fullsized":"2024-05-20T20:05:37.751407+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950714+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:05:37.950714+0000","last_peered":"2024-05-20T20:05:37.950714+0000","last_clean":"2024-05-20T20:05:37.950714+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:05:37.950714+0000","last_undegraded":"2024-05-20T20:05:37.950714+0000","last_fullsized":"2024-05-20T20:05:37.950714+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740756+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:05:37.740756+0000","last_peered":"2024-05-20T20:05:37.740756+0000","last_clean":"2024-05-20T20:05:37.740756+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:05:37.740756+0000","last_undegraded":"2024-05-20T20:05:37.740756+0000","last_fullsized":"2024-05-20T20:05:37.740756+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751178+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:05:37.751178+0000","last_peered":"2024-05-20T20:05:37.751178+0000","last_clean":"2024-05-20T20:05:37.751178+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:05:37.751178+0000","last_undegraded":"2024-05-20T20:05:37.751178+0000","last_fullsized":"2024-05-20T20:05:37.751178+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601398+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:05:37.601398+0000","last_peered":"2024-05-20T20:05:37.601398+0000","last_clean":"2024-05-20T20:05:37.601398+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:05:37.601398+0000","last_undegraded":"2024-05-20T20:05:37.601398+0000","last_fullsized":"2024-05-20T20:05:37.601398+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'7","reported_seq":550,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:03.068821+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:06:03.068821+0000","last_peered":"2024-05-20T20:06:03.068821+0000","last_clean":"2024-05-20T20:06:03.068821+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:06:03.068821+0000","last_undegraded":"2024-05-20T20:06:03.068821+0000","last_fullsized":"2024-05-20T20:06:03.068821+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":514,"num_read_kb":514,"num_write":8,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941795+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:05:37.941795+0000","last_peered":"2024-05-20T20:05:37.941795+0000","last_clean":"2024-05-20T20:05:37.941795+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:05:37.941795+0000","last_undegraded":"2024-05-20T20:05:37.941795+0000","last_fullsized":"2024-05-20T20:05:37.941795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947471+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:05:37.947471+0000","last_peered":"2024-05-20T20:05:37.947471+0000","last_clean":"2024-05-20T20:05:37.947471+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:05:37.947471+0000","last_undegraded":"2024-05-20T20:05:37.947471+0000","last_fullsized":"2024-05-20T20:05:37.947471+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750655+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:05:37.750655+0000","last_peered":"2024-05-20T20:05:37.750655+0000","last_clean":"2024-05-20T20:05:37.750655+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:05:37.750655+0000","last_undegraded":"2024-05-20T20:05:37.750655+0000","last_fullsized":"2024-05-20T20:05:37.750655+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751776+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:05:37.751776+0000","last_peered":"2024-05-20T20:05:37.751776+0000","last_clean":"2024-05-20T20:05:37.751776+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:05:37.751776+0000","last_undegraded":"2024-05-20T20:05:37.751776+0000","last_fullsized":"2024-05-20T20:05:37.751776+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941312+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:05:37.941312+0000","last_peered":"2024-05-20T20:05:37.941312+0000","last_clean":"2024-05-20T20:05:37.941312+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:05:37.941312+0000","last_undegraded":"2024-05-20T20:05:37.941312+0000","last_fullsized":"2024-05-20T20:05:37.941312+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'44","reported_seq":605,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:03.070464+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:06:03.070464+0000","last_peered":"2024-05-20T20:06:03.070464+0000","last_clean":"2024-05-20T20:06:03.070464+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:06:03.070464+0000","last_undegraded":"2024-05-20T20:06:03.070464+0000","last_fullsized":"2024-05-20T20:06:03.070464+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":44,"log_dups_size":0,"ondisk_log_size":44,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72032,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":531,"num_read_kb":337,"num_write":46,"num_write_kb":90,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950750+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:05:37.950750+0000","last_peered":"2024-05-20T20:05:37.950750+0000","last_clean":"2024-05-20T20:05:37.950750+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:05:37.950750+0000","last_undegraded":"2024-05-20T20:05:37.950750+0000","last_fullsized":"2024-05-20T20:05:37.950750+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602898+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:05:37.602898+0000","last_peered":"2024-05-20T20:05:37.602898+0000","last_clean":"2024-05-20T20:05:37.602898+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:05:37.602898+0000","last_undegraded":"2024-05-20T20:05:37.602898+0000","last_fullsized":"2024-05-20T20:05:37.602898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950860+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:05:37.950860+0000","last_peered":"2024-05-20T20:05:37.950860+0000","last_clean":"2024-05-20T20:05:37.950860+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:05:37.950860+0000","last_undegraded":"2024-05-20T20:05:37.950860+0000","last_fullsized":"2024-05-20T20:05:37.950860+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751331+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:05:37.751331+0000","last_peered":"2024-05-20T20:05:37.751331+0000","last_clean":"2024-05-20T20:05:37.751331+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:05:37.751331+0000","last_undegraded":"2024-05-20T20:05:37.751331+0000","last_fullsized":"2024-05-20T20:05:37.751331+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947438+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:05:37.947438+0000","last_peered":"2024-05-20T20:05:37.947438+0000","last_clean":"2024-05-20T20:05:37.947438+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:05:37.947438+0000","last_undegraded":"2024-05-20T20:05:37.947438+0000","last_fullsized":"2024-05-20T20:05:37.947438+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'9","reported_seq":302,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:03.277959+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:06:03.277959+0000","last_peered":"2024-05-20T20:06:03.277959+0000","last_clean":"2024-05-20T20:06:03.277959+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:06:03.277959+0000","last_undegraded":"2024-05-20T20:06:03.277959+0000","last_fullsized":"2024-05-20T20:06:03.277959+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2071,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":264,"num_read_kb":7,"num_write":9,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946197+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:05:37.946197+0000","last_peered":"2024-05-20T20:05:37.946197+0000","last_clean":"2024-05-20T20:05:37.946197+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:05:37.946197+0000","last_undegraded":"2024-05-20T20:05:37.946197+0000","last_fullsized":"2024-05-20T20:05:37.946197+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":72,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950363+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:05:37.950363+0000","last_peered":"2024-05-20T20:05:37.950363+0000","last_clean":"2024-05-20T20:05:37.950363+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:05:37.950363+0000","last_undegraded":"2024-05-20T20:05:37.950363+0000","last_fullsized":"2024-05-20T20:05:37.950363+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751376+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:05:37.751376+0000","last_peered":"2024-05-20T20:05:37.751376+0000","last_clean":"2024-05-20T20:05:37.751376+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:05:37.751376+0000","last_undegraded":"2024-05-20T20:05:37.751376+0000","last_fullsized":"2024-05-20T20:05:37.751376+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'5","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:39.658956+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:05:39.658956+0000","last_peered":"2024-05-20T20:05:39.658956+0000","last_clean":"2024-05-20T20:05:39.658956+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:05:39.658956+0000","last_undegraded":"2024-05-20T20:05:39.658956+0000","last_fullsized":"2024-05-20T20:05:39.658956+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603167+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:05:37.603167+0000","last_peered":"2024-05-20T20:05:37.603167+0000","last_clean":"2024-05-20T20:05:37.603167+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:05:37.603167+0000","last_undegraded":"2024-05-20T20:05:37.603167+0000","last_fullsized":"2024-05-20T20:05:37.603167+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602390+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:05:37.602390+0000","last_peered":"2024-05-20T20:05:37.602390+0000","last_clean":"2024-05-20T20:05:37.602390+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:05:37.602390+0000","last_undegraded":"2024-05-20T20:05:37.602390+0000","last_fullsized":"2024-05-20T20:05:37.602390+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601741+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:05:37.601741+0000","last_peered":"2024-05-20T20:05:37.601741+0000","last_clean":"2024-05-20T20:05:37.601741+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:05:37.601741+0000","last_undegraded":"2024-05-20T20:05:37.601741+0000","last_fullsized":"2024-05-20T20:05:37.601741+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'6","reported_seq":295,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:02.274570+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:06:02.274570+0000","last_peered":"2024-05-20T20:06:02.274570+0000","last_clean":"2024-05-20T20:06:02.274570+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:06:02.274570+0000","last_undegraded":"2024-05-20T20:06:02.274570+0000","last_fullsized":"2024-05-20T20:06:02.274570+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":259,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601417+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:05:37.601417+0000","last_peered":"2024-05-20T20:05:37.601417+0000","last_clean":"2024-05-20T20:05:37.601417+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:05:37.601417+0000","last_undegraded":"2024-05-20T20:05:37.601417+0000","last_fullsized":"2024-05-20T20:05:37.601417+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751541+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:05:37.751541+0000","last_peered":"2024-05-20T20:05:37.751541+0000","last_clean":"2024-05-20T20:05:37.751541+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:05:37.751541+0000","last_undegraded":"2024-05-20T20:05:37.751541+0000","last_fullsized":"2024-05-20T20:05:37.751541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"81'7","reported_seq":295,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:02.274439+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:06:02.274439+0000","last_peered":"2024-05-20T20:06:02.274439+0000","last_clean":"2024-05-20T20:06:02.274439+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:06:02.274439+0000","last_undegraded":"2024-05-20T20:06:02.274439+0000","last_fullsized":"2024-05-20T20:06:02.274439+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":258,"num_read_kb":258,"num_write":7,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741186+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:05:37.741186+0000","last_peered":"2024-05-20T20:05:37.741186+0000","last_clean":"2024-05-20T20:05:37.741186+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:05:37.741186+0000","last_undegraded":"2024-05-20T20:05:37.741186+0000","last_fullsized":"2024-05-20T20:05:37.741186+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.945964+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:05:37.945964+0000","last_peered":"2024-05-20T20:05:37.945964+0000","last_clean":"2024-05-20T20:05:37.945964+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:05:37.945964+0000","last_undegraded":"2024-05-20T20:05:37.945964+0000","last_fullsized":"2024-05-20T20:05:37.945964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"81'99","reported_seq":131,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:39.743703+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:05:39.743703+0000","last_peered":"2024-05-20T20:05:39.743703+0000","last_clean":"2024-05-20T20:05:39.743703+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:05:39.743703+0000","last_undegraded":"2024-05-20T20:05:39.743703+0000","last_fullsized":"2024-05-20T20:05:39.743703+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950636+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:05:37.950636+0000","last_peered":"2024-05-20T20:05:37.950636+0000","last_clean":"2024-05-20T20:05:37.950636+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:05:37.950636+0000","last_undegraded":"2024-05-20T20:05:37.950636+0000","last_fullsized":"2024-05-20T20:05:37.950636+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946570+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:05:37.946570+0000","last_peered":"2024-05-20T20:05:37.946570+0000","last_clean":"2024-05-20T20:05:37.946570+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:05:37.946570+0000","last_undegraded":"2024-05-20T20:05:37.946570+0000","last_fullsized":"2024-05-20T20:05:37.946570+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950840+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:05:37.950840+0000","last_peered":"2024-05-20T20:05:37.950840+0000","last_clean":"2024-05-20T20:05:37.950840+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:05:37.950840+0000","last_undegraded":"2024-05-20T20:05:37.950840+0000","last_fullsized":"2024-05-20T20:05:37.950840+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952108+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:05:37.952108+0000","last_peered":"2024-05-20T20:05:37.952108+0000","last_clean":"2024-05-20T20:05:37.952108+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:05:37.952108+0000","last_undegraded":"2024-05-20T20:05:37.952108+0000","last_fullsized":"2024-05-20T20:05:37.952108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'129","reported_seq":300,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:01.140633+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:06:01.140633+0000","last_peered":"2024-05-20T20:06:01.140633+0000","last_clean":"2024-05-20T20:06:01.140633+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:06:01.140633+0000","last_undegraded":"2024-05-20T20:06:01.140633+0000","last_fullsized":"2024-05-20T20:06:01.140633+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":129,"log_dups_size":0,"ondisk_log_size":129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":422,"num_read_kb":376,"num_write":238,"num_write_kb":1415,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741294+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:05:37.741294+0000","last_peered":"2024-05-20T20:05:37.741294+0000","last_clean":"2024-05-20T20:05:37.741294+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:05:37.741294+0000","last_undegraded":"2024-05-20T20:05:37.741294+0000","last_fullsized":"2024-05-20T20:05:37.741294+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946101+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:05:37.946101+0000","last_peered":"2024-05-20T20:05:37.946101+0000","last_clean":"2024-05-20T20:05:37.946101+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:05:37.946101+0000","last_undegraded":"2024-05-20T20:05:37.946101+0000","last_fullsized":"2024-05-20T20:05:37.946101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750609+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:05:37.750609+0000","last_peered":"2024-05-20T20:05:37.750609+0000","last_clean":"2024-05-20T20:05:37.750609+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:05:37.750609+0000","last_undegraded":"2024-05-20T20:05:37.750609+0000","last_fullsized":"2024-05-20T20:05:37.750609+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941387+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:05:37.941387+0000","last_peered":"2024-05-20T20:05:37.941387+0000","last_clean":"2024-05-20T20:05:37.941387+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:05:37.941387+0000","last_undegraded":"2024-05-20T20:05:37.941387+0000","last_fullsized":"2024-05-20T20:05:37.941387+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":26,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950366+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:05:37.950366+0000","last_peered":"2024-05-20T20:05:37.950366+0000","last_clean":"2024-05-20T20:05:37.950366+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:05:37.950366+0000","last_undegraded":"2024-05-20T20:05:37.950366+0000","last_fullsized":"2024-05-20T20:05:37.950366+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741016+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:05:37.741016+0000","last_peered":"2024-05-20T20:05:37.741016+0000","last_clean":"2024-05-20T20:05:37.741016+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:05:37.741016+0000","last_undegraded":"2024-05-20T20:05:37.741016+0000","last_fullsized":"2024-05-20T20:05:37.741016+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602899+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:05:37.602899+0000","last_peered":"2024-05-20T20:05:37.602899+0000","last_clean":"2024-05-20T20:05:37.602899+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:05:37.602899+0000","last_undegraded":"2024-05-20T20:05:37.602899+0000","last_fullsized":"2024-05-20T20:05:37.602899+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940518+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:05:37.940518+0000","last_peered":"2024-05-20T20:05:37.940518+0000","last_clean":"2024-05-20T20:05:37.940518+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:05:37.940518+0000","last_undegraded":"2024-05-20T20:05:37.940518+0000","last_fullsized":"2024-05-20T20:05:37.940518+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741426+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:05:37.741426+0000","last_peered":"2024-05-20T20:05:37.741426+0000","last_clean":"2024-05-20T20:05:37.741426+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:05:37.741426+0000","last_undegraded":"2024-05-20T20:05:37.741426+0000","last_fullsized":"2024-05-20T20:05:37.741426+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950516+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:05:37.950516+0000","last_peered":"2024-05-20T20:05:37.950516+0000","last_clean":"2024-05-20T20:05:37.950516+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:05:37.950516+0000","last_undegraded":"2024-05-20T20:05:37.950516+0000","last_fullsized":"2024-05-20T20:05:37.950516+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947502+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:05:37.947502+0000","last_peered":"2024-05-20T20:05:37.947502+0000","last_clean":"2024-05-20T20:05:37.947502+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:05:37.947502+0000","last_undegraded":"2024-05-20T20:05:37.947502+0000","last_fullsized":"2024-05-20T20:05:37.947502+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'6","reported_seq":296,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:02.067976+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:06:02.067976+0000","last_peered":"2024-05-20T20:06:02.067976+0000","last_clean":"2024-05-20T20:06:02.067976+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:06:02.067976+0000","last_undegraded":"2024-05-20T20:06:02.067976+0000","last_fullsized":"2024-05-20T20:06:02.067976+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1561,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":260,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947819+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:05:37.947819+0000","last_peered":"2024-05-20T20:05:37.947819+0000","last_clean":"2024-05-20T20:05:37.947819+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:05:37.947819+0000","last_undegraded":"2024-05-20T20:05:37.947819+0000","last_fullsized":"2024-05-20T20:05:37.947819+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"81'75","reported_seq":147,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:01.126122+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:06:01.126122+0000","last_peered":"2024-05-20T20:06:01.126122+0000","last_clean":"2024-05-20T20:06:01.126122+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:06:01.126122+0000","last_undegraded":"2024-05-20T20:06:01.126122+0000","last_fullsized":"2024-05-20T20:06:01.126122+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":75,"log_dups_size":0,"ondisk_log_size":75,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328302,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":45,"num_read_kb":44,"num_write":84,"num_write_kb":1075,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947490+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:05:37.947490+0000","last_peered":"2024-05-20T20:05:37.947490+0000","last_clean":"2024-05-20T20:05:37.947490+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:05:37.947490+0000","last_undegraded":"2024-05-20T20:05:37.947490+0000","last_fullsized":"2024-05-20T20:05:37.947490+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952372+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:05:37.952372+0000","last_peered":"2024-05-20T20:05:37.952372+0000","last_clean":"2024-05-20T20:05:37.952372+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:05:37.952372+0000","last_undegraded":"2024-05-20T20:05:37.952372+0000","last_fullsized":"2024-05-20T20:05:37.952372+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947860+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:05:37.947860+0000","last_peered":"2024-05-20T20:05:37.947860+0000","last_clean":"2024-05-20T20:05:37.947860+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:05:37.947860+0000","last_undegraded":"2024-05-20T20:05:37.947860+0000","last_fullsized":"2024-05-20T20:05:37.947860+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603108+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:05:37.603108+0000","last_peered":"2024-05-20T20:05:37.603108+0000","last_clean":"2024-05-20T20:05:37.603108+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:05:37.603108+0000","last_undegraded":"2024-05-20T20:05:37.603108+0000","last_fullsized":"2024-05-20T20:05:37.603108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940537+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:05:37.940537+0000","last_peered":"2024-05-20T20:05:37.940537+0000","last_clean":"2024-05-20T20:05:37.940537+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:05:37.940537+0000","last_undegraded":"2024-05-20T20:05:37.940537+0000","last_fullsized":"2024-05-20T20:05:37.940537+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940554+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:05:37.940554+0000","last_peered":"2024-05-20T20:05:37.940554+0000","last_clean":"2024-05-20T20:05:37.940554+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:05:37.940554+0000","last_undegraded":"2024-05-20T20:05:37.940554+0000","last_fullsized":"2024-05-20T20:05:37.940554+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750191+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:05:37.750191+0000","last_peered":"2024-05-20T20:05:37.750191+0000","last_clean":"2024-05-20T20:05:37.750191+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:05:37.750191+0000","last_undegraded":"2024-05-20T20:05:37.750191+0000","last_fullsized":"2024-05-20T20:05:37.750191+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952180+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:05:37.952180+0000","last_peered":"2024-05-20T20:05:37.952180+0000","last_clean":"2024-05-20T20:05:37.952180+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:05:37.952180+0000","last_undegraded":"2024-05-20T20:05:37.952180+0000","last_fullsized":"2024-05-20T20:05:37.952180+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603224+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:05:37.603224+0000","last_peered":"2024-05-20T20:05:37.603224+0000","last_clean":"2024-05-20T20:05:37.603224+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:05:37.603224+0000","last_undegraded":"2024-05-20T20:05:37.603224+0000","last_fullsized":"2024-05-20T20:05:37.603224+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940832+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:05:37.940832+0000","last_peered":"2024-05-20T20:05:37.940832+0000","last_clean":"2024-05-20T20:05:37.940832+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:05:37.940832+0000","last_undegraded":"2024-05-20T20:05:37.940832+0000","last_fullsized":"2024-05-20T20:05:37.940832+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":26,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741159+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:05:37.741159+0000","last_peered":"2024-05-20T20:05:37.741159+0000","last_clean":"2024-05-20T20:05:37.741159+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:05:37.741159+0000","last_undegraded":"2024-05-20T20:05:37.741159+0000","last_fullsized":"2024-05-20T20:05:37.741159+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751338+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:05:37.751338+0000","last_peered":"2024-05-20T20:05:37.751338+0000","last_clean":"2024-05-20T20:05:37.751338+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:05:37.751338+0000","last_undegraded":"2024-05-20T20:05:37.751338+0000","last_fullsized":"2024-05-20T20:05:37.751338+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740731+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:05:37.740731+0000","last_peered":"2024-05-20T20:05:37.740731+0000","last_clean":"2024-05-20T20:05:37.740731+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:05:37.740731+0000","last_undegraded":"2024-05-20T20:05:37.740731+0000","last_fullsized":"2024-05-20T20:05:37.740731+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941843+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:05:37.941843+0000","last_peered":"2024-05-20T20:05:37.941843+0000","last_clean":"2024-05-20T20:05:37.941843+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:05:37.941843+0000","last_undegraded":"2024-05-20T20:05:37.941843+0000","last_fullsized":"2024-05-20T20:05:37.941843+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951652+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:05:37.951652+0000","last_peered":"2024-05-20T20:05:37.951652+0000","last_clean":"2024-05-20T20:05:37.951652+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:05:37.951652+0000","last_undegraded":"2024-05-20T20:05:37.951652+0000","last_fullsized":"2024-05-20T20:05:37.951652+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602990+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:05:37.602990+0000","last_peered":"2024-05-20T20:05:37.602990+0000","last_clean":"2024-05-20T20:05:37.602990+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:05:37.602990+0000","last_undegraded":"2024-05-20T20:05:37.602990+0000","last_fullsized":"2024-05-20T20:05:37.602990+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":108381,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":803090,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":3093,"num_read_kb":1824,"num_write":573,"num_write_kb":2657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":811008,"data_stored":2409270,"data_compressed":103497,"data_compressed_allocated":135168,"data_compressed_original":2101248,"omap_allocated":0,"internal_metadata":0},"log_size":426,"ondisk_log_size":426,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710793,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":917504,"data_stored":828458,"data_compressed":27977,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877907022,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28096,"kb_used_data":1072,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717056,"statfs":{"total":95995035648,"available":95966265344,"internally_reserved":0,"allocated":1097728,"data_stored":1427475,"data_compressed":60009,"data_compressed_allocated":77824,"data_compressed_original":921600,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070547,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32088,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713064,"statfs":{"total":95995035648,"available":95962177536,"internally_reserved":0,"allocated":933888,"data_stored":827929,"data_compressed":27977,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266777,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":905216,"data_stored":435091,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863463007,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32112,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713040,"statfs":{"total":95995035648,"available":95962152960,"internally_reserved":0,"allocated":958464,"data_stored":1030015,"data_compressed":44807,"data_compressed_allocated":57344,"data_compressed_original":614400,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626533,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":892928,"data_stored":434821,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574977,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27960,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717192,"statfs":{"total":95995035648,"available":95966404608,"internally_reserved":0,"allocated":966656,"data_stored":507682,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738503,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":892,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":913408,"data_stored":436160,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411452,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":970752,"data_stored":508645,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182518,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27980,"kb_used_data":956,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717172,"statfs":{"total":95995035648,"available":95966384128,"internally_reserved":0,"allocated":978944,"data_stored":1155703,"data_compressed":43179,"data_compressed_allocated":57344,"data_compressed_original":724992,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986289,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":435886,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822763,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32176,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712976,"statfs":{"total":95995035648,"available":95962087424,"internally_reserved":0,"allocated":1015808,"data_stored":1356051,"data_compressed":60009,"data_compressed_allocated":77824,"data_compressed_original":921600,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":74175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":75138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":721878,"data_compressed":34499,"data_compressed_allocated":45056,"data_compressed_original":700416,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":2295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":331975,"data_compressed":15202,"data_compressed_allocated":20480,"data_compressed_original":307200,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":394338,"data_compressed":19297,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":172032,"data_stored":403483,"data_compressed":15202,"data_compressed_allocated":20480,"data_compressed_original":307200,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394867,"data_compressed":19297,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:06:07.698 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-20T20:06:07.698 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T20:06:08.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:07 smithi028 ceph-mon[31178]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:08.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:07 smithi154 ceph-mon[36314]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:08.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:07 smithi033 ceph-mon[36179]: from='client.25330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:08.163 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:08 smithi028 ceph-mon[31178]: pgmap v18: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 99 KiB/s wr, 55 op/s 2024-05-20T20:06:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:08 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:06:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:08 smithi154 ceph-mon[36314]: pgmap v18: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 99 KiB/s wr, 55 op/s 2024-05-20T20:06:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:08 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:06:09.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:08 smithi033 ceph-mon[36179]: pgmap v18: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 99 KiB/s wr, 55 op/s 2024-05-20T20:06:09.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:08 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:06:10.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 -- 172.21.15.28:0/3732786361 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c10b760 msgr2=0x7f744c10bb40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:10.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 --2- 172.21.15.28:0/3732786361 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c10b760 0x7f744c10bb40 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f7440009990 tx=0x7f7440009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 -- 172.21.15.28:0/3732786361 shutdown_connections 2024-05-20T20:06:10.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 --2- 172.21.15.28:0/3732786361 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f744c106120 0x7f744c112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 --2- 172.21.15.28:0/3732786361 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c10b760 0x7f744c10bb40 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 --2- 172.21.15.28:0/3732786361 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 0x7f744c105be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 -- 172.21.15.28:0/3732786361 >> 172.21.15.28:0/3732786361 conn(0x7f744c101250 msgr2=0x7f744c103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:10.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 -- 172.21.15.28:0/3732786361 shutdown_connections 2024-05-20T20:06:10.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.504+0000 7f7454200700 1 -- 172.21.15.28:0/3732786361 wait complete. 2024-05-20T20:06:10.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 Processor -- start 2024-05-20T20:06:10.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 -- start start 2024-05-20T20:06:10.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 0x7f744c19bea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:10.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c106120 0x7f744c19c3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f744c10b760 0x7f744c1a0770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f744c1159b0 con 0x7f744c10b760 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f744c115830 con 0x7f744c105760 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.506+0000 7f7454200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f744c115b30 con 0x7f744c106120 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 0x7f744c19bea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 0x7f744c19bea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48760/0 (socket says 172.21.15.28:48760) 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 -- 172.21.15.28:0/1880161457 learned_addr learned my addr 172.21.15.28:0/1880161457 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:10.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7452200700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f744c10b760 0x7f744c1a0770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7450e00700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c106120 0x7f744c19c3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 -- 172.21.15.28:0/1880161457 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c106120 msgr2=0x7f744c19c3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c106120 0x7f744c19c3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 -- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f744c10b760 msgr2=0x7f744c1a0770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f744c10b760 0x7f744c1a0770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 -- 172.21.15.28:0/1880161457 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f744c1a0eb0 con 0x7f744c105760 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.507+0000 7f7451800700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 0x7f744c19bea0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f7440000c00 tx=0x7f74400057c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.508+0000 7f744a000700 1 -- 172.21.15.28:0/1880161457 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7440027070 con 0x7f744c105760 2024-05-20T20:06:10.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.508+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f744c1a1140 con 0x7f744c105760 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.508+0000 7f744a000700 1 -- 172.21.15.28:0/1880161457 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7440022470 con 0x7f744c105760 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.508+0000 7f744a000700 1 -- 172.21.15.28:0/1880161457 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7440017650 con 0x7f744c105760 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.508+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f744c1a8a20 con 0x7f744c105760 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.509+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f744c10dcf0 con 0x7f744c105760 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.512+0000 7f744a000700 1 -- 172.21.15.28:0/1880161457 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f744002a070 con 0x7f744c105760 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.513+0000 7f744a000700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74340b2c20 0x7f74340b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.513+0000 7f744a000700 1 -- 172.21.15.28:0/1880161457 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f74400e3470 con 0x7f744c105760 2024-05-20T20:06:10.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.513+0000 7f7450e00700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74340b2c20 0x7f74340b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:10.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.514+0000 7f7450e00700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74340b2c20 0x7f74340b50e0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f744c1038f0 tx=0x7f7438009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:10.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.515+0000 7f744a000700 1 -- 172.21.15.28:0/1880161457 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7440013070 con 0x7f744c105760 2024-05-20T20:06:10.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.749+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f744c19d200 con 0x7f74340b2c20 2024-05-20T20:06:10.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.757+0000 7f744a000700 1 -- 172.21.15.28:0/1880161457 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+258091 (secure 0 0 0) 0x7f744c19d200 con 0x7f74340b2c20 2024-05-20T20:06:10.758 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:10 smithi028 ceph-mon[31178]: pgmap v19: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 99 KiB/s wr, 59 op/s 2024-05-20T20:06:10.763 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74340b2c20 msgr2=0x7f74340b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74340b2c20 0x7f74340b50e0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f744c1038f0 tx=0x7f7438009260 comp rx=0 tx=0).stop 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 msgr2=0x7f744c19bea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 0x7f744c19bea0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f7440000c00 tx=0x7f74400057c0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 shutdown_connections 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f744c10b760 0x7f744c1a0770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74340b2c20 0x7f74340b50e0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f744c105760 0x7f744c19bea0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 --2- 172.21.15.28:0/1880161457 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f744c106120 0x7f744c19c3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:10.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 >> 172.21.15.28:0/1880161457 conn(0x7f744c101250 msgr2=0x7f744c101aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 shutdown_connections 2024-05-20T20:06:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:10.763+0000 7f7454200700 1 -- 172.21.15.28:0/1880161457 wait complete. 2024-05-20T20:06:10.766 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-05-20T20:06:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:10 smithi154 ceph-mon[36314]: pgmap v19: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 99 KiB/s wr, 59 op/s 2024-05-20T20:06:11.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:10 smithi033 ceph-mon[36179]: pgmap v19: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 99 KiB/s wr, 59 op/s 2024-05-20T20:06:11.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 -- 172.21.15.28:0/2961668020 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 msgr2=0x7f5d0010f300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2961668020 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d0010f300 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f5cf0009930 tx=0x7f5cf002a420 comp rx=0 tx=0).stop 2024-05-20T20:06:11.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 -- 172.21.15.28:0/2961668020 shutdown_connections 2024-05-20T20:06:11.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2961668020 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5d00109910 0x7f5d00115d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2961668020 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5d00108f90 0x7f5d001093d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2961668020 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d0010f300 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 -- 172.21.15.28:0/2961668020 >> 172.21.15.28:0/2961668020 conn(0x7f5d00078830 msgr2=0x7f5d00107870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:11.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 -- 172.21.15.28:0/2961668020 shutdown_connections 2024-05-20T20:06:11.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.483+0000 7f5d04a00640 1 -- 172.21.15.28:0/2961668020 wait complete. 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.484+0000 7f5d04a00640 1 Processor -- start 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.484+0000 7f5d04a00640 1 -- start start 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5d04a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5d00108f90 0x7f5d0019f660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5d04a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5d00109910 0x7f5d0019fba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5d04a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d001a3f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5d04a00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5d00119120 con 0x7f5d0010ef20 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5d04a00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5d00118fa0 con 0x7f5d00108f90 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5d04a00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5d001192a0 con 0x7f5d00109910 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d001a3f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5d00109910 0x7f5d0019fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d001a3f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41970/0 (socket says 172.21.15.28:41970) 2024-05-20T20:06:11.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 -- 172.21.15.28:0/2571802499 learned_addr learned my addr 172.21.15.28:0/2571802499 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:11.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cfe800640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5d00108f90 0x7f5d0019f660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 -- 172.21.15.28:0/2571802499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5d00109910 msgr2=0x7f5d0019fba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5d00109910 0x7f5d0019fba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 -- 172.21.15.28:0/2571802499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5d00108f90 msgr2=0x7f5d0019f660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5d00108f90 0x7f5d0019f660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.485+0000 7f5cff200640 1 -- 172.21.15.28:0/2571802499 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5d001a4650 con 0x7f5d0010ef20 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.486+0000 7f5cff200640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d001a3f20 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f5cf8009fd0 tx=0x7f5cf800b260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.486+0000 7f5cf7400640 1 -- 172.21.15.28:0/2571802499 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5cf8017070 con 0x7f5d0010ef20 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.486+0000 7f5cf7400640 1 -- 172.21.15.28:0/2571802499 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5cf8003e20 con 0x7f5d0010ef20 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.486+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5d001a4940 con 0x7f5d0010ef20 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.486+0000 7f5cf7400640 1 -- 172.21.15.28:0/2571802499 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5cf8010460 con 0x7f5d0010ef20 2024-05-20T20:06:11.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.486+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5d001ac1d0 con 0x7f5d0010ef20 2024-05-20T20:06:11.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.490+0000 7f5d08209740 1 Processor -- start 2024-05-20T20:06:11.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5cf7400640 1 -- 172.21.15.28:0/2571802499 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f5cf8003970 con 0x7f5d0010ef20 2024-05-20T20:06:11.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5d08209740 1 -- start start 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5d08209740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d886230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5d08209740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d890af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5d08209740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d893420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5d08209740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558b9d7f6b90 con 0x558b9d885e50 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5d08209740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558b9d7e6730 con 0x558b9d891030 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5d08209740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558b9d7ea770 con 0x558b9d886770 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.491+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d886230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d886230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41974/0 (socket says 172.21.15.28:41974) 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfde00640 1 -- 172.21.15.28:0/1712817699 learned_addr learned my addr 172.21.15.28:0/1712817699 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfe800640 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d890af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfde00640 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d893420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfe800640 1 -- 172.21.15.28:0/1712817699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 msgr2=0x558b9d893420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfe800640 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d893420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfe800640 1 -- 172.21.15.28:0/1712817699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 msgr2=0x558b9d886230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfe800640 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d886230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfe800640 1 -- 172.21.15.28:0/1712817699 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558b9d77f8b0 con 0x558b9d886770 2024-05-20T20:06:11.495 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.492+0000 7f5cfe800640 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d890af0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f5cf002a940 tx=0x7f5cf0005c00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=920608c8cc3fc66e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:11.495 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.493+0000 7f5cf7400640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5ce40b2b00 0x7f5ce40b4fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.495 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.493+0000 7f5cef400640 1 -- 172.21.15.28:0/1712817699 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5cf0035040 con 0x558b9d886770 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.493+0000 7f5cef400640 1 -- 172.21.15.28:0/1712817699 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5cf00054f0 con 0x558b9d886770 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.493+0000 7f5cef400640 1 -- 172.21.15.28:0/1712817699 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5cf00035f0 con 0x558b9d886770 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.493+0000 7f5cf7400640 1 -- 172.21.15.28:0/2571802499 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5cf80e2530 con 0x7f5d0010ef20 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.493+0000 7f5cfde00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5ce40b2b00 0x7f5ce40b4fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 -- 172.21.15.28:0/1712817699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 msgr2=0x558b9d890af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d890af0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f5cf002a940 tx=0x7f5cf0005c00 comp rx=0 tx=0).stop 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 -- 172.21.15.28:0/1712817699 shutdown_connections 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d893420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d890af0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 --2- 172.21.15.28:0/1712817699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d886230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 -- 172.21.15.28:0/1712817699 >> 172.21.15.28:0/1712817699 conn(0x558b9d882360 msgr2=0x558b9d882740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5cfde00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5ce40b2b00 0x7f5ce40b4fc0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x558b9d883780 tx=0x7f5ce8005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:11.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 -- 172.21.15.28:0/1712817699 shutdown_connections 2024-05-20T20:06:11.497 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 -- 172.21.15.28:0/1712817699 wait complete. 2024-05-20T20:06:11.497 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.494+0000 7f5d08209740 1 -- start start 2024-05-20T20:06:11.497 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 1 Processor -- start 2024-05-20T20:06:11.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d885e50 0x558b9d8909c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d886770 0x558b9d88dcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d88e230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558b9d4cc790 con 0x558b9d886770 2024-05-20T20:06:11.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558b9d78ebc0 con 0x558b9d891030 2024-05-20T20:06:11.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558b9d893c00 con 0x558b9d885e50 2024-05-20T20:06:11.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5d08209740 10 client.0 _ll_register_callbacks cb 0x558b9d76c400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d886770 0x558b9d88dcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d886770 0x558b9d88dcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41982/0 (socket says 172.21.15.28:41982) 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.495+0000 7f5cfde00640 1 -- 172.21.15.28:0/101864773 learned_addr learned my addr 172.21.15.28:0/101864773 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cfe800640 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d885e50 0x558b9d8909c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cff200640 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d88e230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cfde00640 1 -- 172.21.15.28:0/101864773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d885e50 msgr2=0x558b9d8909c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cfde00640 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d885e50 0x558b9d8909c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cfde00640 1 -- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 msgr2=0x558b9d88e230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cfde00640 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d88e230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cfde00640 1 -- 172.21.15.28:0/101864773 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558b9d4fb210 con 0x558b9d886770 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5cfde00640 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d886770 0x558b9d88dcb0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f5ce800ff90 tx=0x7f5ce8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=605035f4e2cd6136 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.496+0000 7f5ce3e00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5ce3e00640 1 -- 172.21.15.28:0/101864773 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ce8004030 con 0x558b9d886770 2024-05-20T20:06:11.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5ce3e00640 1 -- 172.21.15.28:0/101864773 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5ce801c030 con 0x558b9d886770 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5cf00054f0 con 0x558b9d886770 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5ce3e00640 1 -- 172.21.15.28:0/101864773 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5ce80041d0 con 0x558b9d886770 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5d08209740 10 client.15231 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558b9d5c7620 con 0x558b9d886770 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5ce3e00640 1 -- 172.21.15.28:0/101864773 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5cf00054f0 con 0x558b9d886770 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.497+0000 7f5ce3e00640 1 -- 172.21.15.28:0/101864773 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5ce80047a0 con 0x558b9d886770 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.498+0000 7f5d08209740 10 client.15231 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.498+0000 7f5d08209740 10 client.15231 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.498+0000 7f5d08209740 20 client.15231 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.498+0000 7f5d08209740 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d8a09f0 0x558b9d8a2de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.498+0000 7f5d08209740 4 client.15231 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.498+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558b9d7f8ae0 con 0x558b9d8a09f0 2024-05-20T20:06:11.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.498+0000 7f5cfe800640 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d8a09f0 0x558b9d8a2de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.499+0000 7f5cfe800640 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d8a09f0 0x558b9d8a2de0 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2af9288e6bc75412 server_cookie=f92844ca20db53bf in_seq=0 out_seq=0 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.499+0000 7f5ce3e00640 10 client.15231 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.499+0000 7f5ce3e00640 1 -- 172.21.15.28:0/101864773 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558b9d7f8ae0 con 0x558b9d8a09f0 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.499+0000 7f5ce3e00640 10 client.15231 handle_command_reply: tid=0 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 client.15231 shutdown 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d886770 msgr2=0x558b9d88dcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d886770 0x558b9d88dcb0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f5ce800ff90 tx=0x7f5ce8004730 comp rx=0 tx=0).stop 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 shutdown_connections 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d8a09f0 0x558b9d8a2de0 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d88e230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d886770 0x558b9d88dcb0 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 --2- 172.21.15.28:0/101864773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d885e50 0x558b9d8909c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 >> 172.21.15.28:0/101864773 conn(0x558b9d882360 msgr2=0x558b9d8880e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:11.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5ce3e00640 0 client.15231 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:11.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 shutdown_connections 2024-05-20T20:06:11.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 1 -- 172.21.15.28:0/101864773 wait complete. 2024-05-20T20:06:11.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.500+0000 7f5d08209740 20 client.15231 trim_cache size 0 max 16384 2024-05-20T20:06:11.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.518+0000 7f5d08209740 1 Processor -- start 2024-05-20T20:06:11.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.518+0000 7f5d08209740 1 -- start start 2024-05-20T20:06:11.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5d08209740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d885e50 0x558b9d89cc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5d08209740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8904f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5d08209740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d891030 0x558b9d8a86a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5d08209740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5d001b4af0 con 0x558b9d891030 2024-05-20T20:06:11.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5d08209740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5d001b6630 con 0x558b9d885e50 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5d08209740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5d001b51c0 con 0x558b9d886770 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8904f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfe800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d885e50 0x558b9d89cc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8904f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59694/0 (socket says 172.21.15.28:59694) 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 -- 172.21.15.28:0/95593978 learned_addr learned my addr 172.21.15.28:0/95593978 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cff200640 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d891030 0x558b9d8a86a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 -- 172.21.15.28:0/95593978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d885e50 msgr2=0x558b9d89cc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d885e50 0x558b9d89cc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 -- 172.21.15.28:0/95593978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d891030 msgr2=0x558b9d8a86a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d891030 0x558b9d8a86a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.519+0000 7f5cfde00640 1 -- 172.21.15.28:0/95593978 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558b9d88e5a0 con 0x558b9d886770 2024-05-20T20:06:11.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.520+0000 7f5cfde00640 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8904f0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f5ce8004db0 tx=0x7f5ce8009dd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a97f332295f22a86 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:11.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.521+0000 7f5ceea00640 1 -- 172.21.15.28:0/95593978 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ce800fb40 con 0x558b9d886770 2024-05-20T20:06:11.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.521+0000 7f5ceea00640 1 -- 172.21.15.28:0/95593978 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5ce8004370 con 0x558b9d886770 2024-05-20T20:06:11.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.521+0000 7f5ceea00640 1 -- 172.21.15.28:0/95593978 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ce80052e0 con 0x558b9d886770 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 -- 172.21.15.28:0/95593978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 msgr2=0x558b9d8904f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8904f0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f5ce8004db0 tx=0x7f5ce8009dd0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 -- 172.21.15.28:0/95593978 shutdown_connections 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d891030 0x558b9d8a86a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8904f0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 --2- 172.21.15.28:0/95593978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d885e50 0x558b9d89cc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 -- 172.21.15.28:0/95593978 >> 172.21.15.28:0/95593978 conn(0x558b9d882360 msgr2=0x558b9d889320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:11.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 -- 172.21.15.28:0/95593978 shutdown_connections 2024-05-20T20:06:11.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 -- 172.21.15.28:0/95593978 wait complete. 2024-05-20T20:06:11.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.522+0000 7f5d08209740 1 -- start start 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.523+0000 7f5d08209740 1 Processor -- start 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.523+0000 7f5d08209740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d899330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.523+0000 7f5d08209740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8946a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.523+0000 7f5d08209740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d894be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.523+0000 7f5d08209740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5d001b5f60 con 0x558b9d885e50 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.523+0000 7f5d08209740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5d001b5890 con 0x558b9d891030 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.523+0000 7f5d08209740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5d001b2ad0 con 0x558b9d886770 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5d08209740 10 client.0 _ll_register_callbacks cb 0x7f5ce8013ed0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8946a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8946a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59700/0 (socket says 172.21.15.28:59700) 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 -- 172.21.15.28:0/3749822182 learned_addr learned my addr 172.21.15.28:0/3749822182 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cff200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d894be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfe800640 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d899330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 -- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 msgr2=0x558b9d894be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d894be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 -- 172.21.15.28:0/3749822182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 msgr2=0x558b9d899330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d899330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cfde00640 1 -- 172.21.15.28:0/3749822182 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558b9d8951d0 con 0x558b9d886770 2024-05-20T20:06:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.524+0000 7f5cff200640 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d894be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.525+0000 7f5cfde00640 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8946a0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f5ce8010c90 tx=0x7f5ce8004750 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8af2a5ebd50e144a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.525+0000 7f5ce3e00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.525+0000 7f5ce3e00640 1 -- 172.21.15.28:0/3749822182 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ce803f070 con 0x558b9d886770 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.525+0000 7f5ce3e00640 1 -- 172.21.15.28:0/3749822182 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5ce8004ff0 con 0x558b9d886770 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.525+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5ce8004370 con 0x558b9d886770 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5ce3e00640 1 -- 172.21.15.28:0/3749822182 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5ce8049050 con 0x558b9d886770 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 10 client.25354 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558b9d90b380 con 0x558b9d886770 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5ce3e00640 1 -- 172.21.15.28:0/3749822182 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5ce8004370 con 0x558b9d886770 2024-05-20T20:06:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5ce3e00640 1 -- 172.21.15.28:0/3749822182 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5ce8044820 con 0x558b9d886770 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 10 client.25354 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 10 client.25354 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 20 client.25354 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d90ba10 0x558b9d90de30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 4 client.25354 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.526+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x558b9d7f22c0 con 0x558b9d90ba10 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.527+0000 7f5cfe800640 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d90ba10 0x558b9d90de30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.527+0000 7f5cfe800640 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d90ba10 0x558b9d90de30 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=100f822f941a425b server_cookie=6e51cff141102811 in_seq=0 out_seq=0 2024-05-20T20:06:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.527+0000 7f5ce3e00640 10 client.25354 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:11.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.540+0000 7f5ce3e00640 1 -- 172.21.15.28:0/3749822182 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x558b9d7f22c0 con 0x558b9d90ba10 2024-05-20T20:06:11.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.540+0000 7f5ce3e00640 10 client.25354 handle_command_reply: tid=0 2024-05-20T20:06:11.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.540+0000 7f5d08209740 1 client.25354 shutdown 2024-05-20T20:06:11.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 msgr2=0x558b9d8946a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8946a0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f5ce8010c90 tx=0x7f5ce8004750 comp rx=0 tx=0).stop 2024-05-20T20:06:11.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 shutdown_connections 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558b9d90ba10 0x558b9d90de30 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b9d891030 0x558b9d894be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b9d886770 0x558b9d8946a0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5ce3e00640 0 client.25354 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 --2- 172.21.15.28:0/3749822182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b9d885e50 0x558b9d899330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 >> 172.21.15.28:0/3749822182 conn(0x558b9d882360 msgr2=0x558b9d880c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 shutdown_connections 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 1 -- 172.21.15.28:0/3749822182 wait complete. 2024-05-20T20:06:11.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.541+0000 7f5d08209740 20 client.25354 trim_cache size 0 max 16384 2024-05-20T20:06:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:06:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:06:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5ce40b2b00 msgr2=0x7f5ce40b4fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5ce40b2b00 0x7f5ce40b4fc0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x558b9d883780 tx=0x7f5ce8005f90 comp rx=0 tx=0).stop 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 msgr2=0x7f5d001a3f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d001a3f20 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f5cf8009fd0 tx=0x7f5cf800b260 comp rx=0 tx=0).stop 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 shutdown_connections 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5ce40b2b00 0x7f5ce40b4fc0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5d0010ef20 0x7f5d001a3f20 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5d00109910 0x7f5d0019fba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 --2- 172.21.15.28:0/2571802499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5d00108f90 0x7f5d0019f660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 >> 172.21.15.28:0/2571802499 conn(0x7f5d00078830 msgr2=0x7f5d000799f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 shutdown_connections 2024-05-20T20:06:11.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:11.542+0000 7f5d04a00640 1 -- 172.21.15.28:0/2571802499 wait complete. 2024-05-20T20:06:11.878 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2024-05-20T20:06:09.638700+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1393752,"num_objects":104,"num_object_clones":0,"num_object_copies":312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104,"num_whiteouts":0,"num_read":3288,"num_read_kb":2098,"num_write":711,"num_write_kb":4810,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":2952020,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1028,"ondisk_log_size":1028,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":347892,"kb_used_data":11184,"kb_used_omap":18,"kb_used_meta":336557,"kb_avail":1124593932,"statfs":{"total":1151940427776,"available":1151584186368,"internally_reserved":0,"allocated":11452416,"data_stored":9384168,"data_compressed":316038,"data_compressed_allocated":417792,"data_compressed_original":4165632,"omap_allocated":19143,"internal_metadata":344634681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":198605,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":530,"num_read_kb":422,"num_write":188,"num_write_kb":1184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004777"},"pg_stats":[{"pgid":"3.3f","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603392+0000","last_change":"2024-05-20T20:01:23.536120+0000","last_active":"2024-05-20T20:05:37.603392+0000","last_peered":"2024-05-20T20:05:37.603392+0000","last_clean":"2024-05-20T20:05:37.603392+0000","last_became_active":"2024-05-20T20:01:23.535742+0000","last_became_peered":"2024-05-20T20:01:23.535742+0000","last_unstale":"2024-05-20T20:05:37.603392+0000","last_undegraded":"2024-05-20T20:05:37.603392+0000","last_fullsized":"2024-05-20T20:05:37.603392+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:24:32.199637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750621+0000","last_change":"2024-05-20T20:01:23.534175+0000","last_active":"2024-05-20T20:05:37.750621+0000","last_peered":"2024-05-20T20:05:37.750621+0000","last_clean":"2024-05-20T20:05:37.750621+0000","last_became_active":"2024-05-20T20:01:23.533801+0000","last_became_peered":"2024-05-20T20:01:23.533801+0000","last_unstale":"2024-05-20T20:05:37.750621+0000","last_undegraded":"2024-05-20T20:05:37.750621+0000","last_fullsized":"2024-05-20T20:05:37.750621+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:51:42.460883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750419+0000","last_change":"2024-05-20T20:01:23.529908+0000","last_active":"2024-05-20T20:05:37.750419+0000","last_peered":"2024-05-20T20:05:37.750419+0000","last_clean":"2024-05-20T20:05:37.750419+0000","last_became_active":"2024-05-20T20:01:23.526753+0000","last_became_peered":"2024-05-20T20:01:23.526753+0000","last_unstale":"2024-05-20T20:05:37.750419+0000","last_undegraded":"2024-05-20T20:05:37.750419+0000","last_fullsized":"2024-05-20T20:05:37.750419+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:46:30.549180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946345+0000","last_change":"2024-05-20T20:01:23.506759+0000","last_active":"2024-05-20T20:05:37.946345+0000","last_peered":"2024-05-20T20:05:37.946345+0000","last_clean":"2024-05-20T20:05:37.946345+0000","last_became_active":"2024-05-20T20:01:23.506396+0000","last_became_peered":"2024-05-20T20:01:23.506396+0000","last_unstale":"2024-05-20T20:05:37.946345+0000","last_undegraded":"2024-05-20T20:05:37.946345+0000","last_fullsized":"2024-05-20T20:05:37.946345+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:25:37.911622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"80'4","reported_seq":27,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950341+0000","last_change":"2024-05-20T20:01:23.531879+0000","last_active":"2024-05-20T20:05:37.950341+0000","last_peered":"2024-05-20T20:05:37.950341+0000","last_clean":"2024-05-20T20:05:37.950341+0000","last_became_active":"2024-05-20T20:01:23.531514+0000","last_became_peered":"2024-05-20T20:01:23.531514+0000","last_unstale":"2024-05-20T20:05:37.950341+0000","last_undegraded":"2024-05-20T20:05:37.950341+0000","last_fullsized":"2024-05-20T20:05:37.950341+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:41:39.285978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"80'6","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601481+0000","last_change":"2024-05-20T20:02:04.762828+0000","last_active":"2024-05-20T20:05:37.601481+0000","last_peered":"2024-05-20T20:05:37.601481+0000","last_clean":"2024-05-20T20:05:37.601481+0000","last_became_active":"2024-05-20T20:02:04.762444+0000","last_became_peered":"2024-05-20T20:02:04.762444+0000","last_unstale":"2024-05-20T20:05:37.601481+0000","last_undegraded":"2024-05-20T20:05:37.601481+0000","last_fullsized":"2024-05-20T20:05:37.601481+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:54:39.045783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,8],"acting":[10,9,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"80'5","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946714+0000","last_change":"2024-05-20T20:01:23.515082+0000","last_active":"2024-05-20T20:05:37.946714+0000","last_peered":"2024-05-20T20:05:37.946714+0000","last_clean":"2024-05-20T20:05:37.946714+0000","last_became_active":"2024-05-20T20:01:23.514709+0000","last_became_peered":"2024-05-20T20:01:23.514709+0000","last_unstale":"2024-05-20T20:05:37.946714+0000","last_undegraded":"2024-05-20T20:05:37.946714+0000","last_fullsized":"2024-05-20T20:05:37.946714+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:14:49.413825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740698+0000","last_change":"2024-05-20T20:01:23.518731+0000","last_active":"2024-05-20T20:05:37.740698+0000","last_peered":"2024-05-20T20:05:37.740698+0000","last_clean":"2024-05-20T20:05:37.740698+0000","last_became_active":"2024-05-20T20:01:23.517934+0000","last_became_peered":"2024-05-20T20:01:23.517934+0000","last_unstale":"2024-05-20T20:05:37.740698+0000","last_undegraded":"2024-05-20T20:05:37.740698+0000","last_fullsized":"2024-05-20T20:05:37.740698+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:27:48.789414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.942016+0000","last_change":"2024-05-20T20:01:23.512410+0000","last_active":"2024-05-20T20:05:37.942016+0000","last_peered":"2024-05-20T20:05:37.942016+0000","last_clean":"2024-05-20T20:05:37.942016+0000","last_became_active":"2024-05-20T20:01:23.511956+0000","last_became_peered":"2024-05-20T20:01:23.511956+0000","last_unstale":"2024-05-20T20:05:37.942016+0000","last_undegraded":"2024-05-20T20:05:37.942016+0000","last_fullsized":"2024-05-20T20:05:37.942016+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:06:32.992245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"80'16","reported_seq":45,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601908+0000","last_change":"2024-05-20T20:02:04.763055+0000","last_active":"2024-05-20T20:05:37.601908+0000","last_peered":"2024-05-20T20:05:37.601908+0000","last_clean":"2024-05-20T20:05:37.601908+0000","last_became_active":"2024-05-20T20:02:04.762531+0000","last_became_peered":"2024-05-20T20:02:04.762531+0000","last_unstale":"2024-05-20T20:05:37.601908+0000","last_undegraded":"2024-05-20T20:05:37.601908+0000","last_fullsized":"2024-05-20T20:05:37.601908+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:31:28.836040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,9],"acting":[10,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947610+0000","last_change":"2024-05-20T20:01:23.513722+0000","last_active":"2024-05-20T20:05:37.947610+0000","last_peered":"2024-05-20T20:05:37.947610+0000","last_clean":"2024-05-20T20:05:37.947610+0000","last_became_active":"2024-05-20T20:01:23.513404+0000","last_became_peered":"2024-05-20T20:01:23.513404+0000","last_unstale":"2024-05-20T20:05:37.947610+0000","last_undegraded":"2024-05-20T20:05:37.947610+0000","last_fullsized":"2024-05-20T20:05:37.947610+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:11:07.530055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"80'7","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946190+0000","last_change":"2024-05-20T20:01:23.505490+0000","last_active":"2024-05-20T20:05:37.946190+0000","last_peered":"2024-05-20T20:05:37.946190+0000","last_clean":"2024-05-20T20:05:37.946190+0000","last_became_active":"2024-05-20T20:01:23.504103+0000","last_became_peered":"2024-05-20T20:01:23.504103+0000","last_unstale":"2024-05-20T20:05:37.946190+0000","last_undegraded":"2024-05-20T20:05:37.946190+0000","last_fullsized":"2024-05-20T20:05:37.946190+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:09.479520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"80'10","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951989+0000","last_change":"2024-05-20T20:01:23.527346+0000","last_active":"2024-05-20T20:05:37.951989+0000","last_peered":"2024-05-20T20:05:37.951989+0000","last_clean":"2024-05-20T20:05:37.951989+0000","last_became_active":"2024-05-20T20:01:23.523128+0000","last_became_peered":"2024-05-20T20:01:23.523128+0000","last_unstale":"2024-05-20T20:05:37.951989+0000","last_undegraded":"2024-05-20T20:05:37.951989+0000","last_fullsized":"2024-05-20T20:05:37.951989+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:24:45.339388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749970+0000","last_change":"2024-05-20T20:01:23.531099+0000","last_active":"2024-05-20T20:05:37.749970+0000","last_peered":"2024-05-20T20:05:37.749970+0000","last_clean":"2024-05-20T20:05:37.749970+0000","last_became_active":"2024-05-20T20:01:23.530312+0000","last_became_peered":"2024-05-20T20:01:23.530312+0000","last_unstale":"2024-05-20T20:05:37.749970+0000","last_undegraded":"2024-05-20T20:05:37.749970+0000","last_fullsized":"2024-05-20T20:05:37.749970+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:39:24.625619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.749804+0000","last_change":"2024-05-20T20:01:23.530928+0000","last_active":"2024-05-20T20:05:37.749804+0000","last_peered":"2024-05-20T20:05:37.749804+0000","last_clean":"2024-05-20T20:05:37.749804+0000","last_became_active":"2024-05-20T20:01:23.529343+0000","last_became_peered":"2024-05-20T20:01:23.529343+0000","last_unstale":"2024-05-20T20:05:37.749804+0000","last_undegraded":"2024-05-20T20:05:37.749804+0000","last_fullsized":"2024-05-20T20:05:37.749804+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:33:59.571664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941202+0000","last_change":"2024-05-20T20:01:23.534923+0000","last_active":"2024-05-20T20:05:37.941202+0000","last_peered":"2024-05-20T20:05:37.941202+0000","last_clean":"2024-05-20T20:05:37.941202+0000","last_became_active":"2024-05-20T20:01:23.534527+0000","last_became_peered":"2024-05-20T20:01:23.534527+0000","last_unstale":"2024-05-20T20:05:37.941202+0000","last_undegraded":"2024-05-20T20:05:37.941202+0000","last_fullsized":"2024-05-20T20:05:37.941202+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:58:08.834093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603579+0000","last_change":"2024-05-20T20:01:23.535429+0000","last_active":"2024-05-20T20:05:37.603579+0000","last_peered":"2024-05-20T20:05:37.603579+0000","last_clean":"2024-05-20T20:05:37.603579+0000","last_became_active":"2024-05-20T20:01:23.534924+0000","last_became_peered":"2024-05-20T20:01:23.534924+0000","last_unstale":"2024-05-20T20:05:37.603579+0000","last_undegraded":"2024-05-20T20:05:37.603579+0000","last_fullsized":"2024-05-20T20:05:37.603579+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:15:35.728473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"80'10","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602307+0000","last_change":"2024-05-20T20:01:23.521267+0000","last_active":"2024-05-20T20:05:37.602307+0000","last_peered":"2024-05-20T20:05:37.602307+0000","last_clean":"2024-05-20T20:05:37.602307+0000","last_became_active":"2024-05-20T20:01:23.520519+0000","last_became_peered":"2024-05-20T20:01:23.520519+0000","last_unstale":"2024-05-20T20:05:37.602307+0000","last_undegraded":"2024-05-20T20:05:37.602307+0000","last_fullsized":"2024-05-20T20:05:37.602307+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:40:03.856342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940697+0000","last_change":"2024-05-20T20:01:23.529059+0000","last_active":"2024-05-20T20:05:37.940697+0000","last_peered":"2024-05-20T20:05:37.940697+0000","last_clean":"2024-05-20T20:05:37.940697+0000","last_became_active":"2024-05-20T20:01:23.528767+0000","last_became_peered":"2024-05-20T20:01:23.528767+0000","last_unstale":"2024-05-20T20:05:37.940697+0000","last_undegraded":"2024-05-20T20:05:37.940697+0000","last_fullsized":"2024-05-20T20:05:37.940697+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:33:52.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'14","reported_seq":308,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:07.286601+0000","last_change":"2024-05-20T20:01:19.456797+0000","last_active":"2024-05-20T20:06:07.286601+0000","last_peered":"2024-05-20T20:06:07.286601+0000","last_clean":"2024-05-20T20:06:07.286601+0000","last_became_active":"2024-05-20T20:01:19.456389+0000","last_became_peered":"2024-05-20T20:01:19.456389+0000","last_unstale":"2024-05-20T20:06:07.286601+0000","last_undegraded":"2024-05-20T20:06:07.286601+0000","last_fullsized":"2024-05-20T20:06:07.286601+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:34:00.563049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":264,"num_read_kb":264,"num_write":15,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741585+0000","last_change":"2024-05-20T20:01:23.516932+0000","last_active":"2024-05-20T20:05:37.741585+0000","last_peered":"2024-05-20T20:05:37.741585+0000","last_clean":"2024-05-20T20:05:37.741585+0000","last_became_active":"2024-05-20T20:01:23.516406+0000","last_became_peered":"2024-05-20T20:01:23.516406+0000","last_unstale":"2024-05-20T20:05:37.741585+0000","last_undegraded":"2024-05-20T20:05:37.741585+0000","last_fullsized":"2024-05-20T20:05:37.741585+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:01:48.583566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741764+0000","last_change":"2024-05-20T20:01:19.648399+0000","last_active":"2024-05-20T20:05:37.741764+0000","last_peered":"2024-05-20T20:05:37.741764+0000","last_clean":"2024-05-20T20:05:37.741764+0000","last_became_active":"2024-05-20T20:01:19.647908+0000","last_became_peered":"2024-05-20T20:01:19.647908+0000","last_unstale":"2024-05-20T20:05:37.741764+0000","last_undegraded":"2024-05-20T20:05:37.741764+0000","last_fullsized":"2024-05-20T20:05:37.741764+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:56:27.871313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750654+0000","last_change":"2024-05-20T20:01:23.538405+0000","last_active":"2024-05-20T20:05:37.750654+0000","last_peered":"2024-05-20T20:05:37.750654+0000","last_clean":"2024-05-20T20:05:37.750654+0000","last_became_active":"2024-05-20T20:01:23.534713+0000","last_became_peered":"2024-05-20T20:01:23.534713+0000","last_unstale":"2024-05-20T20:05:37.750654+0000","last_undegraded":"2024-05-20T20:05:37.750654+0000","last_fullsized":"2024-05-20T20:05:37.750654+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:20:59.600768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'13","reported_seq":43,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950989+0000","last_change":"2024-05-20T20:01:19.456692+0000","last_active":"2024-05-20T20:05:37.950989+0000","last_peered":"2024-05-20T20:05:37.950989+0000","last_clean":"2024-05-20T20:05:37.950989+0000","last_became_active":"2024-05-20T20:01:19.456218+0000","last_became_peered":"2024-05-20T20:01:19.456218+0000","last_unstale":"2024-05-20T20:05:37.950989+0000","last_undegraded":"2024-05-20T20:05:37.950989+0000","last_fullsized":"2024-05-20T20:05:37.950989+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:48:30.376431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":18,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941131+0000","last_change":"2024-05-20T20:01:23.530020+0000","last_active":"2024-05-20T20:05:37.941131+0000","last_peered":"2024-05-20T20:05:37.941131+0000","last_clean":"2024-05-20T20:05:37.941131+0000","last_became_active":"2024-05-20T20:01:23.529594+0000","last_became_peered":"2024-05-20T20:01:23.529594+0000","last_unstale":"2024-05-20T20:05:37.941131+0000","last_undegraded":"2024-05-20T20:05:37.941131+0000","last_fullsized":"2024-05-20T20:05:37.941131+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:58.290388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950749+0000","last_change":"2024-05-20T20:01:19.459732+0000","last_active":"2024-05-20T20:05:37.950749+0000","last_peered":"2024-05-20T20:05:37.950749+0000","last_clean":"2024-05-20T20:05:37.950749+0000","last_became_active":"2024-05-20T20:01:19.458753+0000","last_became_peered":"2024-05-20T20:01:19.458753+0000","last_unstale":"2024-05-20T20:05:37.950749+0000","last_undegraded":"2024-05-20T20:05:37.950749+0000","last_fullsized":"2024-05-20T20:05:37.950749+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:17:04.876406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"80'15","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602506+0000","last_change":"2024-05-20T20:02:05.159343+0000","last_active":"2024-05-20T20:05:37.602506+0000","last_peered":"2024-05-20T20:05:37.602506+0000","last_clean":"2024-05-20T20:05:37.602506+0000","last_became_active":"2024-05-20T20:02:05.158020+0000","last_became_peered":"2024-05-20T20:02:05.158020+0000","last_unstale":"2024-05-20T20:05:37.602506+0000","last_undegraded":"2024-05-20T20:05:37.602506+0000","last_fullsized":"2024-05-20T20:05:37.602506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:34:59.456717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,1],"acting":[9,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.1b","version":"81'8","reported_seq":304,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:07.076953+0000","last_change":"2024-05-20T20:01:19.457302+0000","last_active":"2024-05-20T20:06:07.076953+0000","last_peered":"2024-05-20T20:06:07.076953+0000","last_clean":"2024-05-20T20:06:07.076953+0000","last_became_active":"2024-05-20T20:01:19.456881+0000","last_became_peered":"2024-05-20T20:01:19.456881+0000","last_unstale":"2024-05-20T20:06:07.076953+0000","last_undegraded":"2024-05-20T20:06:07.076953+0000","last_fullsized":"2024-05-20T20:06:07.076953+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:44:24.399614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1545,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":267,"num_read_kb":5,"num_write":5,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"80'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950345+0000","last_change":"2024-05-20T20:02:04.931600+0000","last_active":"2024-05-20T20:05:37.950345+0000","last_peered":"2024-05-20T20:05:37.950345+0000","last_clean":"2024-05-20T20:05:37.950345+0000","last_became_active":"2024-05-20T20:02:04.931310+0000","last_became_peered":"2024-05-20T20:02:04.931310+0000","last_unstale":"2024-05-20T20:05:37.950345+0000","last_undegraded":"2024-05-20T20:05:37.950345+0000","last_fullsized":"2024-05-20T20:05:37.950345+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:04:10.182373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,7],"acting":[8,9,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"78'7","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601320+0000","last_change":"2024-05-20T20:01:19.465250+0000","last_active":"2024-05-20T20:05:37.601320+0000","last_peered":"2024-05-20T20:05:37.601320+0000","last_clean":"2024-05-20T20:05:37.601320+0000","last_became_active":"2024-05-20T20:01:19.464826+0000","last_became_peered":"2024-05-20T20:01:19.464826+0000","last_unstale":"2024-05-20T20:05:37.601320+0000","last_undegraded":"2024-05-20T20:05:37.601320+0000","last_fullsized":"2024-05-20T20:05:37.601320+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:31:38.415519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751407+0000","last_change":"2024-05-20T20:01:23.519226+0000","last_active":"2024-05-20T20:05:37.751407+0000","last_peered":"2024-05-20T20:05:37.751407+0000","last_clean":"2024-05-20T20:05:37.751407+0000","last_became_active":"2024-05-20T20:01:23.518757+0000","last_became_peered":"2024-05-20T20:01:23.518757+0000","last_unstale":"2024-05-20T20:05:37.751407+0000","last_undegraded":"2024-05-20T20:05:37.751407+0000","last_fullsized":"2024-05-20T20:05:37.751407+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:23:11.732912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"78'3","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950714+0000","last_change":"2024-05-20T20:01:19.467334+0000","last_active":"2024-05-20T20:05:37.950714+0000","last_peered":"2024-05-20T20:05:37.950714+0000","last_clean":"2024-05-20T20:05:37.950714+0000","last_became_active":"2024-05-20T20:01:19.466263+0000","last_became_peered":"2024-05-20T20:01:19.466263+0000","last_unstale":"2024-05-20T20:05:37.950714+0000","last_undegraded":"2024-05-20T20:05:37.950714+0000","last_fullsized":"2024-05-20T20:05:37.950714+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:36:59.218641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"80'8","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740756+0000","last_change":"2024-05-20T20:01:23.523781+0000","last_active":"2024-05-20T20:05:37.740756+0000","last_peered":"2024-05-20T20:05:37.740756+0000","last_clean":"2024-05-20T20:05:37.740756+0000","last_became_active":"2024-05-20T20:01:23.523448+0000","last_became_peered":"2024-05-20T20:01:23.523448+0000","last_unstale":"2024-05-20T20:05:37.740756+0000","last_undegraded":"2024-05-20T20:05:37.740756+0000","last_fullsized":"2024-05-20T20:05:37.740756+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:01:55.876942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751178+0000","last_change":"2024-05-20T20:01:19.459499+0000","last_active":"2024-05-20T20:05:37.751178+0000","last_peered":"2024-05-20T20:05:37.751178+0000","last_clean":"2024-05-20T20:05:37.751178+0000","last_became_active":"2024-05-20T20:01:19.459147+0000","last_became_peered":"2024-05-20T20:01:19.459147+0000","last_unstale":"2024-05-20T20:05:37.751178+0000","last_undegraded":"2024-05-20T20:05:37.751178+0000","last_fullsized":"2024-05-20T20:05:37.751178+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:40:59.385221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"80'11","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601398+0000","last_change":"2024-05-20T20:01:23.525678+0000","last_active":"2024-05-20T20:05:37.601398+0000","last_peered":"2024-05-20T20:05:37.601398+0000","last_clean":"2024-05-20T20:05:37.601398+0000","last_became_active":"2024-05-20T20:01:23.523759+0000","last_became_peered":"2024-05-20T20:01:23.523759+0000","last_unstale":"2024-05-20T20:05:37.601398+0000","last_undegraded":"2024-05-20T20:05:37.601398+0000","last_fullsized":"2024-05-20T20:05:37.601398+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:29:05.214243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'7","reported_seq":560,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:08.077846+0000","last_change":"2024-05-20T20:01:19.462743+0000","last_active":"2024-05-20T20:06:08.077846+0000","last_peered":"2024-05-20T20:06:08.077846+0000","last_clean":"2024-05-20T20:06:08.077846+0000","last_became_active":"2024-05-20T20:01:19.462342+0000","last_became_peered":"2024-05-20T20:01:19.462342+0000","last_unstale":"2024-05-20T20:06:08.077846+0000","last_undegraded":"2024-05-20T20:06:08.077846+0000","last_fullsized":"2024-05-20T20:06:08.077846+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:17:46.694097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":524,"num_read_kb":524,"num_write":8,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"80'10","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941795+0000","last_change":"2024-05-20T20:01:23.521071+0000","last_active":"2024-05-20T20:05:37.941795+0000","last_peered":"2024-05-20T20:05:37.941795+0000","last_clean":"2024-05-20T20:05:37.941795+0000","last_became_active":"2024-05-20T20:01:23.520703+0000","last_became_peered":"2024-05-20T20:01:23.520703+0000","last_unstale":"2024-05-20T20:05:37.941795+0000","last_undegraded":"2024-05-20T20:05:37.941795+0000","last_fullsized":"2024-05-20T20:05:37.941795+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:35:40.433651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947538+0000","last_change":"2024-05-20T20:01:19.462316+0000","last_active":"2024-05-20T20:05:37.947538+0000","last_peered":"2024-05-20T20:05:37.947538+0000","last_clean":"2024-05-20T20:05:37.947538+0000","last_became_active":"2024-05-20T20:01:19.461929+0000","last_became_peered":"2024-05-20T20:01:19.461929+0000","last_unstale":"2024-05-20T20:05:37.947538+0000","last_undegraded":"2024-05-20T20:05:37.947538+0000","last_fullsized":"2024-05-20T20:05:37.947538+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:25:44.877126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"80'9","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750655+0000","last_change":"2024-05-20T20:01:23.534836+0000","last_active":"2024-05-20T20:05:37.750655+0000","last_peered":"2024-05-20T20:05:37.750655+0000","last_clean":"2024-05-20T20:05:37.750655+0000","last_became_active":"2024-05-20T20:01:23.534442+0000","last_became_peered":"2024-05-20T20:01:23.534442+0000","last_unstale":"2024-05-20T20:05:37.750655+0000","last_undegraded":"2024-05-20T20:05:37.750655+0000","last_fullsized":"2024-05-20T20:05:37.750655+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:02:25.299270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"78'5","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751839+0000","last_change":"2024-05-20T20:01:19.648880+0000","last_active":"2024-05-20T20:05:37.751839+0000","last_peered":"2024-05-20T20:05:37.751839+0000","last_clean":"2024-05-20T20:05:37.751839+0000","last_became_active":"2024-05-20T20:01:19.648325+0000","last_became_peered":"2024-05-20T20:01:19.648325+0000","last_unstale":"2024-05-20T20:05:37.751839+0000","last_undegraded":"2024-05-20T20:05:37.751839+0000","last_fullsized":"2024-05-20T20:05:37.751839+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:17:01.466175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"80'11","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941385+0000","last_change":"2024-05-20T20:01:23.514170+0000","last_active":"2024-05-20T20:05:37.941385+0000","last_peered":"2024-05-20T20:05:37.941385+0000","last_clean":"2024-05-20T20:05:37.941385+0000","last_became_active":"2024-05-20T20:01:23.513587+0000","last_became_peered":"2024-05-20T20:01:23.513587+0000","last_unstale":"2024-05-20T20:05:37.941385+0000","last_undegraded":"2024-05-20T20:05:37.941385+0000","last_fullsized":"2024-05-20T20:05:37.941385+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:16:10.288894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'45","reported_seq":617,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:08.078613+0000","last_change":"2024-05-20T20:01:19.458983+0000","last_active":"2024-05-20T20:06:08.078613+0000","last_peered":"2024-05-20T20:06:08.078613+0000","last_clean":"2024-05-20T20:06:08.078613+0000","last_became_active":"2024-05-20T20:01:19.458538+0000","last_became_peered":"2024-05-20T20:01:19.458538+0000","last_unstale":"2024-05-20T20:06:08.078613+0000","last_undegraded":"2024-05-20T20:06:08.078613+0000","last_fullsized":"2024-05-20T20:06:08.078613+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:55:03.844296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72092,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":542,"num_read_kb":343,"num_write":47,"num_write_kb":91,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"80'6","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950750+0000","last_change":"2024-05-20T20:01:23.514904+0000","last_active":"2024-05-20T20:05:37.950750+0000","last_peered":"2024-05-20T20:05:37.950750+0000","last_clean":"2024-05-20T20:05:37.950750+0000","last_became_active":"2024-05-20T20:01:23.512934+0000","last_became_peered":"2024-05-20T20:01:23.512934+0000","last_unstale":"2024-05-20T20:05:37.950750+0000","last_undegraded":"2024-05-20T20:05:37.950750+0000","last_fullsized":"2024-05-20T20:05:37.950750+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:10.081551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"78'9","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602898+0000","last_change":"2024-05-20T20:02:04.772778+0000","last_active":"2024-05-20T20:05:37.602898+0000","last_peered":"2024-05-20T20:05:37.602898+0000","last_clean":"2024-05-20T20:05:37.602898+0000","last_became_active":"2024-05-20T20:02:04.760114+0000","last_became_peered":"2024-05-20T20:02:04.760114+0000","last_unstale":"2024-05-20T20:05:37.602898+0000","last_undegraded":"2024-05-20T20:05:37.602898+0000","last_fullsized":"2024-05-20T20:05:37.602898+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:19:12.568418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":68,"num_keys_recovered":10,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,10,3],"acting":[11,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.12","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950860+0000","last_change":"2024-05-20T20:01:23.524713+0000","last_active":"2024-05-20T20:05:37.950860+0000","last_peered":"2024-05-20T20:05:37.950860+0000","last_clean":"2024-05-20T20:05:37.950860+0000","last_became_active":"2024-05-20T20:01:23.524382+0000","last_became_peered":"2024-05-20T20:01:23.524382+0000","last_unstale":"2024-05-20T20:05:37.950860+0000","last_undegraded":"2024-05-20T20:05:37.950860+0000","last_fullsized":"2024-05-20T20:05:37.950860+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:51:30.180642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"78'5","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751331+0000","last_change":"2024-05-20T20:01:19.470504+0000","last_active":"2024-05-20T20:05:37.751331+0000","last_peered":"2024-05-20T20:05:37.751331+0000","last_clean":"2024-05-20T20:05:37.751331+0000","last_became_active":"2024-05-20T20:01:19.470080+0000","last_became_peered":"2024-05-20T20:01:19.470080+0000","last_unstale":"2024-05-20T20:05:37.751331+0000","last_undegraded":"2024-05-20T20:05:37.751331+0000","last_fullsized":"2024-05-20T20:05:37.751331+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:55:01.566399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"80'15","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947438+0000","last_change":"2024-05-20T20:01:23.532408+0000","last_active":"2024-05-20T20:05:37.947438+0000","last_peered":"2024-05-20T20:05:37.947438+0000","last_clean":"2024-05-20T20:05:37.947438+0000","last_became_active":"2024-05-20T20:01:23.531970+0000","last_became_peered":"2024-05-20T20:01:23.531970+0000","last_unstale":"2024-05-20T20:05:37.947438+0000","last_undegraded":"2024-05-20T20:05:37.947438+0000","last_fullsized":"2024-05-20T20:05:37.947438+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:03:47.985613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'9","reported_seq":307,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:07.288083+0000","last_change":"2024-05-20T20:01:19.456823+0000","last_active":"2024-05-20T20:06:07.288083+0000","last_peered":"2024-05-20T20:06:07.288083+0000","last_clean":"2024-05-20T20:06:07.288083+0000","last_became_active":"2024-05-20T20:01:19.456377+0000","last_became_peered":"2024-05-20T20:01:19.456377+0000","last_unstale":"2024-05-20T20:06:07.288083+0000","last_undegraded":"2024-05-20T20:06:07.288083+0000","last_fullsized":"2024-05-20T20:06:07.288083+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:33.707050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2071,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":269,"num_read_kb":7,"num_write":9,"num_write_kb":5,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946197+0000","last_change":"2024-05-20T20:01:23.513014+0000","last_active":"2024-05-20T20:05:37.946197+0000","last_peered":"2024-05-20T20:05:37.946197+0000","last_clean":"2024-05-20T20:05:37.946197+0000","last_became_active":"2024-05-20T20:01:23.512566+0000","last_became_peered":"2024-05-20T20:01:23.512566+0000","last_unstale":"2024-05-20T20:05:37.946197+0000","last_undegraded":"2024-05-20T20:05:37.946197+0000","last_fullsized":"2024-05-20T20:05:37.946197+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:57:41.343236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"78'17","reported_seq":72,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950363+0000","last_change":"2024-05-20T20:02:05.169342+0000","last_active":"2024-05-20T20:05:37.950363+0000","last_peered":"2024-05-20T20:05:37.950363+0000","last_clean":"2024-05-20T20:05:37.950363+0000","last_became_active":"2024-05-20T20:02:05.156279+0000","last_became_peered":"2024-05-20T20:02:05.156279+0000","last_unstale":"2024-05-20T20:05:37.950363+0000","last_undegraded":"2024-05-20T20:05:37.950363+0000","last_fullsized":"2024-05-20T20:05:37.950363+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:29.870046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,4],"acting":[8,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751593+0000","last_change":"2024-05-20T20:01:23.512117+0000","last_active":"2024-05-20T20:05:37.751593+0000","last_peered":"2024-05-20T20:05:37.751593+0000","last_clean":"2024-05-20T20:05:37.751593+0000","last_became_active":"2024-05-20T20:01:23.511697+0000","last_became_peered":"2024-05-20T20:01:23.511697+0000","last_unstale":"2024-05-20T20:05:37.751593+0000","last_undegraded":"2024-05-20T20:05:37.751593+0000","last_fullsized":"2024-05-20T20:05:37.751593+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:45:47.823986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'5","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:39.658956+0000","last_change":"2024-05-20T20:01:19.456753+0000","last_active":"2024-05-20T20:05:39.658956+0000","last_peered":"2024-05-20T20:05:39.658956+0000","last_clean":"2024-05-20T20:05:39.658956+0000","last_became_active":"2024-05-20T20:01:19.456191+0000","last_became_peered":"2024-05-20T20:01:19.456191+0000","last_unstale":"2024-05-20T20:05:39.658956+0000","last_undegraded":"2024-05-20T20:05:39.658956+0000","last_fullsized":"2024-05-20T20:05:39.658956+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:26:44.160142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603167+0000","last_change":"2024-05-20T20:01:23.506675+0000","last_active":"2024-05-20T20:05:37.603167+0000","last_peered":"2024-05-20T20:05:37.603167+0000","last_clean":"2024-05-20T20:05:37.603167+0000","last_became_active":"2024-05-20T20:01:23.506272+0000","last_became_peered":"2024-05-20T20:01:23.506272+0000","last_unstale":"2024-05-20T20:05:37.603167+0000","last_undegraded":"2024-05-20T20:05:37.603167+0000","last_fullsized":"2024-05-20T20:05:37.603167+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:41:20.552533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"78'1","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602390+0000","last_change":"2024-05-20T20:01:19.649078+0000","last_active":"2024-05-20T20:05:37.602390+0000","last_peered":"2024-05-20T20:05:37.602390+0000","last_clean":"2024-05-20T20:05:37.602390+0000","last_became_active":"2024-05-20T20:01:19.648045+0000","last_became_peered":"2024-05-20T20:01:19.648045+0000","last_unstale":"2024-05-20T20:05:37.602390+0000","last_undegraded":"2024-05-20T20:05:37.602390+0000","last_fullsized":"2024-05-20T20:05:37.602390+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:17:19.636956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601741+0000","last_change":"2024-05-20T20:01:23.527528+0000","last_active":"2024-05-20T20:05:37.601741+0000","last_peered":"2024-05-20T20:05:37.601741+0000","last_clean":"2024-05-20T20:05:37.601741+0000","last_became_active":"2024-05-20T20:01:23.527136+0000","last_became_peered":"2024-05-20T20:01:23.527136+0000","last_unstale":"2024-05-20T20:05:37.601741+0000","last_undegraded":"2024-05-20T20:05:37.601741+0000","last_fullsized":"2024-05-20T20:05:37.601741+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:09:34.265133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'6","reported_seq":300,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:07.287502+0000","last_change":"2024-05-20T20:01:19.461558+0000","last_active":"2024-05-20T20:06:07.287502+0000","last_peered":"2024-05-20T20:06:07.287502+0000","last_clean":"2024-05-20T20:06:07.287502+0000","last_became_active":"2024-05-20T20:01:19.461215+0000","last_became_peered":"2024-05-20T20:01:19.461215+0000","last_unstale":"2024-05-20T20:06:07.287502+0000","last_undegraded":"2024-05-20T20:06:07.287502+0000","last_fullsized":"2024-05-20T20:06:07.287502+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:49:11.138562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":979,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":264,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.601417+0000","last_change":"2024-05-20T20:01:23.534996+0000","last_active":"2024-05-20T20:05:37.601417+0000","last_peered":"2024-05-20T20:05:37.601417+0000","last_clean":"2024-05-20T20:05:37.601417+0000","last_became_active":"2024-05-20T20:01:23.534581+0000","last_became_peered":"2024-05-20T20:01:23.534581+0000","last_unstale":"2024-05-20T20:05:37.601417+0000","last_undegraded":"2024-05-20T20:05:37.601417+0000","last_fullsized":"2024-05-20T20:05:37.601417+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:19:54.036529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751541+0000","last_change":"2024-05-20T20:01:23.514055+0000","last_active":"2024-05-20T20:05:37.751541+0000","last_peered":"2024-05-20T20:05:37.751541+0000","last_clean":"2024-05-20T20:05:37.751541+0000","last_became_active":"2024-05-20T20:01:23.513709+0000","last_became_peered":"2024-05-20T20:01:23.513709+0000","last_unstale":"2024-05-20T20:05:37.751541+0000","last_undegraded":"2024-05-20T20:05:37.751541+0000","last_fullsized":"2024-05-20T20:05:37.751541+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:59:34.400885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"81'7","reported_seq":300,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:07.287149+0000","last_change":"2024-05-20T20:01:19.647114+0000","last_active":"2024-05-20T20:06:07.287149+0000","last_peered":"2024-05-20T20:06:07.287149+0000","last_clean":"2024-05-20T20:06:07.287149+0000","last_became_active":"2024-05-20T20:01:19.646662+0000","last_became_peered":"2024-05-20T20:01:19.646662+0000","last_unstale":"2024-05-20T20:06:07.287149+0000","last_undegraded":"2024-05-20T20:06:07.287149+0000","last_fullsized":"2024-05-20T20:06:07.287149+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:34:31.418348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":263,"num_read_kb":263,"num_write":7,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741186+0000","last_change":"2024-05-20T20:01:23.513464+0000","last_active":"2024-05-20T20:05:37.741186+0000","last_peered":"2024-05-20T20:05:37.741186+0000","last_clean":"2024-05-20T20:05:37.741186+0000","last_became_active":"2024-05-20T20:01:23.512999+0000","last_became_peered":"2024-05-20T20:01:23.512999+0000","last_unstale":"2024-05-20T20:05:37.741186+0000","last_undegraded":"2024-05-20T20:05:37.741186+0000","last_fullsized":"2024-05-20T20:05:37.741186+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:45:55.027047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'6","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.945964+0000","last_change":"2024-05-20T20:01:19.460320+0000","last_active":"2024-05-20T20:05:37.945964+0000","last_peered":"2024-05-20T20:05:37.945964+0000","last_clean":"2024-05-20T20:05:37.945964+0000","last_became_active":"2024-05-20T20:01:19.459938+0000","last_became_peered":"2024-05-20T20:01:19.459938+0000","last_unstale":"2024-05-20T20:05:37.945964+0000","last_undegraded":"2024-05-20T20:05:37.945964+0000","last_fullsized":"2024-05-20T20:05:37.945964+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:47:32.248550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"81'99","reported_seq":131,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:39.743703+0000","last_change":"2024-05-20T19:59:43.618633+0000","last_active":"2024-05-20T20:05:39.743703+0000","last_peered":"2024-05-20T20:05:39.743703+0000","last_clean":"2024-05-20T20:05:39.743703+0000","last_became_active":"2024-05-20T19:59:43.612799+0000","last_became_peered":"2024-05-20T19:59:43.612799+0000","last_unstale":"2024-05-20T20:05:39.743703+0000","last_undegraded":"2024-05-20T20:05:39.743703+0000","last_fullsized":"2024-05-20T20:05:39.743703+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T19:55:47.651756+0000","last_clean_scrub_stamp":"2024-05-20T19:55:47.651756+0000","objects_scrubbed":0,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:27:09.858916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950636+0000","last_change":"2024-05-20T20:01:23.528053+0000","last_active":"2024-05-20T20:05:37.950636+0000","last_peered":"2024-05-20T20:05:37.950636+0000","last_clean":"2024-05-20T20:05:37.950636+0000","last_became_active":"2024-05-20T20:01:23.527641+0000","last_became_peered":"2024-05-20T20:01:23.527641+0000","last_unstale":"2024-05-20T20:05:37.950636+0000","last_undegraded":"2024-05-20T20:05:37.950636+0000","last_fullsized":"2024-05-20T20:05:37.950636+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:07:21.556184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'4","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946570+0000","last_change":"2024-05-20T20:01:19.457333+0000","last_active":"2024-05-20T20:05:37.946570+0000","last_peered":"2024-05-20T20:05:37.946570+0000","last_clean":"2024-05-20T20:05:37.946570+0000","last_became_active":"2024-05-20T20:01:19.456932+0000","last_became_peered":"2024-05-20T20:01:19.456932+0000","last_unstale":"2024-05-20T20:05:37.946570+0000","last_undegraded":"2024-05-20T20:05:37.946570+0000","last_fullsized":"2024-05-20T20:05:37.946570+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:00.436896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950840+0000","last_change":"2024-05-20T20:01:19.648879+0000","last_active":"2024-05-20T20:05:37.950840+0000","last_peered":"2024-05-20T20:05:37.950840+0000","last_clean":"2024-05-20T20:05:37.950840+0000","last_became_active":"2024-05-20T20:01:19.648373+0000","last_became_peered":"2024-05-20T20:01:19.648373+0000","last_unstale":"2024-05-20T20:05:37.950840+0000","last_undegraded":"2024-05-20T20:05:37.950840+0000","last_fullsized":"2024-05-20T20:05:37.950840+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:08:49.651088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952108+0000","last_change":"2024-05-20T20:01:23.529117+0000","last_active":"2024-05-20T20:05:37.952108+0000","last_peered":"2024-05-20T20:05:37.952108+0000","last_clean":"2024-05-20T20:05:37.952108+0000","last_became_active":"2024-05-20T20:01:23.528494+0000","last_became_peered":"2024-05-20T20:01:23.528494+0000","last_unstale":"2024-05-20T20:05:37.952108+0000","last_undegraded":"2024-05-20T20:05:37.952108+0000","last_fullsized":"2024-05-20T20:05:37.952108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T23:52:24.219825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'129","reported_seq":300,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:01.140633+0000","last_change":"2024-05-20T20:02:04.784995+0000","last_active":"2024-05-20T20:06:01.140633+0000","last_peered":"2024-05-20T20:06:01.140633+0000","last_clean":"2024-05-20T20:06:01.140633+0000","last_became_active":"2024-05-20T20:02:04.766148+0000","last_became_peered":"2024-05-20T20:02:04.766148+0000","last_unstale":"2024-05-20T20:06:01.140633+0000","last_undegraded":"2024-05-20T20:06:01.140633+0000","last_fullsized":"2024-05-20T20:06:01.140633+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":129,"log_dups_size":0,"ondisk_log_size":129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:02:06.285583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":422,"num_read_kb":376,"num_write":238,"num_write_kb":1415,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":6,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741294+0000","last_change":"2024-05-20T20:01:23.511125+0000","last_active":"2024-05-20T20:05:37.741294+0000","last_peered":"2024-05-20T20:05:37.741294+0000","last_clean":"2024-05-20T20:05:37.741294+0000","last_became_active":"2024-05-20T20:01:23.507361+0000","last_became_peered":"2024-05-20T20:01:23.507361+0000","last_unstale":"2024-05-20T20:05:37.741294+0000","last_undegraded":"2024-05-20T20:05:37.741294+0000","last_fullsized":"2024-05-20T20:05:37.741294+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T02:02:01.573339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'7","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.946101+0000","last_change":"2024-05-20T20:01:19.558874+0000","last_active":"2024-05-20T20:05:37.946101+0000","last_peered":"2024-05-20T20:05:37.946101+0000","last_clean":"2024-05-20T20:05:37.946101+0000","last_became_active":"2024-05-20T20:01:19.558487+0000","last_became_peered":"2024-05-20T20:01:19.558487+0000","last_unstale":"2024-05-20T20:05:37.946101+0000","last_undegraded":"2024-05-20T20:05:37.946101+0000","last_fullsized":"2024-05-20T20:05:37.946101+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:14:59.733464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"80'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750609+0000","last_change":"2024-05-20T20:02:05.155654+0000","last_active":"2024-05-20T20:05:37.750609+0000","last_peered":"2024-05-20T20:05:37.750609+0000","last_clean":"2024-05-20T20:05:37.750609+0000","last_became_active":"2024-05-20T20:02:05.155052+0000","last_became_peered":"2024-05-20T20:02:05.155052+0000","last_unstale":"2024-05-20T20:05:37.750609+0000","last_undegraded":"2024-05-20T20:05:37.750609+0000","last_fullsized":"2024-05-20T20:05:37.750609+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:08:52.026576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"78'2","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941387+0000","last_change":"2024-05-20T20:01:19.647453+0000","last_active":"2024-05-20T20:05:37.941387+0000","last_peered":"2024-05-20T20:05:37.941387+0000","last_clean":"2024-05-20T20:05:37.941387+0000","last_became_active":"2024-05-20T20:01:19.647182+0000","last_became_peered":"2024-05-20T20:01:19.647182+0000","last_unstale":"2024-05-20T20:05:37.941387+0000","last_undegraded":"2024-05-20T20:05:37.941387+0000","last_fullsized":"2024-05-20T20:05:37.941387+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:07:14.350560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"80'2","reported_seq":26,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950366+0000","last_change":"2024-05-20T20:01:23.507094+0000","last_active":"2024-05-20T20:05:37.950366+0000","last_peered":"2024-05-20T20:05:37.950366+0000","last_clean":"2024-05-20T20:05:37.950366+0000","last_became_active":"2024-05-20T20:01:23.506741+0000","last_became_peered":"2024-05-20T20:01:23.506741+0000","last_unstale":"2024-05-20T20:05:37.950366+0000","last_undegraded":"2024-05-20T20:05:37.950366+0000","last_fullsized":"2024-05-20T20:05:37.950366+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:51:19.643145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"78'7","reported_seq":37,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741016+0000","last_change":"2024-05-20T20:01:19.471220+0000","last_active":"2024-05-20T20:05:37.741016+0000","last_peered":"2024-05-20T20:05:37.741016+0000","last_clean":"2024-05-20T20:05:37.741016+0000","last_became_active":"2024-05-20T20:01:19.470601+0000","last_became_peered":"2024-05-20T20:01:19.470601+0000","last_unstale":"2024-05-20T20:05:37.741016+0000","last_undegraded":"2024-05-20T20:05:37.741016+0000","last_fullsized":"2024-05-20T20:05:37.741016+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:40:07.392822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602899+0000","last_change":"2024-05-20T20:01:23.535227+0000","last_active":"2024-05-20T20:05:37.602899+0000","last_peered":"2024-05-20T20:05:37.602899+0000","last_clean":"2024-05-20T20:05:37.602899+0000","last_became_active":"2024-05-20T20:01:23.534817+0000","last_became_peered":"2024-05-20T20:01:23.534817+0000","last_unstale":"2024-05-20T20:05:37.602899+0000","last_undegraded":"2024-05-20T20:05:37.602899+0000","last_fullsized":"2024-05-20T20:05:37.602899+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T07:09:09.781861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"78'3","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940518+0000","last_change":"2024-05-20T20:01:19.469695+0000","last_active":"2024-05-20T20:05:37.940518+0000","last_peered":"2024-05-20T20:05:37.940518+0000","last_clean":"2024-05-20T20:05:37.940518+0000","last_became_active":"2024-05-20T20:01:19.469250+0000","last_became_peered":"2024-05-20T20:01:19.469250+0000","last_unstale":"2024-05-20T20:05:37.940518+0000","last_undegraded":"2024-05-20T20:05:37.940518+0000","last_fullsized":"2024-05-20T20:05:37.940518+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:05:40.226916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741426+0000","last_change":"2024-05-20T20:01:23.536412+0000","last_active":"2024-05-20T20:05:37.741426+0000","last_peered":"2024-05-20T20:05:37.741426+0000","last_clean":"2024-05-20T20:05:37.741426+0000","last_became_active":"2024-05-20T20:01:23.536047+0000","last_became_peered":"2024-05-20T20:01:23.536047+0000","last_unstale":"2024-05-20T20:05:37.741426+0000","last_undegraded":"2024-05-20T20:05:37.741426+0000","last_fullsized":"2024-05-20T20:05:37.741426+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:18:08.363063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"78'4","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.950516+0000","last_change":"2024-05-20T20:01:19.459939+0000","last_active":"2024-05-20T20:05:37.950516+0000","last_peered":"2024-05-20T20:05:37.950516+0000","last_clean":"2024-05-20T20:05:37.950516+0000","last_became_active":"2024-05-20T20:01:19.458284+0000","last_became_peered":"2024-05-20T20:01:19.458284+0000","last_unstale":"2024-05-20T20:05:37.950516+0000","last_undegraded":"2024-05-20T20:05:37.950516+0000","last_fullsized":"2024-05-20T20:05:37.950516+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:17:52.962050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"80'5","reported_seq":29,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947502+0000","last_change":"2024-05-20T20:01:23.529145+0000","last_active":"2024-05-20T20:05:37.947502+0000","last_peered":"2024-05-20T20:05:37.947502+0000","last_clean":"2024-05-20T20:05:37.947502+0000","last_became_active":"2024-05-20T20:01:23.528742+0000","last_became_peered":"2024-05-20T20:01:23.528742+0000","last_unstale":"2024-05-20T20:05:37.947502+0000","last_undegraded":"2024-05-20T20:05:37.947502+0000","last_fullsized":"2024-05-20T20:05:37.947502+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:29:42.670826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'6","reported_seq":301,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:07.076933+0000","last_change":"2024-05-20T20:01:19.647450+0000","last_active":"2024-05-20T20:06:07.076933+0000","last_peered":"2024-05-20T20:06:07.076933+0000","last_clean":"2024-05-20T20:06:07.076933+0000","last_became_active":"2024-05-20T20:01:19.647012+0000","last_became_peered":"2024-05-20T20:01:19.647012+0000","last_unstale":"2024-05-20T20:06:07.076933+0000","last_undegraded":"2024-05-20T20:06:07.076933+0000","last_fullsized":"2024-05-20T20:06:07.076933+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T21:26:48.809033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1561,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":265,"num_read_kb":3,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947819+0000","last_change":"2024-05-20T20:01:23.510328+0000","last_active":"2024-05-20T20:05:37.947819+0000","last_peered":"2024-05-20T20:05:37.947819+0000","last_clean":"2024-05-20T20:05:37.947819+0000","last_became_active":"2024-05-20T20:01:23.509953+0000","last_became_peered":"2024-05-20T20:01:23.509953+0000","last_unstale":"2024-05-20T20:05:37.947819+0000","last_undegraded":"2024-05-20T20:05:37.947819+0000","last_fullsized":"2024-05-20T20:05:37.947819+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:07:03.176635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"81'75","reported_seq":147,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:06:01.126122+0000","last_change":"2024-05-20T20:01:19.473567+0000","last_active":"2024-05-20T20:06:01.126122+0000","last_peered":"2024-05-20T20:06:01.126122+0000","last_clean":"2024-05-20T20:06:01.126122+0000","last_became_active":"2024-05-20T20:01:19.473265+0000","last_became_peered":"2024-05-20T20:01:19.473265+0000","last_unstale":"2024-05-20T20:06:01.126122+0000","last_undegraded":"2024-05-20T20:06:01.126122+0000","last_fullsized":"2024-05-20T20:06:01.126122+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":75,"log_dups_size":0,"ondisk_log_size":75,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:42:11.983360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328302,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":45,"num_read_kb":44,"num_write":84,"num_write_kb":1075,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947490+0000","last_change":"2024-05-20T20:01:23.518792+0000","last_active":"2024-05-20T20:05:37.947490+0000","last_peered":"2024-05-20T20:05:37.947490+0000","last_clean":"2024-05-20T20:05:37.947490+0000","last_became_active":"2024-05-20T20:01:23.518404+0000","last_became_peered":"2024-05-20T20:01:23.518404+0000","last_unstale":"2024-05-20T20:05:37.947490+0000","last_undegraded":"2024-05-20T20:05:37.947490+0000","last_fullsized":"2024-05-20T20:05:37.947490+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:34:45.675377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"78'5","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952372+0000","last_change":"2024-05-20T20:01:19.473989+0000","last_active":"2024-05-20T20:05:37.952372+0000","last_peered":"2024-05-20T20:05:37.952372+0000","last_clean":"2024-05-20T20:05:37.952372+0000","last_became_active":"2024-05-20T20:01:19.473337+0000","last_became_peered":"2024-05-20T20:01:19.473337+0000","last_unstale":"2024-05-20T20:05:37.952372+0000","last_undegraded":"2024-05-20T20:05:37.952372+0000","last_fullsized":"2024-05-20T20:05:37.952372+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":72,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:18.420423+0000","last_clean_scrub_stamp":"2024-05-20T20:01:18.420423+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:41.875091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.947860+0000","last_change":"2024-05-20T20:01:23.510130+0000","last_active":"2024-05-20T20:05:37.947860+0000","last_peered":"2024-05-20T20:05:37.947860+0000","last_clean":"2024-05-20T20:05:37.947860+0000","last_became_active":"2024-05-20T20:01:23.509714+0000","last_became_peered":"2024-05-20T20:01:23.509714+0000","last_unstale":"2024-05-20T20:05:37.947860+0000","last_undegraded":"2024-05-20T20:05:37.947860+0000","last_fullsized":"2024-05-20T20:05:37.947860+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:08:07.932874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"80'9","reported_seq":33,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603108+0000","last_change":"2024-05-20T20:01:23.513826+0000","last_active":"2024-05-20T20:05:37.603108+0000","last_peered":"2024-05-20T20:05:37.603108+0000","last_clean":"2024-05-20T20:05:37.603108+0000","last_became_active":"2024-05-20T20:01:23.513438+0000","last_became_peered":"2024-05-20T20:01:23.513438+0000","last_unstale":"2024-05-20T20:05:37.603108+0000","last_undegraded":"2024-05-20T20:05:37.603108+0000","last_fullsized":"2024-05-20T20:05:37.603108+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:47:49.094196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"80'12","reported_seq":36,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940537+0000","last_change":"2024-05-20T20:01:23.505067+0000","last_active":"2024-05-20T20:05:37.940537+0000","last_peered":"2024-05-20T20:05:37.940537+0000","last_clean":"2024-05-20T20:05:37.940537+0000","last_became_active":"2024-05-20T20:01:23.504570+0000","last_became_peered":"2024-05-20T20:01:23.504570+0000","last_unstale":"2024-05-20T20:05:37.940537+0000","last_undegraded":"2024-05-20T20:05:37.940537+0000","last_fullsized":"2024-05-20T20:05:37.940537+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:09:08.895791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"80'8","reported_seq":38,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940554+0000","last_change":"2024-05-20T20:02:04.770679+0000","last_active":"2024-05-20T20:05:37.940554+0000","last_peered":"2024-05-20T20:05:37.940554+0000","last_clean":"2024-05-20T20:05:37.940554+0000","last_became_active":"2024-05-20T20:02:04.770313+0000","last_became_peered":"2024-05-20T20:02:04.770313+0000","last_unstale":"2024-05-20T20:05:37.940554+0000","last_undegraded":"2024-05-20T20:05:37.940554+0000","last_fullsized":"2024-05-20T20:05:37.940554+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T06:31:30.316666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,9,0],"acting":[2,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"80'7","reported_seq":31,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.750191+0000","last_change":"2024-05-20T20:01:23.535858+0000","last_active":"2024-05-20T20:05:37.750191+0000","last_peered":"2024-05-20T20:05:37.750191+0000","last_clean":"2024-05-20T20:05:37.750191+0000","last_became_active":"2024-05-20T20:01:23.535431+0000","last_became_peered":"2024-05-20T20:01:23.535431+0000","last_unstale":"2024-05-20T20:05:37.750191+0000","last_undegraded":"2024-05-20T20:05:37.750191+0000","last_fullsized":"2024-05-20T20:05:37.750191+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:32:01.242906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"80'4","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.952180+0000","last_change":"2024-05-20T20:01:23.535947+0000","last_active":"2024-05-20T20:05:37.952180+0000","last_peered":"2024-05-20T20:05:37.952180+0000","last_clean":"2024-05-20T20:05:37.952180+0000","last_became_active":"2024-05-20T20:01:23.535606+0000","last_became_peered":"2024-05-20T20:01:23.535606+0000","last_unstale":"2024-05-20T20:05:37.952180+0000","last_undegraded":"2024-05-20T20:05:37.952180+0000","last_fullsized":"2024-05-20T20:05:37.952180+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:49:48.722380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.603224+0000","last_change":"2024-05-20T20:01:23.510375+0000","last_active":"2024-05-20T20:05:37.603224+0000","last_peered":"2024-05-20T20:05:37.603224+0000","last_clean":"2024-05-20T20:05:37.603224+0000","last_became_active":"2024-05-20T20:01:23.509005+0000","last_became_peered":"2024-05-20T20:01:23.509005+0000","last_unstale":"2024-05-20T20:05:37.603224+0000","last_undegraded":"2024-05-20T20:05:37.603224+0000","last_fullsized":"2024-05-20T20:05:37.603224+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:17:34.039337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"80'15","reported_seq":39,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.940832+0000","last_change":"2024-05-20T20:01:23.509080+0000","last_active":"2024-05-20T20:05:37.940832+0000","last_peered":"2024-05-20T20:05:37.940832+0000","last_clean":"2024-05-20T20:05:37.940832+0000","last_became_active":"2024-05-20T20:01:23.508689+0000","last_became_peered":"2024-05-20T20:01:23.508689+0000","last_unstale":"2024-05-20T20:05:37.940832+0000","last_undegraded":"2024-05-20T20:05:37.940832+0000","last_fullsized":"2024-05-20T20:05:37.940832+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T20:49:11.579907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"80'2","reported_seq":26,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.741159+0000","last_change":"2024-05-20T20:01:23.514296+0000","last_active":"2024-05-20T20:05:37.741159+0000","last_peered":"2024-05-20T20:05:37.741159+0000","last_clean":"2024-05-20T20:05:37.741159+0000","last_became_active":"2024-05-20T20:01:23.513881+0000","last_became_peered":"2024-05-20T20:01:23.513881+0000","last_unstale":"2024-05-20T20:05:37.741159+0000","last_undegraded":"2024-05-20T20:05:37.741159+0000","last_fullsized":"2024-05-20T20:05:37.741159+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:27:18.980555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"80'6","reported_seq":30,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.751338+0000","last_change":"2024-05-20T20:01:23.519172+0000","last_active":"2024-05-20T20:05:37.751338+0000","last_peered":"2024-05-20T20:05:37.751338+0000","last_clean":"2024-05-20T20:05:37.751338+0000","last_became_active":"2024-05-20T20:01:23.518306+0000","last_became_peered":"2024-05-20T20:01:23.518306+0000","last_unstale":"2024-05-20T20:05:37.751338+0000","last_undegraded":"2024-05-20T20:05:37.751338+0000","last_fullsized":"2024-05-20T20:05:37.751338+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T05:29:38.561319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.740731+0000","last_change":"2024-05-20T20:01:23.515172+0000","last_active":"2024-05-20T20:05:37.740731+0000","last_peered":"2024-05-20T20:05:37.740731+0000","last_clean":"2024-05-20T20:05:37.740731+0000","last_became_active":"2024-05-20T20:01:23.511832+0000","last_became_peered":"2024-05-20T20:01:23.511832+0000","last_unstale":"2024-05-20T20:05:37.740731+0000","last_undegraded":"2024-05-20T20:05:37.740731+0000","last_fullsized":"2024-05-20T20:05:37.740731+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-21T22:46:20.675982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"80'10","reported_seq":34,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.941843+0000","last_change":"2024-05-20T20:01:23.518471+0000","last_active":"2024-05-20T20:05:37.941843+0000","last_peered":"2024-05-20T20:05:37.941843+0000","last_clean":"2024-05-20T20:05:37.941843+0000","last_became_active":"2024-05-20T20:01:23.518082+0000","last_became_peered":"2024-05-20T20:01:23.518082+0000","last_unstale":"2024-05-20T20:05:37.941843+0000","last_undegraded":"2024-05-20T20:05:37.941843+0000","last_fullsized":"2024-05-20T20:05:37.941843+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T00:58:58.632460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"80'11","reported_seq":35,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.951652+0000","last_change":"2024-05-20T20:01:23.537844+0000","last_active":"2024-05-20T20:05:37.951652+0000","last_peered":"2024-05-20T20:05:37.951652+0000","last_clean":"2024-05-20T20:05:37.951652+0000","last_became_active":"2024-05-20T20:01:23.537181+0000","last_became_peered":"2024-05-20T20:01:23.537181+0000","last_unstale":"2024-05-20T20:05:37.951652+0000","last_undegraded":"2024-05-20T20:05:37.951652+0000","last_fullsized":"2024-05-20T20:05:37.951652+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T03:49:42.100520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"80'8","reported_seq":32,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-05-20T20:05:37.602990+0000","last_change":"2024-05-20T20:01:23.508151+0000","last_active":"2024-05-20T20:05:37.602990+0000","last_peered":"2024-05-20T20:05:37.602990+0000","last_clean":"2024-05-20T20:05:37.602990+0000","last_became_active":"2024-05-20T20:01:23.507753+0000","last_became_peered":"2024-05-20T20:01:23.507753+0000","last_unstale":"2024-05-20T20:05:37.602990+0000","last_undegraded":"2024-05-20T20:05:37.602990+0000","last_fullsized":"2024-05-20T20:05:37.602990+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T20:01:22.455134+0000","last_clean_scrub_stamp":"2024-05-20T20:01:22.455134+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T04:00:07.539977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":144,"num_read_kb":248,"num_write":135,"num_write_kb":2150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":108381,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":803150,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":3144,"num_read_kb":1850,"num_write":574,"num_write_kb":2658,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":180,"num_keys_recovered":16,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":811008,"data_stored":2409450,"data_compressed":103497,"data_compressed_allocated":135168,"data_compressed_original":2101248,"omap_allocated":0,"internal_metadata":0},"log_size":427,"ondisk_log_size":427,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":70,"seq":300647710794,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":917504,"data_stored":828458,"data_compressed":27977,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1596,"internal_metadata":27654596},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":10,"up_from":64,"seq":274877907023,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28096,"kb_used_data":1072,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717056,"statfs":{"total":95995035648,"available":95966265344,"internally_reserved":0,"allocated":1097728,"data_stored":1427559,"data_compressed":60009,"data_compressed_allocated":77824,"data_compressed_original":921600,"omap_allocated":1600,"internal_metadata":27654592},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":9,"up_from":59,"seq":253403070548,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32088,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713064,"statfs":{"total":95995035648,"available":95962177536,"internally_reserved":0,"allocated":933888,"data_stored":827929,"data_compressed":27977,"data_compressed_allocated":36864,"data_compressed_original":417792,"omap_allocated":1600,"internal_metadata":31914432},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":8,"up_from":53,"seq":227633266778,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27904,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717248,"statfs":{"total":95995035648,"available":95966461952,"internally_reserved":0,"allocated":905216,"data_stored":435091,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":7,"up_from":47,"seq":201863463008,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32112,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93713040,"statfs":{"total":95995035648,"available":95962152960,"internally_reserved":0,"allocated":958464,"data_stored":1030015,"data_compressed":44807,"data_compressed_allocated":57344,"data_compressed_original":614400,"omap_allocated":1590,"internal_metadata":31914442},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":42,"seq":180388626534,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27892,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717260,"statfs":{"total":95995035648,"available":95966474240,"internally_reserved":0,"allocated":892928,"data_stored":434821,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":13,"seq":55834574978,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27960,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717192,"statfs":{"total":95995035648,"available":95966404608,"internally_reserved":0,"allocated":966656,"data_stored":507766,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738504,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27912,"kb_used_data":892,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717240,"statfs":{"total":95995035648,"available":95966453760,"internally_reserved":0,"allocated":913408,"data_stored":436160,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":18,"seq":77309411453,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":970752,"data_stored":508729,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":25,"seq":107374182519,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27980,"kb_used_data":956,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717172,"statfs":{"total":95995035648,"available":95966384128,"internally_reserved":0,"allocated":978944,"data_stored":1155703,"data_compressed":43179,"data_compressed_allocated":57344,"data_compressed_original":724992,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986290,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":435886,"data_compressed":8680,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":36,"seq":154618822764,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32176,"kb_used_data":992,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":93712976,"statfs":{"total":95995035648,"available":95962087424,"internally_reserved":0,"allocated":1015808,"data_stored":1356051,"data_compressed":60009,"data_compressed_allocated":77824,"data_compressed_original":921600,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":36127,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":94208,"data_stored":74235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":75198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":721878,"data_compressed":34499,"data_compressed_allocated":45056,"data_compressed_original":700416,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":2295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":331975,"data_compressed":15202,"data_compressed_allocated":20480,"data_compressed_original":307200,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":394338,"data_compressed":19297,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":172032,"data_stored":403543,"data_compressed":15202,"data_compressed_allocated":20480,"data_compressed_original":307200,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394867,"data_compressed":19297,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T20:06:11.883 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:06:12.005 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:11 smithi028 ceph-mon[31178]: from='client.25358 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:12.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:11 smithi154 ceph-mon[36314]: from='client.25358 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:12.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:11 smithi033 ceph-mon[36179]: from='client.25358 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T20:06:12.391 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:12.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:12 smithi028 ceph-mon[31178]: pgmap v20: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:06:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:12 smithi154 ceph-mon[36314]: pgmap v20: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:06:13.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:12 smithi033 ceph-mon[36179]: pgmap v20: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:06:14.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 -- 172.21.15.28:0/3274241555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 msgr2=0x7fb8c8110310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:14.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 --2- 172.21.15.28:0/3274241555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 0x7fb8c8110310 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7fb8c000b0d0 tx=0x7fb8c000b3e0 comp rx=0 tx=0).stop 2024-05-20T20:06:14.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 -- 172.21.15.28:0/3274241555 shutdown_connections 2024-05-20T20:06:14.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 --2- 172.21.15.28:0/3274241555 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 0x7fb8c8110310 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:14.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 --2- 172.21.15.28:0/3274241555 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8109540 0x7fb8c8109920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:14.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 --2- 172.21.15.28:0/3274241555 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c8103490 0x7fb8c8103910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:14.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 -- 172.21.15.28:0/3274241555 >> 172.21.15.28:0/3274241555 conn(0x7fb8c80fefc0 msgr2=0x7fb8c81013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:14.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 -- 172.21.15.28:0/3274241555 shutdown_connections 2024-05-20T20:06:14.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.755+0000 7fb8cd800700 1 -- 172.21.15.28:0/3274241555 wait complete. 2024-05-20T20:06:14.771 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:14 smithi028 ceph-mon[31178]: pgmap v21: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 24 KiB/s wr, 31 op/s 2024-05-20T20:06:14.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.757+0000 7fb8cd800700 1 Processor -- start 2024-05-20T20:06:14.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.757+0000 7fb8cd800700 1 -- start start 2024-05-20T20:06:14.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8cd800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8103490 0x7fb8c819f760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:14.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8cd800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 0x7fb8c819fca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:14.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8cd800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c8109540 0x7fb8c8199970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:14.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8cd800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb8c8113670 con 0x7fb8c8103e50 2024-05-20T20:06:14.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8cd800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb8c81134f0 con 0x7fb8c8109540 2024-05-20T20:06:14.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8cd800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb8c81137f0 con 0x7fb8c8103490 2024-05-20T20:06:14.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8c7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8103490 0x7fb8c819f760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:14.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8bf400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 0x7fb8c819fca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:14.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8cce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c8109540 0x7fb8c8199970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:14.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8c7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8103490 0x7fb8c819f760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59716/0 (socket says 172.21.15.28:59716) 2024-05-20T20:06:14.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.758+0000 7fb8c7e00700 1 -- 172.21.15.28:0/4050082362 learned_addr learned my addr 172.21.15.28:0/4050082362 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:14.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c7e00700 1 -- 172.21.15.28:0/4050082362 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c8109540 msgr2=0x7fb8c8199970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:14.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c7e00700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c8109540 0x7fb8c8199970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:14.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c7e00700 1 -- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 msgr2=0x7fb8c819fca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:14.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c7e00700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 0x7fb8c819fca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:14.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c7e00700 1 -- 172.21.15.28:0/4050082362 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb8c819a0d0 con 0x7fb8c8103490 2024-05-20T20:06:14.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c7e00700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8103490 0x7fb8c819f760 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fb8b800d950 tx=0x7fb8b800dc60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:14.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c5600700 1 -- 172.21.15.28:0/4050082362 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8b8016070 con 0x7fb8c8103490 2024-05-20T20:06:14.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c5600700 1 -- 172.21.15.28:0/4050082362 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb8b800fd40 con 0x7fb8c8103490 2024-05-20T20:06:14.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb8c819a3c0 con 0x7fb8c8103490 2024-05-20T20:06:14.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.759+0000 7fb8c5600700 1 -- 172.21.15.28:0/4050082362 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8b800e3b0 con 0x7fb8c8103490 2024-05-20T20:06:14.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.760+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb8c8075770 con 0x7fb8c8103490 2024-05-20T20:06:14.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.761+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb8c806a230 con 0x7fb8c8103490 2024-05-20T20:06:14.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.763+0000 7fb8c5600700 1 -- 172.21.15.28:0/4050082362 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7fb8b8019070 con 0x7fb8c8103490 2024-05-20T20:06:14.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.764+0000 7fb8c5600700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2c20 0x7fb8ac0b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:14.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.765+0000 7fb8c5600700 1 -- 172.21.15.28:0/4050082362 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb8b80e2290 con 0x7fb8c8103490 2024-05-20T20:06:14.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.765+0000 7fb8bf400700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2c20 0x7fb8ac0b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:14.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.765+0000 7fb8bf400700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2c20 0x7fb8ac0b50e0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fb8c8108bf0 tx=0x7fb8b0009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:14.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:14.767+0000 7fb8c5600700 1 -- 172.21.15.28:0/4050082362 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb8b80a9a40 con 0x7fb8c8103490 2024-05-20T20:06:15.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.058+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fb8c802e560 con 0x7fb8c8103490 2024-05-20T20:06:15.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.059+0000 7fb8c5600700 1 -- 172.21.15.28:0/4050082362 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fb8b80a9340 con 0x7fb8c8103490 2024-05-20T20:06:15.061 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:15.061 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:06:15.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2c20 msgr2=0x7fb8ac0b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:15.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2c20 0x7fb8ac0b50e0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fb8c8108bf0 tx=0x7fb8b0009260 comp rx=0 tx=0).stop 2024-05-20T20:06:15.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8103490 msgr2=0x7fb8c819f760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:15.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8103490 0x7fb8c819f760 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fb8b800d950 tx=0x7fb8b800dc60 comp rx=0 tx=0).stop 2024-05-20T20:06:15.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 shutdown_connections 2024-05-20T20:06:15.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8ac0b2c20 0x7fb8ac0b50e0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:15.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb8c8109540 0x7fb8c8199970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:15.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb8c8103490 0x7fb8c819f760 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:15.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 --2- 172.21.15.28:0/4050082362 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb8c8103e50 0x7fb8c819fca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:15.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 >> 172.21.15.28:0/4050082362 conn(0x7fb8c80fefc0 msgr2=0x7fb8c8100750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:15.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 shutdown_connections 2024-05-20T20:06:15.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:15.063+0000 7fb8cd800700 1 -- 172.21.15.28:0/4050082362 wait complete. 2024-05-20T20:06:15.066 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:06:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:14 smithi154 ceph-mon[36314]: pgmap v21: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 24 KiB/s wr, 31 op/s 2024-05-20T20:06:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:14 smithi033 ceph-mon[36179]: pgmap v21: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 24 KiB/s wr, 31 op/s 2024-05-20T20:06:15.998 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:15 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/4050082362' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:16.003 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:06:16.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:15 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/4050082362' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:16.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:15 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/4050082362' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:16.474 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:16.998 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:16 smithi028 ceph-mon[31178]: pgmap v22: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 13 KiB/s wr, 26 op/s 2024-05-20T20:06:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:16 smithi154 ceph-mon[36314]: pgmap v22: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 13 KiB/s wr, 26 op/s 2024-05-20T20:06:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:16 smithi033 ceph-mon[36179]: pgmap v22: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 13 KiB/s wr, 26 op/s 2024-05-20T20:06:18.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 -- 172.21.15.28:0/2203646206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc105760 msgr2=0x7f76cc105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:18.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 --2- 172.21.15.28:0/2203646206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc105760 0x7f76cc105be0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f76b80098f0 tx=0x7f76b8009c00 comp rx=0 tx=0).stop 2024-05-20T20:06:18.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 -- 172.21.15.28:0/2203646206 shutdown_connections 2024-05-20T20:06:18.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 --2- 172.21.15.28:0/2203646206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76cc106120 0x7f76cc112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 --2- 172.21.15.28:0/2203646206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc10b760 0x7f76cc10bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 --2- 172.21.15.28:0/2203646206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc105760 0x7f76cc105be0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 -- 172.21.15.28:0/2203646206 >> 172.21.15.28:0/2203646206 conn(0x7f76cc101250 msgr2=0x7f76cc103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:18.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.681+0000 7f76d4400700 1 -- 172.21.15.28:0/2203646206 shutdown_connections 2024-05-20T20:06:18.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.682+0000 7f76d4400700 1 -- 172.21.15.28:0/2203646206 wait complete. 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 Processor -- start 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 -- start start 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76cc105760 0x7f76cc19be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc106120 0x7f76cc19c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc10b760 0x7f76cc1a0730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f76cc115990 con 0x7f76cc106120 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f76cc115810 con 0x7f76cc10b760 2024-05-20T20:06:18.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.683+0000 7f76d4400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f76cc115b10 con 0x7f76cc105760 2024-05-20T20:06:18.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc106120 0x7f76cc19c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:18.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76cc105760 0x7f76cc19be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:18.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc106120 0x7f76cc19c3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40550/0 (socket says 172.21.15.28:40550) 2024-05-20T20:06:18.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 -- 172.21.15.28:0/379748178 learned_addr learned my addr 172.21.15.28:0/379748178 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:18.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d2400700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc10b760 0x7f76cc1a0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:18.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 -- 172.21.15.28:0/379748178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76cc105760 msgr2=0x7f76cc19be60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:18.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76cc105760 0x7f76cc19be60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 -- 172.21.15.28:0/379748178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc10b760 msgr2=0x7f76cc1a0730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:18.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc10b760 0x7f76cc1a0730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.684+0000 7f76d1000700 1 -- 172.21.15.28:0/379748178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f76cc1a0e70 con 0x7f76cc106120 2024-05-20T20:06:18.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.685+0000 7f76d1000700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc106120 0x7f76cc19c3a0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f76b8009230 tx=0x7f76b800fa70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:18.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.685+0000 7f76ca000700 1 -- 172.21.15.28:0/379748178 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76b80050c0 con 0x7f76cc106120 2024-05-20T20:06:18.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.685+0000 7f76ca000700 1 -- 172.21.15.28:0/379748178 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f76b800b560 con 0x7f76cc106120 2024-05-20T20:06:18.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.685+0000 7f76ca000700 1 -- 172.21.15.28:0/379748178 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f76b8003a00 con 0x7f76cc106120 2024-05-20T20:06:18.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.685+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f76cc1a1100 con 0x7f76cc106120 2024-05-20T20:06:18.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.685+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f76cc1a89e0 con 0x7f76cc106120 2024-05-20T20:06:18.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.686+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f76cc06a230 con 0x7f76cc106120 2024-05-20T20:06:18.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.689+0000 7f76ca000700 1 -- 172.21.15.28:0/379748178 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f76b800b710 con 0x7f76cc106120 2024-05-20T20:06:18.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.690+0000 7f76ca000700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f76b40b2c20 0x7f76b40b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:18.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.690+0000 7f76ca000700 1 -- 172.21.15.28:0/379748178 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f76b80e3740 con 0x7f76cc106120 2024-05-20T20:06:18.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.691+0000 7f76d1a00700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f76b40b2c20 0x7f76b40b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:18.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.691+0000 7f76d1a00700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f76b40b2c20 0x7f76b40b50e0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f76c0009750 tx=0x7f76c0009310 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:18.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.695+0000 7f76ca000700 1 -- 172.21.15.28:0/379748178 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f76b8013030 con 0x7f76cc106120 2024-05-20T20:06:18.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.985+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f76cc19d1c0 con 0x7f76cc106120 2024-05-20T20:06:18.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:18 smithi028 ceph-mon[31178]: pgmap v23: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 1.8 KiB/s wr, 12 op/s 2024-05-20T20:06:18.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.988+0000 7f76ca000700 1 -- 172.21.15.28:0/379748178 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f76b80aaf80 con 0x7f76cc106120 2024-05-20T20:06:18.989 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:18.989 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:06:18.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f76b40b2c20 msgr2=0x7f76b40b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:18.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f76b40b2c20 0x7f76b40b50e0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f76c0009750 tx=0x7f76c0009310 comp rx=0 tx=0).stop 2024-05-20T20:06:18.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc106120 msgr2=0x7f76cc19c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:18.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc106120 0x7f76cc19c3a0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f76b8009230 tx=0x7f76b800fa70 comp rx=0 tx=0).stop 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 shutdown_connections 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f76b40b2c20 0x7f76b40b50e0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f76cc10b760 0x7f76cc1a0730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f76cc105760 0x7f76cc19be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 --2- 172.21.15.28:0/379748178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f76cc106120 0x7f76cc19c3a0 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.991+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 >> 172.21.15.28:0/379748178 conn(0x7f76cc101250 msgr2=0x7f76cc1019e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.992+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 shutdown_connections 2024-05-20T20:06:18.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:18.992+0000 7f76d4400700 1 -- 172.21.15.28:0/379748178 wait complete. 2024-05-20T20:06:18.994 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:06:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:18 smithi154 ceph-mon[36314]: pgmap v23: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 1.8 KiB/s wr, 12 op/s 2024-05-20T20:06:19.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:18 smithi033 ceph-mon[36179]: pgmap v23: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 1.8 KiB/s wr, 12 op/s 2024-05-20T20:06:19.932 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:06:20.014 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:19 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/379748178' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:20.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:19 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/379748178' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:20.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:19 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/379748178' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:20.396 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:20.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:20 smithi028 ceph-mon[31178]: pgmap v24: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 1.8 KiB/s wr, 16 op/s 2024-05-20T20:06:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:20 smithi154 ceph-mon[36314]: pgmap v24: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 1.8 KiB/s wr, 16 op/s 2024-05-20T20:06:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:20 smithi033 ceph-mon[36179]: pgmap v24: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 1.8 KiB/s wr, 16 op/s 2024-05-20T20:06:22.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 -- 172.21.15.28:0/1302955052 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 msgr2=0x7fb43c1028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 --2- 172.21.15.28:0/1302955052 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c1028c0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fb428009990 tx=0x7fb428009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 -- 172.21.15.28:0/1302955052 shutdown_connections 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 --2- 172.21.15.28:0/1302955052 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb43c102e00 0x7fb43c10f2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 --2- 172.21.15.28:0/1302955052 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb43c108520 0x7fb43c108900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 --2- 172.21.15.28:0/1302955052 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c1028c0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 -- 172.21.15.28:0/1302955052 >> 172.21.15.28:0/1302955052 conn(0x7fb43c0793f0 msgr2=0x7fb43c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 -- 172.21.15.28:0/1302955052 shutdown_connections 2024-05-20T20:06:22.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.641+0000 7fb443e00700 1 -- 172.21.15.28:0/1302955052 wait complete. 2024-05-20T20:06:22.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 Processor -- start 2024-05-20T20:06:22.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 -- start start 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c19be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb43c102e00 0x7fb43c19c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb43c108520 0x7fb43c1a06f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb43c113750 con 0x7fb43c102440 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb43c1135d0 con 0x7fb43c102e00 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.643+0000 7fb443e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb43c1138d0 con 0x7fb43c108520 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c19be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:22.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c19be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40560/0 (socket says 172.21.15.28:40560) 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 -- 172.21.15.28:0/2301368837 learned_addr learned my addr 172.21.15.28:0/2301368837 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb440a00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb43c102e00 0x7fb43c19c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441e00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb43c108520 0x7fb43c1a06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 -- 172.21.15.28:0/2301368837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb43c108520 msgr2=0x7fb43c1a06f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb43c108520 0x7fb43c1a06f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 -- 172.21.15.28:0/2301368837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb43c102e00 msgr2=0x7fb43c19c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb43c102e00 0x7fb43c19c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb441400700 1 -- 172.21.15.28:0/2301368837 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb43c1a0e30 con 0x7fb43c102440 2024-05-20T20:06:22.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.644+0000 7fb440a00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb43c102e00 0x7fb43c19c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:22.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.645+0000 7fb441400700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c19be20 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fb43000e9a0 tx=0x7fb43000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:22.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.645+0000 7fb43a000700 1 -- 172.21.15.28:0/2301368837 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb43000cbd0 con 0x7fb43c102440 2024-05-20T20:06:22.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.645+0000 7fb43a000700 1 -- 172.21.15.28:0/2301368837 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb43000cd70 con 0x7fb43c102440 2024-05-20T20:06:22.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.645+0000 7fb43a000700 1 -- 172.21.15.28:0/2301368837 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb430018680 con 0x7fb43c102440 2024-05-20T20:06:22.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.645+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb43c1a1120 con 0x7fb43c102440 2024-05-20T20:06:22.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.645+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb43c1a89a0 con 0x7fb43c102440 2024-05-20T20:06:22.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.646+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb43c06a230 con 0x7fb43c102440 2024-05-20T20:06:22.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.649+0000 7fb43a000700 1 -- 172.21.15.28:0/2301368837 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7fb430010430 con 0x7fb43c102440 2024-05-20T20:06:22.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.650+0000 7fb43a000700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb4240b2c20 0x7fb4240b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:22.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.650+0000 7fb43a000700 1 -- 172.21.15.28:0/2301368837 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb4300e29f0 con 0x7fb43c102440 2024-05-20T20:06:22.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.651+0000 7fb440a00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb4240b2c20 0x7fb4240b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:22.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.651+0000 7fb440a00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb4240b2c20 0x7fb4240b50e0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fb43c101300 tx=0x7fb42800b260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:22.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.655+0000 7fb43a000700 1 -- 172.21.15.28:0/2301368837 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb4300aa1a0 con 0x7fb43c102440 2024-05-20T20:06:22.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.897+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fb43c1a8c30 con 0x7fb43c102440 2024-05-20T20:06:22.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.902+0000 7fb43a000700 1 -- 172.21.15.28:0/2301368837 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) ==== 74+0+22947 (secure 0 0 0) 0x7fb4300a9aa0 con 0x7fb43c102440 2024-05-20T20:06:22.904 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:22.904 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":81,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:05:37.589166+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6800/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/2759010565":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/1496067961":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1975849601":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/19609679":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:06:22.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb4240b2c20 msgr2=0x7fb4240b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:22.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb4240b2c20 0x7fb4240b50e0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fb43c101300 tx=0x7fb42800b260 comp rx=0 tx=0).stop 2024-05-20T20:06:22.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 msgr2=0x7fb43c19be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:22.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c19be20 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fb43000e9a0 tx=0x7fb43000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 shutdown_connections 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb43c108520 0x7fb43c1a06f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb43c102440 0x7fb43c19be20 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb4240b2c20 0x7fb4240b50e0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 --2- 172.21.15.28:0/2301368837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb43c102e00 0x7fb43c19c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.905+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 >> 172.21.15.28:0/2301368837 conn(0x7fb43c0793f0 msgr2=0x7fb43c1002b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.906+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 shutdown_connections 2024-05-20T20:06:22.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:22.906+0000 7fb443e00700 1 -- 172.21.15.28:0/2301368837 wait complete. 2024-05-20T20:06:23.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:22 smithi028 ceph-mon[31178]: pgmap v25: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-05-20T20:06:23.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:22 smithi154 ceph-mon[36314]: pgmap v25: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-05-20T20:06:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:22 smithi033 ceph-mon[36179]: pgmap v25: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-05-20T20:06:23.892 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mon.a --format=json config get run_dir 2024-05-20T20:06:24.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:23 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2301368837' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:06:24.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:23 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2301368837' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:06:24.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:23 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2301368837' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:06:24.353 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:24.691 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 -- 172.21.15.28:0/2095451063 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb4076f80 msgr2=0x7f8bb4077360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 --2- 172.21.15.28:0/2095451063 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb4076f80 0x7f8bb4077360 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f8ba8009ca0 tx=0x7f8ba802a8c0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 -- 172.21.15.28:0/2095451063 shutdown_connections 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 --2- 172.21.15.28:0/2095451063 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb41138e0 0x7f8bb4115cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 --2- 172.21.15.28:0/2095451063 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb40778a0 0x7f8bb41133a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 --2- 172.21.15.28:0/2095451063 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb4076f80 0x7f8bb4077360 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 -- 172.21.15.28:0/2095451063 >> 172.21.15.28:0/2095451063 conn(0x7f8bb4100750 msgr2=0x7f8bb4102b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 -- 172.21.15.28:0/2095451063 shutdown_connections 2024-05-20T20:06:24.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.691+0000 7f8bbc000640 1 -- 172.21.15.28:0/2095451063 wait complete. 2024-05-20T20:06:24.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.692+0000 7f8bbc000640 1 Processor -- start 2024-05-20T20:06:24.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.692+0000 7f8bbc000640 1 -- start start 2024-05-20T20:06:24.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bbc000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb4076f80 0x7f8bb41a5460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bbc000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb40778a0 0x7f8bb41a59a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bbc000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb41138e0 0x7f8bb419f660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bbc000640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb41192e0 con 0x7f8bb41138e0 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bbc000640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb4119160 con 0x7f8bb40778a0 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bbc000640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb4119460 con 0x7f8bb4076f80 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb9600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb4076f80 0x7f8bb41a5460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb9600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb4076f80 0x7f8bb41a5460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56974/0 (socket says 172.21.15.28:56974) 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb9600640 1 -- 172.21.15.28:0/719887859 learned_addr learned my addr 172.21.15.28:0/719887859 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bba000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb41138e0 0x7f8bb419f660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb40778a0 0x7f8bb41a59a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb8c00640 1 -- 172.21.15.28:0/719887859 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb4076f80 msgr2=0x7f8bb41a5460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb4076f80 0x7f8bb41a5460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb8c00640 1 -- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb41138e0 msgr2=0x7f8bb419f660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.693+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb41138e0 0x7f8bb419f660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.694+0000 7f8bb8c00640 1 -- 172.21.15.28:0/719887859 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8bb419fde0 con 0x7f8bb40778a0 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.694+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb40778a0 0x7f8bb41a59a0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f8ba000e9f0 tx=0x7f8ba000eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.694+0000 7f8bb2000640 1 -- 172.21.15.28:0/719887859 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ba000cd80 con 0x7f8bb40778a0 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.694+0000 7f8bb2000640 1 -- 172.21.15.28:0/719887859 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8ba00035f0 con 0x7f8bb40778a0 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.694+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8bb41a00d0 con 0x7f8bb40778a0 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.694+0000 7f8bb2000640 1 -- 172.21.15.28:0/719887859 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ba0010690 con 0x7f8bb40778a0 2024-05-20T20:06:24.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.694+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8bb41ac320 con 0x7f8bb40778a0 2024-05-20T20:06:24.698 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.698+0000 7f8bbee6a740 1 Processor -- start 2024-05-20T20:06:24.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.699+0000 7f8bbee6a740 1 -- start start 2024-05-20T20:06:24.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.699+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb7c90 0x562cdbfb8070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.699+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb85b0 0x562cdbfc2930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.699+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 0x562cdbfc5260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.699+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562cdbf28b90 con 0x562cdbfb85b0 2024-05-20T20:06:24.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.699+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562cdbf18730 con 0x562cdbfb7c90 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.699+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562cdbf1c770 con 0x562cdbfc2e70 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb7c90 0x562cdbfb8070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb7c90 0x562cdbfb8070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34600/0 (socket says 172.21.15.28:34600) 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 -- 172.21.15.28:0/2835179875 learned_addr learned my addr 172.21.15.28:0/2835179875 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bb9600640 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb85b0 0x562cdbfc2930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bb2000640 1 -- 172.21.15.28:0/719887859 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f8ba00037a0 con 0x7f8bb40778a0 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 -- 172.21.15.28:0/2835179875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 msgr2=0x562cdbfc5260 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:06:24.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 0x562cdbfc5260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 -- 172.21.15.28:0/2835179875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb85b0 msgr2=0x562cdbfc2930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:24.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb85b0 0x562cdbfc2930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.700+0000 7f8bba000640 1 -- 172.21.15.28:0/2835179875 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562cdbeb18b0 con 0x562cdbfb7c90 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.701+0000 7f8bba000640 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb7c90 0x562cdbfb8070 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f8bac009ba0 tx=0x7f8bac011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4de68ec27fb4223a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.701+0000 7f8ba6600640 1 -- 172.21.15.28:0/2835179875 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bac004230 con 0x562cdbfb7c90 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.701+0000 7f8ba6600640 1 -- 172.21.15.28:0/2835179875 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8bac009e10 con 0x562cdbfb7c90 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.701+0000 7f8ba6600640 1 -- 172.21.15.28:0/2835179875 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bac004bc0 con 0x562cdbfb7c90 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bb2000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8b9c0b2c30 0x7f8b9c0b50f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bb2000640 1 -- 172.21.15.28:0/719887859 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8ba00e2b10 con 0x7f8bb40778a0 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2835179875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb7c90 msgr2=0x562cdbfb8070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb7c90 0x562cdbfb8070 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f8bac009ba0 tx=0x7f8bac011810 comp rx=0 tx=0).stop 2024-05-20T20:06:24.704 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bb9600640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8b9c0b2c30 0x7f8b9c0b50f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2835179875 shutdown_connections 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 0x562cdbfc5260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb85b0 0x562cdbfc2930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2835179875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb7c90 0x562cdbfb8070 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.702+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2835179875 >> 172.21.15.28:0/2835179875 conn(0x562cdbfb41a0 msgr2=0x562cdbfb4580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.703+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2835179875 shutdown_connections 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.703+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2835179875 wait complete. 2024-05-20T20:06:24.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.703+0000 7f8bbee6a740 1 -- start start 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.703+0000 7f8bb9600640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8b9c0b2c30 0x7f8b9c0b50f0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f8ba8009a60 tx=0x7f8ba802f3d0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.703+0000 7f8bbee6a740 1 Processor -- start 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc2800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfbfaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdbfc0070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562cdbbfe790 con 0x562cdbfb7c90 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562cdbec0bc0 con 0x562cdbfc2e70 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562cdbfc5aa0 con 0x562cdbfb85b0 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bbee6a740 10 client.0 _ll_register_callbacks cb 0x562cdbe9e400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:24.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bb9600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc2800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bba000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdbfc0070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bba000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdbfc0070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34614/0 (socket says 172.21.15.28:34614) 2024-05-20T20:06:24.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bba000640 1 -- 172.21.15.28:0/1280263609 learned_addr learned my addr 172.21.15.28:0/1280263609 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:24.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.704+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfbfaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8bba000640 1 -- 172.21.15.28:0/1280263609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 msgr2=0x562cdbfbfaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:24.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8bba000640 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfbfaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8bba000640 1 -- 172.21.15.28:0/1280263609 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 msgr2=0x562cdbfc2800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:24.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8bba000640 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc2800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8bba000640 1 -- 172.21.15.28:0/1280263609 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562cdbc2d210 con 0x562cdbfc2e70 2024-05-20T20:06:24.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfbfaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8bba000640 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdbfc0070 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f8bac005790 tx=0x7f8bac019af0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=136f5bf31a3e42d4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8b9aa00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.705+0000 7f8b9aa00640 1 -- 172.21.15.28:0/1280263609 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bac028020 con 0x562cdbfc2e70 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.706+0000 7f8b9aa00640 1 -- 172.21.15.28:0/1280263609 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8bac024470 con 0x562cdbfc2e70 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.706+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8bac009e10 con 0x562cdbfc2e70 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.706+0000 7f8b9aa00640 1 -- 172.21.15.28:0/1280263609 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8bac033050 con 0x562cdbfc2e70 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.706+0000 7f8bbee6a740 10 client.25406 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.706+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562cdbcf9620 con 0x562cdbfc2e70 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.706+0000 7f8b9aa00640 1 -- 172.21.15.28:0/1280263609 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8bac009e10 con 0x562cdbfc2e70 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8b9aa00640 1 -- 172.21.15.28:0/1280263609 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8bac018070 con 0x562cdbfc2e70 2024-05-20T20:06:24.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8bbee6a740 10 client.25406 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8bbee6a740 10 client.25406 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8bbee6a740 20 client.25406 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdbfd2710 0x562cdbfd4b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8bbee6a740 4 client.25406 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562cdbf2aae0 con 0x562cdbfd2710 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.707+0000 7f8bb9600640 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdbfd2710 0x562cdbfd4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.708+0000 7f8bb9600640 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdbfd2710 0x562cdbfd4b00 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=93bb812da75002bb server_cookie=57d094e91e82379 in_seq=0 out_seq=0 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.708+0000 7f8b9aa00640 10 client.25406 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8b9aa00640 1 -- 172.21.15.28:0/1280263609 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562cdbf2aae0 con 0x562cdbfd2710 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8b9aa00640 10 client.25406 handle_command_reply: tid=0 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 client.25406 shutdown 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 msgr2=0x562cdbfc0070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdbfc0070 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f8bac005790 tx=0x7f8bac019af0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 shutdown_connections 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdbfd2710 0x562cdbfd4b00 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdbfc0070 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfbfaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/1280263609 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc2800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8b9aa00640 0 client.25406 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 >> 172.21.15.28:0/1280263609 conn(0x562cdbfb41a0 msgr2=0x562cdbfb9f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.709+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 shutdown_connections 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.710+0000 7f8bbee6a740 1 -- 172.21.15.28:0/1280263609 wait complete. 2024-05-20T20:06:24.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.710+0000 7f8bbee6a740 20 client.25406 trim_cache size 0 max 16384 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 Processor -- start 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 -- start start 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfceb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb85b0 0x562cdbfda270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 0x562cdbfda7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb41b4c90 con 0x562cdbfb7c90 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb41b67d0 con 0x562cdbfb85b0 2024-05-20T20:06:24.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb41b5360 con 0x562cdbfc2e70 2024-05-20T20:06:24.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.727+0000 7f8bb9600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfceb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb9600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfceb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40614/0 (socket says 172.21.15.28:40614) 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb9600640 1 -- 172.21.15.28:0/2661567845 learned_addr learned my addr 172.21.15.28:0/2661567845 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb85b0 0x562cdbfda270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bba000640 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 0x562cdbfda7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb8c00640 1 -- 172.21.15.28:0/2661567845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 msgr2=0x562cdbfda7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 0x562cdbfda7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb8c00640 1 -- 172.21.15.28:0/2661567845 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 msgr2=0x562cdbfceb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfceb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb8c00640 1 -- 172.21.15.28:0/2661567845 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562cdbfb26b0 con 0x562cdbfb85b0 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb85b0 0x562cdbfda270 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f8ba00e0030 tx=0x7f8ba00a5350 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3d27a9f9f972d24b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8ba5c00640 1 -- 172.21.15.28:0/2661567845 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ba00c3070 con 0x562cdbfb85b0 2024-05-20T20:06:24.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8ba5c00640 1 -- 172.21.15.28:0/2661567845 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8ba00aa2c0 con 0x562cdbfb85b0 2024-05-20T20:06:24.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.728+0000 7f8ba5c00640 1 -- 172.21.15.28:0/2661567845 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ba00a5e50 con 0x562cdbfb85b0 2024-05-20T20:06:24.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2661567845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb85b0 msgr2=0x562cdbfda270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb85b0 0x562cdbfda270 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f8ba00e0030 tx=0x7f8ba00a5350 comp rx=0 tx=0).stop 2024-05-20T20:06:24.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2661567845 shutdown_connections 2024-05-20T20:06:24.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfb85b0 0x562cdbfda270 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfc2e70 0x562cdbfda7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/2661567845 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfceb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2661567845 >> 172.21.15.28:0/2661567845 conn(0x562cdbfb41a0 msgr2=0x562cdbfc1bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2661567845 shutdown_connections 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.729+0000 7f8bbee6a740 1 -- 172.21.15.28:0/2661567845 wait complete. 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.730+0000 7f8bbee6a740 1 -- start start 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.730+0000 7f8bbee6a740 1 Processor -- start 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc82e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfc8820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bbee6a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdc03b140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb41b6100 con 0x562cdbfb7c90 2024-05-20T20:06:24.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb41b5a30 con 0x562cdbfc2e70 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bbee6a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8bb41b2c70 con 0x562cdbfb85b0 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bbee6a740 10 client.0 _ll_register_callbacks cb 0x7f8ba0005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc82e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc82e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40624/0 (socket says 172.21.15.28:40624) 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 -- 172.21.15.28:0/789004604 learned_addr learned my addr 172.21.15.28:0/789004604 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bba000640 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdc03b140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 -- 172.21.15.28:0/789004604 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 msgr2=0x562cdbfc8820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfc8820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 -- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 msgr2=0x562cdc03b140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdc03b140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.731+0000 7f8bb9600640 1 -- 172.21.15.28:0/789004604 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562cdc03b780 con 0x562cdbfb7c90 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8bb9600640 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc82e0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f8ba8000f80 tx=0x7f8ba80095c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52a876219926a591 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:24.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8b9aa00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:24.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8b9aa00640 1 -- 172.21.15.28:0/789004604 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ba8003970 con 0x562cdbfb7c90 2024-05-20T20:06:24.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8b9aa00640 1 -- 172.21.15.28:0/789004604 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8ba8042030 con 0x562cdbfb7c90 2024-05-20T20:06:24.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8ba00aa2c0 con 0x562cdbfb7c90 2024-05-20T20:06:24.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8b9aa00640 1 -- 172.21.15.28:0/789004604 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8ba8003b10 con 0x562cdbfb7c90 2024-05-20T20:06:24.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8bbee6a740 10 client.15273 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:24.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.732+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562cdc03ca90 con 0x562cdbfb7c90 2024-05-20T20:06:24.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8b9aa00640 1 -- 172.21.15.28:0/789004604 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8ba00aa2c0 con 0x562cdbfb7c90 2024-05-20T20:06:24.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8b9aa00640 1 -- 172.21.15.28:0/789004604 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8ba8055410 con 0x562cdbfb7c90 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8bbee6a740 10 client.15273 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8bbee6a740 10 client.15273 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8bbee6a740 20 client.15273 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdc03d1b0 0x562cdc03f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8bbee6a740 4 client.15273 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x562cdbf242c0 con 0x562cdc03d1b0 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.733+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdc03d1b0 0x562cdc03f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.734+0000 7f8bb8c00640 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdc03d1b0 0x562cdc03f5f0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8d763a84e4215618 server_cookie=3231c1ca788d5ad2 in_seq=0 out_seq=0 2024-05-20T20:06:24.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.734+0000 7f8b9aa00640 10 client.15273 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:24.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8b9aa00640 1 -- 172.21.15.28:0/789004604 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x562cdbf242c0 con 0x562cdc03d1b0 2024-05-20T20:06:24.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8b9aa00640 10 client.15273 handle_command_reply: tid=0 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 client.15273 shutdown 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 msgr2=0x562cdbfc82e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc82e0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f8ba8000f80 tx=0x7f8ba80095c0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 shutdown_connections 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562cdc03d1b0 0x562cdc03f5f0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562cdbfc2e70 0x562cdc03b140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562cdbfb85b0 0x562cdbfc8820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 --2- 172.21.15.28:0/789004604 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562cdbfb7c90 0x562cdbfc82e0 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 >> 172.21.15.28:0/789004604 conn(0x562cdbfb41a0 msgr2=0x562cdbfb3310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8b9aa00640 0 client.15273 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 shutdown_connections 2024-05-20T20:06:24.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 1 -- 172.21.15.28:0/789004604 wait complete. 2024-05-20T20:06:24.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.741+0000 7f8bbee6a740 20 client.15273 trim_cache size 0 max 16384 2024-05-20T20:06:24.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:06:24.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:06:24.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:06:24.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8b9c0b2c30 msgr2=0x7f8b9c0b50f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8b9c0b2c30 0x7f8b9c0b50f0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f8ba8009a60 tx=0x7f8ba802f3d0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb40778a0 msgr2=0x7f8bb41a59a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:24.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb40778a0 0x7f8bb41a59a0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f8ba000e9f0 tx=0x7f8ba000eeb0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 shutdown_connections 2024-05-20T20:06:24.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bb40778a0 0x7f8bb41a59a0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8b9c0b2c30 0x7f8b9c0b50f0 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bb41138e0 0x7f8bb419f660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 --2- 172.21.15.28:0/719887859 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bb4076f80 0x7f8bb41a5460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:24.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.742+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 >> 172.21.15.28:0/719887859 conn(0x7f8bb4100750 msgr2=0x7f8bb4115000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:24.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.743+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 shutdown_connections 2024-05-20T20:06:24.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:24.743+0000 7f8bbc000640 1 -- 172.21.15.28:0/719887859 wait complete. 2024-05-20T20:06:25.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:24 smithi028 ceph-mon[31178]: pgmap v26: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 1.7 KiB/s wr, 16 op/s 2024-05-20T20:06:25.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:24 smithi154 ceph-mon[36314]: pgmap v26: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 1.7 KiB/s wr, 16 op/s 2024-05-20T20:06:25.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:24 smithi033 ceph-mon[36179]: pgmap v26: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 1.7 KiB/s wr, 16 op/s 2024-05-20T20:06:26.834 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:26 smithi028 ceph-mon[31178]: pgmap v27: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:06:27.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:26 smithi154 ceph-mon[36314]: pgmap v27: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:06:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:26 smithi033 ceph-mon[36179]: pgmap v27: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:06:27.718 INFO:teuthology.orchestra.run.smithi028.stdout:{"run_dir":"/var/run/ceph"} 2024-05-20T20:06:27.718 DEBUG:tasks.cephfs.filesystem:_json_asok output { "run_dir": "/var/run/ceph" } 2024-05-20T20:06:27.719 INFO:tasks.quiescer.fs.[cephfs]:Initializing Quiescer with seed 2891 2024-05-20T20:06:27.719 DEBUG:tasks.quiescer:Yielding 2024-05-20T20:06:27.719 INFO:teuthology.run_tasks:Running task workunit... 2024-05-20T20:06:27.737 INFO:tasks.workunit:Pulling workunits from ref 4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T20:06:27.738 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-05-20T20:06:27.738 INFO:tasks.workunit:timeout=3h 2024-05-20T20:06:27.738 INFO:tasks.workunit:cleanup=True 2024-05-20T20:06:27.738 DEBUG:teuthology.orchestra.run.smithi028:> stat -- /home/ubuntu/cephtest/mnt.0 2024-05-20T20:06:27.739 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:06:27.791 INFO:teuthology.orchestra.run.smithi028.stdout: File: /home/ubuntu/cephtest/mnt.0 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 0 Blocks: 0 IO Block: 65536 directory 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 77h/119d Inode: 1099511628283 Links: 2 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (1777/drwxrwxrwt) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout:Context: system_u:object_r:unlabeled_t:s0 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-05-20 20:02:06.091121010 +0000 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-05-20 20:02:06.091121010 +0000 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-05-20 20:05:35.195801995 +0000 2024-05-20T20:06:27.792 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: 2024-05-20 20:02:06.091121010 +0000 2024-05-20T20:06:27.792 INFO:tasks.workunit:Did not need to create dir /home/ubuntu/cephtest/mnt.0 2024-05-20T20:06:27.792 DEBUG:teuthology.orchestra.run.smithi028:> cd -- /home/ubuntu/cephtest/mnt.0 && sudo install -d -m 0755 --owner=ubuntu -- client.0 2024-05-20T20:06:27.926 DEBUG:teuthology.orchestra.run.smithi028:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 4ebd1d5361d457738957dc0455bba90102296634 2024-05-20T20:06:27.983 INFO:tasks.workunit.client.0.smithi028.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-05-20T20:06:28.208 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:29.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:28 smithi028 ceph-mon[31178]: pgmap v28: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:06:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:28 smithi154 ceph-mon[36314]: pgmap v28: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:06:29.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:28 smithi033 ceph-mon[36179]: pgmap v28: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:06:30.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 -- 172.21.15.28:0/394416345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c075770 msgr2=0x7f6f9c075b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:30.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/394416345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c075770 0x7f6f9c075b50 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c009990 tx=0x7f6f8c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 -- 172.21.15.28:0/394416345 shutdown_connections 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/394416345 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c10ff40 0x7f6f9c112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/394416345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c075770 0x7f6f9c075b50 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/394416345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6f9c076090 0x7f6f9c10fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 -- 172.21.15.28:0/394416345 >> 172.21.15.28:0/394416345 conn(0x7f6f9c07a6a0 msgr2=0x7f6f9c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 -- 172.21.15.28:0/394416345 shutdown_connections 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.681+0000 7f6fa3a00700 1 -- 172.21.15.28:0/394416345 wait complete. 2024-05-20T20:06:30.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 Processor -- start 2024-05-20T20:06:30.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 -- start start 2024-05-20T20:06:30.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c075770 0x7f6f9c19bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:30.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c076090 0x7f6f9c19c410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:30.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6f9c10ff40 0x7f6f9c1a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:30.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6f9c115960 con 0x7f6f9c076090 2024-05-20T20:06:30.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6f9c1157e0 con 0x7f6f9c075770 2024-05-20T20:06:30.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.683+0000 7f6fa3a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6f9c115ae0 con 0x7f6f9c10ff40 2024-05-20T20:06:30.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6f93e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c076090 0x7f6f9c19c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:30.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c075770 0x7f6f9c19bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:30.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6f9c10ff40 0x7f6f9c1a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:30.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c075770 0x7f6f9c19bed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37336/0 (socket says 172.21.15.28:37336) 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 -- 172.21.15.28:0/3554977570 learned_addr learned my addr 172.21.15.28:0/3554977570 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 -- 172.21.15.28:0/3554977570 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6f9c10ff40 msgr2=0x7f6f9c1a0750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6f9c10ff40 0x7f6f9c1a0750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 -- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c076090 msgr2=0x7f6f9c19c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c076090 0x7f6f9c19c410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1000700 1 -- 172.21.15.28:0/3554977570 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6f9c1a0e90 con 0x7f6f9c075770 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6f93e00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c076090 0x7f6f9c19c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.684+0000 7f6fa1a00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6f9c10ff40 0x7f6f9c1a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:30.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.685+0000 7f6fa1000700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c075770 0x7f6f9c19bed0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c000c00 tx=0x7f6f8c0057c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:30.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.685+0000 7f6f9a000700 1 -- 172.21.15.28:0/3554977570 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c027070 con 0x7f6f9c075770 2024-05-20T20:06:30.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.685+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6f9c1a1120 con 0x7f6f9c075770 2024-05-20T20:06:30.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.685+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6f9c1a8a00 con 0x7f6f9c075770 2024-05-20T20:06:30.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.685+0000 7f6f9a000700 1 -- 172.21.15.28:0/3554977570 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6f8c022470 con 0x7f6f9c075770 2024-05-20T20:06:30.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.685+0000 7f6f9a000700 1 -- 172.21.15.28:0/3554977570 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c017650 con 0x7f6f9c075770 2024-05-20T20:06:30.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.687+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6f78005330 con 0x7f6f9c075770 2024-05-20T20:06:30.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.689+0000 7f6f9a000700 1 -- 172.21.15.28:0/3554977570 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f6f8c02a070 con 0x7f6f9c075770 2024-05-20T20:06:30.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.690+0000 7f6f9a000700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f7c0b2c20 0x7f6f7c0b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:30.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.691+0000 7f6f9a000700 1 -- 172.21.15.28:0/3554977570 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f6f8c0e3430 con 0x7f6f9c075770 2024-05-20T20:06:30.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.691+0000 7f6f93e00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f7c0b2c20 0x7f6f7c0b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:30.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.692+0000 7f6f93e00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f7c0b2c20 0x7f6f7c0b50e0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f6f9c111c40 tx=0x7f6f84009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:30.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.693+0000 7f6f9a000700 1 -- 172.21.15.28:0/3554977570 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f6f8c013070 con 0x7f6f9c075770 2024-05-20T20:06:30.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.981+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f6f78005620 con 0x7f6f9c075770 2024-05-20T20:06:30.982 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:30 smithi028 ceph-mon[31178]: pgmap v29: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 1.9 KiB/s wr, 14 op/s 2024-05-20T20:06:30.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.982+0000 7f6f9a000700 1 -- 172.21.15.28:0/3554977570 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f6f8c0aac70 con 0x7f6f9c075770 2024-05-20T20:06:30.983 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:30.983 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:06:30.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f7c0b2c20 msgr2=0x7f6f7c0b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:30.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f7c0b2c20 0x7f6f7c0b50e0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f6f9c111c40 tx=0x7f6f84009260 comp rx=0 tx=0).stop 2024-05-20T20:06:30.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c075770 msgr2=0x7f6f9c19bed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:30.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c075770 0x7f6f9c19bed0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c000c00 tx=0x7f6f8c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 shutdown_connections 2024-05-20T20:06:30.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6f9c10ff40 0x7f6f9c1a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f7c0b2c20 0x7f6f7c0b50e0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6f9c075770 0x7f6f9c19bed0 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 --2- 172.21.15.28:0/3554977570 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6f9c076090 0x7f6f9c19c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:30.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 >> 172.21.15.28:0/3554977570 conn(0x7f6f9c07a6a0 msgr2=0x7f6f9c1007f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:30.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 shutdown_connections 2024-05-20T20:06:30.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:30.986+0000 7f6fa3a00700 1 -- 172.21.15.28:0/3554977570 wait complete. 2024-05-20T20:06:30.988 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:06:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:30 smithi154 ceph-mon[36314]: pgmap v29: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 1.9 KiB/s wr, 14 op/s 2024-05-20T20:06:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:30 smithi033 ceph-mon[36179]: pgmap v29: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 1.9 KiB/s wr, 14 op/s 2024-05-20T20:06:31.912 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 29, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-05-20T20:01:26.231967+0000', 'modified': '2024-05-20T20:05:36.255795+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}, 'max_mds': 5, 'in': [0, 1, 2, 3, 4], 'up': {'mds_0': 24542, 'mds_1': 24500, 'mds_2': 24493, 'mds_3': 24482, 'mds_4': 24475}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_14400': {'gid': 14400, 'name': 'a', 'rank': 4, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.28:6841/1736910968', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.28:6840', 'nonce': 1736910968}, {'type': 'v1', 'addr': '172.21.15.28:6841', 'nonce': 1736910968}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24427': {'gid': 24427, 'name': 'l', 'rank': 3, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.154:6833/2654302434', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6832', 'nonce': 2654302434}, {'type': 'v1', 'addr': '172.21.15.154:6833', 'nonce': 2654302434}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24442': {'gid': 24442, 'name': 'j', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.28:6835/34026174', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.28:6834', 'nonce': 34026174}, {'type': 'v1', 'addr': '172.21.15.28:6835', 'nonce': 34026174}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24455': {'gid': 24455, 'name': 'k', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.33:6833/260147664', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6832', 'nonce': 260147664}, {'type': 'v1', 'addr': '172.21.15.33:6833', 'nonce': 260147664}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24463': {'gid': 24463, 'name': 'f', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.154:6835/3965936826', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6834', 'nonce': 3965936826}, {'type': 'v1', 'addr': '172.21.15.154:6835', 'nonce': 3965936826}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24475': {'gid': 24475, 'name': 'c', 'rank': 4, 'incarnation': 25, 'state': 'up:active', 'state_seq': 17, 'addr': '172.21.15.154:6837/2808233682', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6836', 'nonce': 2808233682}, {'type': 'v1', 'addr': '172.21.15.154:6837', 'nonce': 2808233682}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24482': {'gid': 24482, 'name': 'e', 'rank': 3, 'incarnation': 23, 'state': 'up:active', 'state_seq': 18, 'addr': '172.21.15.33:6835/4076526106', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6834', 'nonce': 4076526106}, {'type': 'v1', 'addr': '172.21.15.33:6835', 'nonce': 4076526106}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24493': {'gid': 24493, 'name': 'i', 'rank': 2, 'incarnation': 21, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.154:6839/1291909351', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.154:6838', 'nonce': 1291909351}, {'type': 'v1', 'addr': '172.21.15.154:6839', 'nonce': 1291909351}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24500': {'gid': 24500, 'name': 'h', 'rank': 1, 'incarnation': 19, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.33:6837/4183908883', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6836', 'nonce': 4183908883}, {'type': 'v1', 'addr': '172.21.15.33:6837', 'nonce': 4183908883}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}, 'gid_24542': {'gid': 24542, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 7, 'addr': '172.21.15.33:6839/2646117588', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.33:6838', 'nonce': 2646117588}, {'type': 'v1', 'addr': '172.21.15.33:6839', 'nonce': 2646117588}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540701547738038271, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2', 'feature_11': 'minor log segments', 'feature_12': 'quiesce subvolumes'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0, 'qdb_leader': 24542, 'qdb_cluster': [24542, 24500, 24493, 24475, 24482]} 2024-05-20T20:06:31.912 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 5/5 2024-05-20T20:06:31.912 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.c status 2024-05-20T20:06:32.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:31 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3554977570' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:32.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:31 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3554977570' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:32.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:31 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3554977570' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:32.393 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:32.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:32 smithi028 ceph-mon[31178]: pgmap v30: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-05-20T20:06:33.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:32 smithi154 ceph-mon[36314]: pgmap v30: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-05-20T20:06:33.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:32 smithi033 ceph-mon[36179]: pgmap v30: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 1.9 KiB/s wr, 10 op/s 2024-05-20T20:06:34.054 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 -- 172.21.15.28:0/330733300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a4102220 msgr2=0x7f63a4102600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 --2- 172.21.15.28:0/330733300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a4102220 0x7f63a4102600 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f6398009990 tx=0x7f6398009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 -- 172.21.15.28:0/330733300 shutdown_connections 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 --2- 172.21.15.28:0/330733300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63a410fdd0 0x7f63a41121c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 --2- 172.21.15.28:0/330733300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a4102220 0x7f63a4102600 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 --2- 172.21.15.28:0/330733300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63a4102bd0 0x7f63a410f890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 -- 172.21.15.28:0/330733300 >> 172.21.15.28:0/330733300 conn(0x7f63a40fde60 msgr2=0x7f63a4100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:34.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.852+0000 7f63aa400700 1 -- 172.21.15.28:0/330733300 shutdown_connections 2024-05-20T20:06:34.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.853+0000 7f63aa400700 1 -- 172.21.15.28:0/330733300 wait complete. 2024-05-20T20:06:34.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.854+0000 7f63aa400700 1 Processor -- start 2024-05-20T20:06:34.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.854+0000 7f63aa400700 1 -- start start 2024-05-20T20:06:34.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.854+0000 7f63aa400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63a4102220 0x7f63a419bd70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63aa400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63a4102bd0 0x7f63a419c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63aa400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a410fdd0 0x7f63a41a05f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63aa400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f63a4115860 con 0x7f63a410fdd0 2024-05-20T20:06:34.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63aa400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f63a41156e0 con 0x7f63a4102bd0 2024-05-20T20:06:34.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63aa400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f63a41159e0 con 0x7f63a4102220 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63a9a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a410fdd0 0x7f63a41a05f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63a9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63a4102220 0x7f63a419bd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63a9a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a410fdd0 0x7f63a41a05f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41802/0 (socket says 172.21.15.28:41802) 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63a9a00700 1 -- 172.21.15.28:0/4046411956 learned_addr learned my addr 172.21.15.28:0/4046411956 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.855+0000 7f63a3e00700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63a4102bd0 0x7f63a419c2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a9a00700 1 -- 172.21.15.28:0/4046411956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63a4102220 msgr2=0x7f63a419bd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a9a00700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63a4102220 0x7f63a419bd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a9a00700 1 -- 172.21.15.28:0/4046411956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63a4102bd0 msgr2=0x7f63a419c2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a9a00700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63a4102bd0 0x7f63a419c2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a9a00700 1 -- 172.21.15.28:0/4046411956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f63a41a0d30 con 0x7f63a410fdd0 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a9000700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63a4102220 0x7f63a419bd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:34.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a9a00700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a410fdd0 0x7f63a41a05f0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f639c00ef90 tx=0x7f639c00c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:34.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a1600700 1 -- 172.21.15.28:0/4046411956 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f639c0092e0 con 0x7f63a410fdd0 2024-05-20T20:06:34.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f63a41a0fc0 con 0x7f63a410fdd0 2024-05-20T20:06:34.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a1600700 1 -- 172.21.15.28:0/4046411956 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f639c004bb0 con 0x7f63a410fdd0 2024-05-20T20:06:34.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63a1600700 1 -- 172.21.15.28:0/4046411956 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f639c004280 con 0x7f63a410fdd0 2024-05-20T20:06:34.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.856+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f63a41a88a0 con 0x7f63a410fdd0 2024-05-20T20:06:34.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.861+0000 7f63a1600700 1 -- 172.21.15.28:0/4046411956 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f639c01b030 con 0x7f63a410fdd0 2024-05-20T20:06:34.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.862+0000 7f63b6316b80 1 Processor -- start 2024-05-20T20:06:34.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.862+0000 7f63b6316b80 1 -- start start 2024-05-20T20:06:34.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.862+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86e2cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86d48e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63b6316b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5626d8632df0 con 0x5626d86e28d0 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63b6316b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5626d8622990 con 0x5626d86d4460 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63b6316b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5626d86269d0 con 0x5626d86d4e20 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63a9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86d48e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63a9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86d48e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37362/0 (socket says 172.21.15.28:37362) 2024-05-20T20:06:34.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63a9000700 1 -- 172.21.15.28:0/3512078728 learned_addr learned my addr 172.21.15.28:0/3512078728 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63a1600700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f638c0b2ca0 0x7f638c0b5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63a1600700 1 -- 172.21.15.28:0/4046411956 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f639c0e22e0 con 0x7f63a410fdd0 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.863+0000 7f63a9000700 1 -- 172.21.15.28:0/3512078728 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 msgr2=0x5626d86e9670 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a3e00700 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86e2cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a9a00700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f638c0b2ca0 0x7f638c0b5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a9000700 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9670 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a9000700 1 -- 172.21.15.28:0/3512078728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 msgr2=0x5626d86e2cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:34.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a9000700 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86e2cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a9000700 1 -- 172.21.15.28:0/3512078728 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5626d85eb3d0 con 0x5626d86d4460 2024-05-20T20:06:34.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a3e00700 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86e2cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:06:34.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.864+0000 7f63a9000700 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86d48e0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f6398009990 tx=0x7f6398005960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eb127f23ea6befab server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:34.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.865+0000 7f6396000700 1 -- 172.21.15.28:0/3512078728 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6398027070 con 0x5626d86d4460 2024-05-20T20:06:34.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.865+0000 7f6396000700 1 -- 172.21.15.28:0/3512078728 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6398003680 con 0x5626d86d4460 2024-05-20T20:06:34.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.865+0000 7f6396000700 1 -- 172.21.15.28:0/3512078728 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f639800f6c0 con 0x5626d86d4460 2024-05-20T20:06:34.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63b6316b80 1 -- 172.21.15.28:0/3512078728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 msgr2=0x5626d86d48e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86d48e0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f6398009990 tx=0x7f6398005960 comp rx=0 tx=0).stop 2024-05-20T20:06:34.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63a9a00700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f638c0b2ca0 0x7f638c0b5160 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f639c0df860 tx=0x7f639c009e70 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63b6316b80 1 -- 172.21.15.28:0/3512078728 shutdown_connections 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86e2cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3512078728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86d48e0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.866+0000 7f63b6316b80 1 -- 172.21.15.28:0/3512078728 >> 172.21.15.28:0/3512078728 conn(0x5626d8602b90 msgr2=0x5626d86cf260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.867+0000 7f63b6316b80 1 -- 172.21.15.28:0/3512078728 shutdown_connections 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.867+0000 7f63b6316b80 1 -- 172.21.15.28:0/3512078728 wait complete. 2024-05-20T20:06:34.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.867+0000 7f63b6316b80 1 -- start start 2024-05-20T20:06:34.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 1 Processor -- start 2024-05-20T20:06:34.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d86e9320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d86e66c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86e6c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5626d8528f50 con 0x5626d86d4460 2024-05-20T20:06:34.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5626d85c3710 con 0x5626d86d4e20 2024-05-20T20:06:34.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5626d86e9c10 con 0x5626d86e28d0 2024-05-20T20:06:34.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.868+0000 7f63b6316b80 10 client.0 _ll_register_callbacks cb 0x5626d85ccc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d86e66c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d86e9320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d86e66c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37364/0 (socket says 172.21.15.28:37364) 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 -- 172.21.15.28:0/3890013095 learned_addr learned my addr 172.21.15.28:0/3890013095 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a3e00700 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86e6c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 -- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 msgr2=0x5626d86e6c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86e6c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 -- 172.21.15.28:0/3890013095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 msgr2=0x5626d86e9320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d86e9320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a9000700 1 -- 172.21.15.28:0/3890013095 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5626d8572880 con 0x5626d86d4e20 2024-05-20T20:06:34.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.869+0000 7f63a3e00700 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d86e9320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:34.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.870+0000 7f63a9000700 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d86e66c0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f63980098e0 tx=0x7f63980051e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=df2e51df374c7e86 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:34.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.870+0000 7f638aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:34.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.870+0000 7f638aa00700 1 -- 172.21.15.28:0/3890013095 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6398030020 con 0x5626d86d4e20 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.870+0000 7f638aa00700 1 -- 172.21.15.28:0/3890013095 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f6398005600 con 0x5626d86d4e20 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.870+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6398003680 con 0x5626d86d4e20 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.871+0000 7f638aa00700 1 -- 172.21.15.28:0/3890013095 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6398027040 con 0x5626d86d4e20 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.871+0000 7f63b6316b80 10 client.25430 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.871+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5626d85f3010 con 0x5626d86d4e20 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.871+0000 7f638aa00700 1 -- 172.21.15.28:0/3890013095 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6398003680 con 0x5626d86d4e20 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.871+0000 7f638aa00700 1 -- 172.21.15.28:0/3890013095 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f639802c410 con 0x5626d86d4e20 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.872+0000 7f63b6316b80 10 client.25430 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.872+0000 7f63b6316b80 10 client.25430 resolve_mds: resolved name 'c' to daemon mds.c 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.872+0000 7f63b6316b80 20 client.25430 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:34.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.872+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d86f5430 0x5626d86f74e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.872+0000 7f63b6316b80 4 client.25430 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:34.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.872+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5626d8634d40 con 0x5626d86f5430 2024-05-20T20:06:34.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.872+0000 7f63a3e00700 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d86f5430 0x5626d86f74e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.873+0000 7f63a3e00700 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d86f5430 0x5626d86f74e0 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=ddce22e89976c65c server_cookie=4db473114d0c8ddd in_seq=0 out_seq=0 2024-05-20T20:06:34.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.873+0000 7f638aa00700 10 client.25430 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:34.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f638aa00700 1 -- 172.21.15.28:0/3890013095 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5626d8634d40 con 0x5626d86f5430 2024-05-20T20:06:34.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f638aa00700 10 client.25430 handle_command_reply: tid=0 2024-05-20T20:06:34.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 client.25430 shutdown 2024-05-20T20:06:34.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 msgr2=0x5626d86e66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d86e66c0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f63980098e0 tx=0x7f63980051e0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 shutdown_connections 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d86f5430 0x5626d86f74e0 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86e6c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d86e9320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3890013095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d86e66c0 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f638aa00700 0 client.25430 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 >> 172.21.15.28:0/3890013095 conn(0x5626d8602b90 msgr2=0x5626d86d3450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 shutdown_connections 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.874+0000 7f63b6316b80 1 -- 172.21.15.28:0/3890013095 wait complete. 2024-05-20T20:06:34.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.875+0000 7f63b6316b80 20 client.25430 trim_cache size 0 max 16384 2024-05-20T20:06:34.878 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:34.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 Processor -- start 2024-05-20T20:06:34.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 -- start start 2024-05-20T20:06:34.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86c9a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86c8f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5626d86f5820 con 0x5626d86e28d0 2024-05-20T20:06:34.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5626d86f5ef0 con 0x5626d86d4460 2024-05-20T20:06:34.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.899+0000 7f63b6316b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5626d86f65c0 con 0x5626d86d4e20 2024-05-20T20:06:34.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86c9a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86c8f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86c8f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41846/0 (socket says 172.21.15.28:41846) 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a3e00700 1 -- 172.21.15.28:0/3701311406 learned_addr learned my addr 172.21.15.28:0/3701311406 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57182/0 (socket says 172.21.15.28:57182) 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a9000700 1 -- 172.21.15.28:0/3701311406 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 msgr2=0x5626d86c9a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a9000700 1 --2- 172.21.15.28:0/3701311406 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86c9a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a9000700 1 -- 172.21.15.28:0/3701311406 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 msgr2=0x5626d86c8f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a9000700 1 --2- 172.21.15.28:0/3701311406 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86c8f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.900+0000 7f63a9000700 1 -- 172.21.15.28:0/3701311406 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5626d86ddc90 con 0x5626d86d4e20 2024-05-20T20:06:34.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.901+0000 7f63a9000700 1 --2- 172.21.15.28:0/3701311406 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9320 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f6398006010 tx=0x7f6398000bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=18b33eb65ce7df3e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:34.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.901+0000 7f6395600700 1 -- 172.21.15.28:0/3701311406 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6398042070 con 0x5626d86d4e20 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.901+0000 7f6395600700 1 -- 172.21.15.28:0/3701311406 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f639802c5d0 con 0x5626d86d4e20 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.901+0000 7f6395600700 1 -- 172.21.15.28:0/3701311406 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6398030040 con 0x5626d86d4e20 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 -- 172.21.15.28:0/3701311406 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 msgr2=0x5626d86e9320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3701311406 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9320 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f6398006010 tx=0x7f6398000bc0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 -- 172.21.15.28:0/3701311406 shutdown_connections 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3701311406 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86e28d0 0x5626d86c8f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3701311406 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4460 0x5626d86c9a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 --2- 172.21.15.28:0/3701311406 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86d4e20 0x5626d86e9320 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 -- 172.21.15.28:0/3701311406 >> 172.21.15.28:0/3701311406 conn(0x5626d8602b90 msgr2=0x5626d86d3d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 -- 172.21.15.28:0/3701311406 shutdown_connections 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.902+0000 7f63b6316b80 1 -- 172.21.15.28:0/3701311406 wait complete. 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 -- start start 2024-05-20T20:06:34.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 Processor -- start 2024-05-20T20:06:34.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d8700b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d8701080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86fb970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5626d86f7360 con 0x5626d86d4460 2024-05-20T20:06:34.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5626d86f6c90 con 0x5626d86d4e20 2024-05-20T20:06:34.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.903+0000 7f63b6316b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5626d86f7a30 con 0x5626d86e28d0 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63b6316b80 10 client.0 _ll_register_callbacks cb 0x5626d85ccc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d8700b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d8701080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d8700b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41852/0 (socket says 172.21.15.28:41852) 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 -- 172.21.15.28:0/2823154934 learned_addr learned my addr 172.21.15.28:0/2823154934 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86fb970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 -- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 msgr2=0x5626d86fb970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86fb970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 -- 172.21.15.28:0/2823154934 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 msgr2=0x5626d8701080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d8701080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a3e00700 1 -- 172.21.15.28:0/2823154934 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5626d86fc1b0 con 0x5626d86d4460 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.904+0000 7f63a9000700 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d8701080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:34.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f63a3e00700 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d8700b40 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f63900097a0 tx=0x7f6390015040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c0543894b18962 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:34.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f638aa00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:34.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f638aa00700 1 -- 172.21.15.28:0/2823154934 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f63900129b0 con 0x5626d86d4460 2024-05-20T20:06:34.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f638aa00700 1 -- 172.21.15.28:0/2823154934 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f6390012b50 con 0x5626d86d4460 2024-05-20T20:06:34.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5626d862e520 con 0x5626d86d4460 2024-05-20T20:06:34.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f638aa00700 1 -- 172.21.15.28:0/2823154934 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6390012d70 con 0x5626d86d4460 2024-05-20T20:06:34.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f63b6316b80 10 client.15294 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:34.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.905+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5626d8762400 con 0x5626d86d4460 2024-05-20T20:06:34.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f638aa00700 1 -- 172.21.15.28:0/2823154934 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x5626d862e520 con 0x5626d86d4460 2024-05-20T20:06:34.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f638aa00700 1 -- 172.21.15.28:0/2823154934 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f639002e4b0 con 0x5626d86d4460 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f63b6316b80 10 client.15294 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f63b6316b80 10 client.15294 resolve_mds: resolved name 'c' to daemon mds.c 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f63b6316b80 20 client.15294 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f63b6316b80 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d8767010 0x5626d8765d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f63b6316b80 4 client.15294 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.906+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "status"}) -- 0x5626d84a5200 con 0x5626d8767010 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.907+0000 7f63a9000700 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d8767010 0x5626d8765d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.907+0000 7f63a9000700 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d8767010 0x5626d8765d20 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=3384494f95c1979f server_cookie=9206ebffc0c69955 in_seq=0 out_seq=0 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.907+0000 7f638aa00700 10 client.15294 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.907+0000 7f638aa00700 1 -- 172.21.15.28:0/2823154934 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+327 (crc 0 0 0) 0x5626d84a5200 con 0x5626d8767010 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.907+0000 7f638aa00700 10 client.15294 handle_command_reply: tid=0 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.907+0000 7f63b6316b80 1 client.15294 shutdown 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 msgr2=0x5626d8700b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d8700b40 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f63900097a0 tx=0x7f6390015040 comp rx=0 tx=0).stop 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 shutdown_connections 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5626d8767010 0x5626d8765d20 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5626d86e28d0 0x5626d86fb970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5626d86d4460 0x5626d8700b40 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 --2- 172.21.15.28:0/2823154934 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5626d86d4e20 0x5626d8701080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 >> 172.21.15.28:0/2823154934 conn(0x5626d8602b90 msgr2=0x5626d86e64a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f638aa00700 0 client.15294 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 shutdown_connections 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 1 -- 172.21.15.28:0/2823154934 wait complete. 2024-05-20T20:06:34.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.908+0000 7f63b6316b80 20 client.15294 trim_cache size 0 max 16384 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 4, 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24475, 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 289.26155564800001, 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 29, 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 81, 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 80, 2024-05-20T20:06:34.911 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 350.372011579 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f638c0b2ca0 msgr2=0x7f638c0b5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f638c0b2ca0 0x7f638c0b5160 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f639c0df860 tx=0x7f639c009e70 comp rx=0 tx=0).stop 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a410fdd0 msgr2=0x7f63a41a05f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a410fdd0 0x7f63a41a05f0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f639c00ef90 tx=0x7f639c00c3a0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 shutdown_connections 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63a410fdd0 0x7f63a41a05f0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63a4102220 0x7f63a419bd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f638c0b2ca0 0x7f638c0b5160 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 --2- 172.21.15.28:0/4046411956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63a4102bd0 0x7f63a419c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:34.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 >> 172.21.15.28:0/4046411956 conn(0x7f63a40fde60 msgr2=0x7f63a4100250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:34.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 shutdown_connections 2024-05-20T20:06:34.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:34.909+0000 7f63aa400700 1 -- 172.21.15.28:0/4046411956 wait complete. 2024-05-20T20:06:35.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:34 smithi028 ceph-mon[31178]: pgmap v31: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-05-20T20:06:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:34 smithi154 ceph-mon[36314]: pgmap v31: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-05-20T20:06:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:34 smithi033 ceph-mon[36179]: pgmap v31: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 2.0 KiB/s wr, 14 op/s 2024-05-20T20:06:36.809 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:36 smithi028 ceph-mon[31178]: pgmap v32: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.0 KiB/s wr, 10 op/s 2024-05-20T20:06:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:36 smithi154 ceph-mon[36314]: pgmap v32: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.0 KiB/s wr, 10 op/s 2024-05-20T20:06:37.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:36 smithi033 ceph-mon[36179]: pgmap v32: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.0 KiB/s wr, 10 op/s 2024-05-20T20:06:37.205 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.e status 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 -- 172.21.15.28:0/243650239 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 msgr2=0x7fb3f01088e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/243650239 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 0x7fb3f01088e0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fb3e0009990 tx=0x7fb3e0009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 -- 172.21.15.28:0/243650239 shutdown_connections 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/243650239 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f0103f90 0x7fb3f0110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/243650239 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 0x7fb3f01088e0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/243650239 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3f01035f0 0x7fb3f0103a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 -- 172.21.15.28:0/243650239 >> 172.21.15.28:0/243650239 conn(0x7fb3f007a6a0 msgr2=0x7fb3f007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 -- 172.21.15.28:0/243650239 shutdown_connections 2024-05-20T20:06:37.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.792+0000 7fb3f5c00700 1 -- 172.21.15.28:0/243650239 wait complete. 2024-05-20T20:06:37.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.793+0000 7fb3f5c00700 1 Processor -- start 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.793+0000 7fb3f5c00700 1 -- start start 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3f5c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f01035f0 0x7fb3f019be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3f5c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3f0103f90 0x7fb3f019c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3f5c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 0x7fb3f01a06e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3f5c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f0113790 con 0x7fb3f0103f90 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3f5c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f0113610 con 0x7fb3f0108500 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3f5c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f0113910 con 0x7fb3f01035f0 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3eea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f01035f0 0x7fb3f019be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3eea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f01035f0 0x7fb3f019be10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38518/0 (socket says 172.21.15.28:38518) 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3eea00700 1 -- 172.21.15.28:0/3311514178 learned_addr learned my addr 172.21.15.28:0/3311514178 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3ef400700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 0x7fb3f01a06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.794+0000 7fb3ee000700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3f0103f90 0x7fb3f019c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3eea00700 1 -- 172.21.15.28:0/3311514178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 msgr2=0x7fb3f01a06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3eea00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 0x7fb3f01a06e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3eea00700 1 -- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3f0103f90 msgr2=0x7fb3f019c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3eea00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3f0103f90 0x7fb3f019c350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3eea00700 1 -- 172.21.15.28:0/3311514178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb3f01a0e20 con 0x7fb3f01035f0 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3ef400700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 0x7fb3f01a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:06:37.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3eea00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f01035f0 0x7fb3f019be10 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fb3e0000c00 tx=0x7fb3e000bfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3e7400700 1 -- 172.21.15.28:0/3311514178 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3e0027070 con 0x7fb3f01035f0 2024-05-20T20:06:37.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3e7400700 1 -- 172.21.15.28:0/3311514178 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3e0022470 con 0x7fb3f01035f0 2024-05-20T20:06:37.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.795+0000 7fb3e7400700 1 -- 172.21.15.28:0/3311514178 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3e0017650 con 0x7fb3f01035f0 2024-05-20T20:06:37.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.796+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb3f01a10b0 con 0x7fb3f01035f0 2024-05-20T20:06:37.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.796+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb3f01a8990 con 0x7fb3f01035f0 2024-05-20T20:06:37.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.800+0000 7fb3e7400700 1 -- 172.21.15.28:0/3311514178 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7fb3e002a070 con 0x7fb3f01035f0 2024-05-20T20:06:37.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.800+0000 7fb400626b80 1 Processor -- start 2024-05-20T20:06:37.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.801+0000 7fb400626b80 1 -- start start 2024-05-20T20:06:37.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.801+0000 7fb3e7400700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3d40b2ba0 0x7fb3d40b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.801+0000 7fb400626b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c18756c0 0x5590c1875aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb400626b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c186fa70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb400626b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c18704f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb400626b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5590c17c5e40 con 0x5590c1875fe0 2024-05-20T20:06:37.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb400626b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5590c17b59e0 con 0x5590c18756c0 2024-05-20T20:06:37.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb400626b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5590c17b9a20 con 0x5590c1870090 2024-05-20T20:06:37.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb3e7400700 1 -- 172.21.15.28:0/3311514178 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb3e00e3640 con 0x7fb3f01035f0 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb3ee000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c186fa70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb3ee000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c186fa70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47566/0 (socket says 172.21.15.28:47566) 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb3ee000700 1 -- 172.21.15.28:0/2618514420 learned_addr learned my addr 172.21.15.28:0/2618514420 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.802+0000 7fb3eea00700 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c18756c0 0x5590c1875aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.803+0000 7fb3ef400700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3d40b2ba0 0x7fb3d40b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.803+0000 7fb3ee000700 1 -- 172.21.15.28:0/2618514420 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 msgr2=0x5590c18704f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.803+0000 7fb3ee000700 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c18704f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.803+0000 7fb3ee000700 1 -- 172.21.15.28:0/2618514420 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c18756c0 msgr2=0x5590c1875aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.803+0000 7fb3ee000700 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c18756c0 0x5590c1875aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.803+0000 7fb3ee000700 1 -- 172.21.15.28:0/2618514420 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5590c177e420 con 0x5590c1875fe0 2024-05-20T20:06:37.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.803+0000 7fb3ee000700 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c186fa70 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fb3dc009d20 tx=0x7fb3dc011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=433e1862b66ebf8b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.804+0000 7fb3ef400700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3d40b2ba0 0x7fb3d40b5060 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fb3e80094e0 tx=0x7fb3e800b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.804+0000 7fb3dbe00700 1 -- 172.21.15.28:0/2618514420 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3dc01e070 con 0x5590c1875fe0 2024-05-20T20:06:37.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.804+0000 7fb3dbe00700 1 -- 172.21.15.28:0/2618514420 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3dc019ae0 con 0x5590c1875fe0 2024-05-20T20:06:37.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.804+0000 7fb3dbe00700 1 -- 172.21.15.28:0/2618514420 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3dc00ea30 con 0x5590c1875fe0 2024-05-20T20:06:37.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.805+0000 7fb400626b80 1 -- 172.21.15.28:0/2618514420 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 msgr2=0x5590c186fa70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.805+0000 7fb400626b80 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c186fa70 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fb3dc009d20 tx=0x7fb3dc011720 comp rx=0 tx=0).stop 2024-05-20T20:06:37.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.806+0000 7fb400626b80 1 -- 172.21.15.28:0/2618514420 shutdown_connections 2024-05-20T20:06:37.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.806+0000 7fb400626b80 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c18704f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.806+0000 7fb400626b80 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c18756c0 0x5590c1875aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.806+0000 7fb400626b80 1 --2- 172.21.15.28:0/2618514420 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c186fa70 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.806+0000 7fb400626b80 1 -- 172.21.15.28:0/2618514420 >> 172.21.15.28:0/2618514420 conn(0x5590c1795be0 msgr2=0x5590c1862010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.806+0000 7fb400626b80 1 -- 172.21.15.28:0/2618514420 shutdown_connections 2024-05-20T20:06:37.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.806+0000 7fb400626b80 1 -- 172.21.15.28:0/2618514420 wait complete. 2024-05-20T20:06:37.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.807+0000 7fb400626b80 1 -- start start 2024-05-20T20:06:37.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.807+0000 7fb400626b80 1 Processor -- start 2024-05-20T20:06:37.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.808+0000 7fb400626b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c187bd30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.808+0000 7fb400626b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1879050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.808+0000 7fb400626b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1875fe0 0x5590c1879590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.808+0000 7fb400626b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5590c16bbfa0 con 0x5590c18756c0 2024-05-20T20:06:37.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.808+0000 7fb400626b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5590c1756760 con 0x5590c1870090 2024-05-20T20:06:37.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.808+0000 7fb400626b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5590c187caa0 con 0x5590c1875fe0 2024-05-20T20:06:37.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.808+0000 7fb400626b80 10 client.0 _ll_register_callbacks cb 0x5590c175fc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:37.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3eea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c187bd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3eea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c187bd30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44832/0 (socket says 172.21.15.28:44832) 2024-05-20T20:06:37.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3eea00700 1 -- 172.21.15.28:0/945724608 learned_addr learned my addr 172.21.15.28:0/945724608 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3ef400700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1875fe0 0x5590c1879590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3ee000700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1879050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3eea00700 1 -- 172.21.15.28:0/945724608 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1875fe0 msgr2=0x5590c1879590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3eea00700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1875fe0 0x5590c1879590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.809+0000 7fb3eea00700 1 -- 172.21.15.28:0/945724608 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 msgr2=0x5590c1879050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.810+0000 7fb3eea00700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1879050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.810+0000 7fb3eea00700 1 -- 172.21.15.28:0/945724608 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5590c17058d0 con 0x5590c1870090 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.810+0000 7fb3ee000700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1879050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.810+0000 7fb3d8c00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.810+0000 7fb3eea00700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c187bd30 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb3e0009750 tx=0x7fb3e00aa020 comp rx=0 tx=0).ready entity=mon.2 client_cookie=83157a4261c76f14 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.811+0000 7fb3d8c00700 1 -- 172.21.15.28:0/945724608 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3e00aa780 con 0x5590c1870090 2024-05-20T20:06:37.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.811+0000 7fb3d8c00700 1 -- 172.21.15.28:0/945724608 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb3e0023740 con 0x5590c1870090 2024-05-20T20:06:37.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.811+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3dc019ae0 con 0x5590c1870090 2024-05-20T20:06:37.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.811+0000 7fb3d8c00700 1 -- 172.21.15.28:0/945724608 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb3e00aeb30 con 0x5590c1870090 2024-05-20T20:06:37.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.811+0000 7fb400626b80 10 client.25445 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:37.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.811+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5590c1786060 con 0x5590c1870090 2024-05-20T20:06:37.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.811+0000 7fb3d8c00700 1 -- 172.21.15.28:0/945724608 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3dc019ae0 con 0x5590c1870090 2024-05-20T20:06:37.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb3d8c00700 1 -- 172.21.15.28:0/945724608 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb3e00aa9e0 con 0x5590c1870090 2024-05-20T20:06:37.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb400626b80 10 client.25445 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:37.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb400626b80 10 client.25445 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:37.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb400626b80 20 client.25445 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:37.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb400626b80 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1889620 0x5590c188baa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb400626b80 4 client.25445 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:37.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5590c17c7d90 con 0x5590c1889620 2024-05-20T20:06:37.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.812+0000 7fb3ee000700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1889620 0x5590c188baa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.813+0000 7fb3ee000700 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1889620 0x5590c188baa0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4aacc30dfdc1df3e server_cookie=38c7d7c1ad2d619f in_seq=0 out_seq=0 2024-05-20T20:06:37.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.813+0000 7fb3d8c00700 10 client.25445 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.814+0000 7fb3d8c00700 1 -- 172.21.15.28:0/945724608 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5590c17c7d90 con 0x5590c1889620 2024-05-20T20:06:37.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.814+0000 7fb3d8c00700 10 client.25445 handle_command_reply: tid=0 2024-05-20T20:06:37.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.814+0000 7fb400626b80 1 client.25445 shutdown 2024-05-20T20:06:37.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 msgr2=0x5590c187bd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c187bd30 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb3e0009750 tx=0x7fb3e00aa020 comp rx=0 tx=0).stop 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 shutdown_connections 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1889620 0x5590c188baa0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1875fe0 0x5590c1879590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c187bd30 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 --2- 172.21.15.28:0/945724608 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1879050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 >> 172.21.15.28:0/945724608 conn(0x5590c1795be0 msgr2=0x5590c186ea60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.815+0000 7fb3d8c00700 0 client.25445 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.816+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 shutdown_connections 2024-05-20T20:06:37.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.816+0000 7fb400626b80 1 -- 172.21.15.28:0/945724608 wait complete. 2024-05-20T20:06:37.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.816+0000 7fb400626b80 20 client.25445 trim_cache size 0 max 16384 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.841+0000 7fb400626b80 1 Processor -- start 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.841+0000 7fb400626b80 1 -- start start 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb400626b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c1873840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb400626b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c18756c0 0x5590c18620f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb400626b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c1891370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb400626b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f01abc20 con 0x5590c1875fe0 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb400626b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f01ad760 con 0x5590c1870090 2024-05-20T20:06:37.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb400626b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f01ac2f0 con 0x5590c18756c0 2024-05-20T20:06:37.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb3ee000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c18756c0 0x5590c18620f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb3eea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c1873840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb3ee000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c18756c0 0x5590c18620f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38540/0 (socket says 172.21.15.28:38540) 2024-05-20T20:06:37.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb3ee000700 1 -- 172.21.15.28:0/3440224930 learned_addr learned my addr 172.21.15.28:0/3440224930 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.842+0000 7fb3eea00700 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c1873840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44848/0 (socket says 172.21.15.28:44848) 2024-05-20T20:06:37.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.843+0000 7fb3ef400700 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c1891370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.843+0000 7fb3eea00700 1 -- 172.21.15.28:0/3440224930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c18756c0 msgr2=0x5590c18620f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.843+0000 7fb3eea00700 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c18756c0 0x5590c18620f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.843+0000 7fb3eea00700 1 -- 172.21.15.28:0/3440224930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 msgr2=0x5590c1891370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.843+0000 7fb3eea00700 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c1891370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.843+0000 7fb3eea00700 1 -- 172.21.15.28:0/3440224930 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5590c185be50 con 0x5590c1870090 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.844+0000 7fb3ef400700 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c1891370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.844+0000 7fb3eea00700 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c1873840 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fb3e0024000 tx=0x7fb3e000b490 comp rx=0 tx=0).ready entity=mon.2 client_cookie=473b6d180002209a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.844+0000 7fb3db400700 1 -- 172.21.15.28:0/3440224930 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3e00b3550 con 0x5590c1870090 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.844+0000 7fb3db400700 1 -- 172.21.15.28:0/3440224930 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3e00aa9e0 con 0x5590c1870090 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.844+0000 7fb3db400700 1 -- 172.21.15.28:0/3440224930 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3e0092330 con 0x5590c1870090 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 -- 172.21.15.28:0/3440224930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 msgr2=0x5590c1873840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c1873840 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fb3e0024000 tx=0x7fb3e000b490 comp rx=0 tx=0).stop 2024-05-20T20:06:37.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 -- 172.21.15.28:0/3440224930 shutdown_connections 2024-05-20T20:06:37.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c1875fe0 0x5590c1891370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1870090 0x5590c1873840 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 --2- 172.21.15.28:0/3440224930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c18756c0 0x5590c18620f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 -- 172.21.15.28:0/3440224930 >> 172.21.15.28:0/3440224930 conn(0x5590c1795be0 msgr2=0x5590c18619e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 -- 172.21.15.28:0/3440224930 shutdown_connections 2024-05-20T20:06:37.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.846+0000 7fb400626b80 1 -- 172.21.15.28:0/3440224930 wait complete. 2024-05-20T20:06:37.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.847+0000 7fb400626b80 1 -- start start 2024-05-20T20:06:37.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.847+0000 7fb400626b80 1 Processor -- start 2024-05-20T20:06:37.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb400626b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c1891040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb400626b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1891580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb400626b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1875fe0 0x5590c1891ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb400626b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f01ad090 con 0x5590c18756c0 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb400626b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f01ac9c0 con 0x5590c1875fe0 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb400626b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb3f010e610 con 0x5590c1870090 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb400626b80 10 client.0 _ll_register_callbacks cb 0x5590c175fc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb3eea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c1891040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb3eea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c1891040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38548/0 (socket says 172.21.15.28:38548) 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.848+0000 7fb3eea00700 1 -- 172.21.15.28:0/739544164 learned_addr learned my addr 172.21.15.28:0/739544164 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3ee000700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1891580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3ef400700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1875fe0 0x5590c1891ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3eea00700 1 -- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1875fe0 msgr2=0x5590c1891ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3eea00700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1875fe0 0x5590c1891ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3eea00700 1 -- 172.21.15.28:0/739544164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 msgr2=0x5590c1891580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3eea00700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1891580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3eea00700 1 -- 172.21.15.28:0/739544164 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5590c1904400 con 0x5590c1870090 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3ee000700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1891580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.849+0000 7fb3ef400700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1875fe0 0x5590c1891ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb3eea00700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c1891040 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fb3e000b520 tx=0x7fb3e00aa020 comp rx=0 tx=0).ready entity=mon.1 client_cookie=778a30b3c4dff2e5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb3d8c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb3d8c00700 1 -- 172.21.15.28:0/739544164 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3e0023950 con 0x5590c1870090 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb3d8c00700 1 -- 172.21.15.28:0/739544164 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb3e00b30c0 con 0x5590c1870090 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3e00aa9e0 con 0x5590c1870090 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb3d8c00700 1 -- 172.21.15.28:0/739544164 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb3e00b32c0 con 0x5590c1870090 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb400626b80 10 client.25420 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:37.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.850+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5590c1904400 con 0x5590c1870090 2024-05-20T20:06:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.851+0000 7fb3d8c00700 1 -- 172.21.15.28:0/739544164 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3e00aa9e0 con 0x5590c1870090 2024-05-20T20:06:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.851+0000 7fb3d8c00700 1 -- 172.21.15.28:0/739544164 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb3e00cb030 con 0x5590c1870090 2024-05-20T20:06:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.852+0000 7fb400626b80 10 client.25420 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.852+0000 7fb400626b80 10 client.25420 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.852+0000 7fb400626b80 20 client.25420 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.852+0000 7fb400626b80 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1905920 0x5590c1907d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.852+0000 7fb400626b80 4 client.25420 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:06:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.852+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x5590c17c1570 con 0x5590c1905920 2024-05-20T20:06:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.853+0000 7fb3ee000700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1905920 0x5590c1907d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.853+0000 7fb3ee000700 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1905920 0x5590c1907d40 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d7cdcee0b25e5de4 server_cookie=6da9447d50fbbd82 in_seq=0 out_seq=0 2024-05-20T20:06:37.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.853+0000 7fb3d8c00700 10 client.25420 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb3d8c00700 1 -- 172.21.15.28:0/739544164 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x5590c17c1570 con 0x5590c1905920 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb3d8c00700 10 client.25420 handle_command_reply: tid=0 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 client.25420 shutdown 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 msgr2=0x5590c1891040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c1891040 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fb3e000b520 tx=0x7fb3e00aa020 comp rx=0 tx=0).stop 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 shutdown_connections 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5590c1905920 0x5590c1907d40 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5590c1875fe0 0x5590c1891ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5590c1870090 0x5590c1891040 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 --2- 172.21.15.28:0/739544164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5590c18756c0 0x5590c1891580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb3d8c00700 0 client.25420 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.854+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 >> 172.21.15.28:0/739544164 conn(0x5590c1795be0 msgr2=0x5590c18763c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.855+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 shutdown_connections 2024-05-20T20:06:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.855+0000 7fb400626b80 1 -- 172.21.15.28:0/739544164 wait complete. 2024-05-20T20:06:37.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.856+0000 7fb400626b80 20 client.25420 trim_cache size 0 max 16384 2024-05-20T20:06:37.856 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:06:37.856 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:06:37.856 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:06:37.856 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.856+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3d40b2ba0 msgr2=0x7fb3d40b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.856+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3d40b2ba0 0x7fb3d40b5060 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fb3e80094e0 tx=0x7fb3e800b040 comp rx=0 tx=0).stop 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.856+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f01035f0 msgr2=0x7fb3f019be10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f01035f0 0x7fb3f019be10 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fb3e0000c00 tx=0x7fb3e000bfd0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 shutdown_connections 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3d40b2ba0 0x7fb3d40b5060 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3f0108500 0x7fb3f01a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3f01035f0 0x7fb3f019be10 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 --2- 172.21.15.28:0/3311514178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3f0103f90 0x7fb3f019c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 >> 172.21.15.28:0/3311514178 conn(0x7fb3f007a6a0 msgr2=0x7fb3f0101500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 shutdown_connections 2024-05-20T20:06:37.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:37.857+0000 7fb3f5c00700 1 -- 172.21.15.28:0/3311514178 wait complete. 2024-05-20T20:06:37.860 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:37.907 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 -- 172.21.15.28:0/2651965774 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 msgr2=0x7f5fd010eed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.907 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 --2- 172.21.15.28:0/2651965774 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 0x7f5fd010eed0 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f5fbc009860 tx=0x7f5fbc02a340 comp rx=0 tx=0).stop 2024-05-20T20:06:37.907 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 -- 172.21.15.28:0/2651965774 shutdown_connections 2024-05-20T20:06:37.907 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 --2- 172.21.15.28:0/2651965774 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5fd010f410 0x7f5fd0111800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.907 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 --2- 172.21.15.28:0/2651965774 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 0x7f5fd010eed0 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 --2- 172.21.15.28:0/2651965774 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd0101860 0x7f5fd0101c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 -- 172.21.15.28:0/2651965774 >> 172.21.15.28:0/2651965774 conn(0x7f5fd0078ad0 msgr2=0x7f5fd0078ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 -- 172.21.15.28:0/2651965774 shutdown_connections 2024-05-20T20:06:37.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.907+0000 7f5fd8200640 1 -- 172.21.15.28:0/2651965774 wait complete. 2024-05-20T20:06:37.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.908+0000 7f5fd8200640 1 Processor -- start 2024-05-20T20:06:37.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd8200640 1 -- start start 2024-05-20T20:06:37.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd8200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5fd0101860 0x7f5fd019b3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd8200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 0x7f5fd019b930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd8200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd010f410 0x7f5fd019fcb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd8200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd0114eb0 con 0x7f5fd0102180 2024-05-20T20:06:37.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd8200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd0114d30 con 0x7f5fd010f410 2024-05-20T20:06:37.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd8200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd0115030 con 0x7f5fd0101860 2024-05-20T20:06:37.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd6200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd010f410 0x7f5fd019fcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5fd0101860 0x7f5fd019b3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd6200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd010f410 0x7f5fd019fcb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44872/0 (socket says 172.21.15.28:44872) 2024-05-20T20:06:37.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.909+0000 7f5fd6200640 1 -- 172.21.15.28:0/1293343569 learned_addr learned my addr 172.21.15.28:0/1293343569 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 0x7f5fd019b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd6200640 1 -- 172.21.15.28:0/1293343569 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5fd0101860 msgr2=0x7f5fd019b3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd6200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5fd0101860 0x7f5fd019b3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd6200640 1 -- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 msgr2=0x7f5fd019b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd6200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 0x7f5fd019b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd6200640 1 -- 172.21.15.28:0/1293343569 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5fd01a03e0 con 0x7f5fd010f410 2024-05-20T20:06:37.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd5800640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5fd0101860 0x7f5fd019b3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:37.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.910+0000 7f5fd6200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd010f410 0x7f5fd019fcb0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f5fc800eb40 tx=0x7f5fc800c2d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.911+0000 7f5fce000640 1 -- 172.21.15.28:0/1293343569 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fc8004b20 con 0x7f5fd010f410 2024-05-20T20:06:37.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.911+0000 7f5fce000640 1 -- 172.21.15.28:0/1293343569 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5fc8004cc0 con 0x7f5fd010f410 2024-05-20T20:06:37.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.911+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5fd01a06d0 con 0x7f5fd010f410 2024-05-20T20:06:37.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.911+0000 7f5fce000640 1 -- 172.21.15.28:0/1293343569 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fc8010720 con 0x7f5fd010f410 2024-05-20T20:06:37.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.911+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5fd01a7f60 con 0x7f5fd010f410 2024-05-20T20:06:37.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.915+0000 7f5fdb193740 1 Processor -- start 2024-05-20T20:06:37.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.915+0000 7f5fce000640 1 -- 172.21.15.28:0/1293343569 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f5fc8005560 con 0x7f5fd010f410 2024-05-20T20:06:37.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.915+0000 7f5fdb193740 1 -- start start 2024-05-20T20:06:37.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887b9c90 0x556e887ba070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c4930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887c7260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fdb193740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556e8872ab90 con 0x556e887c4e70 2024-05-20T20:06:37.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fdb193740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556e8871a730 con 0x556e887b9c90 2024-05-20T20:06:37.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fdb193740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556e8871e770 con 0x556e887ba5b0 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c4930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c4930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38574/0 (socket says 172.21.15.28:38574) 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fd5800640 1 -- 172.21.15.28:0/2819850230 learned_addr learned my addr 172.21.15.28:0/2819850230 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887b9c90 0x556e887ba070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.916+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887c7260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fd5800640 1 -- 172.21.15.28:0/2819850230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887b9c90 msgr2=0x556e887ba070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887b9c90 0x556e887ba070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fd5800640 1 -- 172.21.15.28:0/2819850230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 msgr2=0x556e887c7260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887c7260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fd5800640 1 -- 172.21.15.28:0/2819850230 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556e886b38b0 con 0x556e887ba5b0 2024-05-20T20:06:37.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887c7260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:06:37.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c4930 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f5fc40114f0 tx=0x7f5fc40119b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=428e1a99f4b187bc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fc2a00640 1 -- 172.21.15.28:0/2819850230 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fc4005020 con 0x556e887ba5b0 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.917+0000 7f5fc2a00640 1 -- 172.21.15.28:0/2819850230 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5fc40047a0 con 0x556e887ba5b0 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.918+0000 7f5fc2a00640 1 -- 172.21.15.28:0/2819850230 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fc400f800 con 0x556e887ba5b0 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.918+0000 7f5fce000640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5fb80b2bb0 0x7f5fb80b5070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.918+0000 7f5fce000640 1 -- 172.21.15.28:0/1293343569 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5fc80e2f70 con 0x7f5fd010f410 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.918+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5fb80b2bb0 0x7f5fb80b5070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.918+0000 7f5fdb193740 1 -- 172.21.15.28:0/2819850230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 msgr2=0x556e887c4930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.918+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c4930 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f5fc40114f0 tx=0x7f5fc40119b0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fdb193740 1 -- 172.21.15.28:0/2819850230 shutdown_connections 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887c7260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c4930 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2819850230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887b9c90 0x556e887ba070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fdb193740 1 -- 172.21.15.28:0/2819850230 >> 172.21.15.28:0/2819850230 conn(0x556e887b61a0 msgr2=0x556e887b6580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fdb193740 1 -- 172.21.15.28:0/2819850230 shutdown_connections 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fdb193740 1 -- 172.21.15.28:0/2819850230 wait complete. 2024-05-20T20:06:37.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.919+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5fb80b2bb0 0x7f5fb80b5070 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5fbc009830 tx=0x7f5fbc030340 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 -- start start 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 Processor -- start 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887b9c90 0x556e887c4780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c1a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c1fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556e88400790 con 0x556e887b9c90 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556e886c2bc0 con 0x556e887c4e70 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.920+0000 7f5fdb193740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556e887c7aa0 con 0x556e887ba5b0 2024-05-20T20:06:37.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fdb193740 10 client.0 _ll_register_callbacks cb 0x556e886a0400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887b9c90 0x556e887c4780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887b9c90 0x556e887c4780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47646/0 (socket says 172.21.15.28:47646) 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd5800640 1 -- 172.21.15.28:0/2821949392 learned_addr learned my addr 172.21.15.28:0/2821949392 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd6200640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c1fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c1a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd6200640 1 -- 172.21.15.28:0/2821949392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 msgr2=0x556e887c1a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd6200640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c1a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd6200640 1 -- 172.21.15.28:0/2821949392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887b9c90 msgr2=0x556e887c4780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd6200640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887b9c90 0x556e887c4780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.921+0000 7f5fd6200640 1 -- 172.21.15.28:0/2821949392 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556e8842f210 con 0x556e887c4e70 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887b9c90 0x556e887c4780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:37.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fd6200640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c1fb0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f5fc80e0490 tx=0x7f5fc80aa640 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5d0ec4f1f50e2a01 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fb7400640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fb7400640 1 -- 172.21.15.28:0/2821949392 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fc80c3020 con 0x556e887c4e70 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fb7400640 1 -- 172.21.15.28:0/2821949392 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5fc80a9490 con 0x556e887c4e70 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5fc40047a0 con 0x556e887c4e70 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fb7400640 1 -- 172.21.15.28:0/2821949392 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5fc80aa020 con 0x556e887c4e70 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fdb193740 10 client.25460 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.922+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556e884fb620 con 0x556e887c4e70 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fb7400640 1 -- 172.21.15.28:0/2821949392 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5fc40047a0 con 0x556e887c4e70 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fb7400640 1 -- 172.21.15.28:0/2821949392 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5fc8005000 con 0x556e887c4e70 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fdb193740 10 client.25460 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fdb193740 10 client.25460 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fdb193740 20 client.25460 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:37.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e887d47c0 0x556e887d6bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fdb193740 4 client.25460 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:37.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.923+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556e8872cae0 con 0x556e887d47c0 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.924+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e887d47c0 0x556e887d6bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.924+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e887d47c0 0x556e887d6bb0 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f1e4dab2081417b6 server_cookie=f8dd5fee381897da in_seq=0 out_seq=0 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.924+0000 7f5fb7400640 10 client.25460 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fb7400640 1 -- 172.21.15.28:0/2821949392 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x556e8872cae0 con 0x556e887d47c0 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fb7400640 10 client.25460 handle_command_reply: tid=0 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 client.25460 shutdown 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 msgr2=0x556e887c1fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c1fb0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f5fc80e0490 tx=0x7f5fc80aa640 comp rx=0 tx=0).stop 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 shutdown_connections 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e887d47c0 0x556e887d6bb0 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c1fb0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887ba5b0 0x556e887c1a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2821949392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887b9c90 0x556e887c4780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fb7400640 0 client.25460 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 >> 172.21.15.28:0/2821949392 conn(0x556e887b61a0 msgr2=0x556e887bbed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.925+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 shutdown_connections 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.926+0000 7f5fdb193740 1 -- 172.21.15.28:0/2821949392 wait complete. 2024-05-20T20:06:37.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.926+0000 7f5fdb193740 20 client.25460 trim_cache size 0 max 16384 2024-05-20T20:06:37.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.945+0000 7f5fdb193740 1 Processor -- start 2024-05-20T20:06:37.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.945+0000 7f5fdb193740 1 -- start start 2024-05-20T20:06:37.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c42b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887ba5b0 0x556e887dc4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887dca00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fdb193740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd01b0860 con 0x556e887c4e70 2024-05-20T20:06:37.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fdb193740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd01b23a0 con 0x556e887ba5b0 2024-05-20T20:06:37.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fdb193740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd01b0f30 con 0x556e887b9c90 2024-05-20T20:06:37.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fd4e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887ba5b0 0x556e887dc4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c42b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c42b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38590/0 (socket says 172.21.15.28:38590) 2024-05-20T20:06:37.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fd4e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887ba5b0 0x556e887dc4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44896/0 (socket says 172.21.15.28:44896) 2024-05-20T20:06:37.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fd5800640 1 -- 172.21.15.28:0/2729453577 learned_addr learned my addr 172.21.15.28:0/2729453577 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.946+0000 7f5fd6200640 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887dca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fd5800640 1 -- 172.21.15.28:0/2729453577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887ba5b0 msgr2=0x556e887dc4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887ba5b0 0x556e887dc4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fd5800640 1 -- 172.21.15.28:0/2729453577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 msgr2=0x556e887dca00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887dca00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fd5800640 1 -- 172.21.15.28:0/2729453577 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556e887b46b0 con 0x556e887b9c90 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fd5800640 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c42b0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f5fc40057c0 tx=0x7f5fc4005a00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fc1d8ecf80a34b39 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fc2000640 1 -- 172.21.15.28:0/2729453577 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fc4009cb0 con 0x556e887b9c90 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fc2000640 1 -- 172.21.15.28:0/2729453577 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5fc4005b30 con 0x556e887b9c90 2024-05-20T20:06:37.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.947+0000 7f5fc2000640 1 -- 172.21.15.28:0/2729453577 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fc4004460 con 0x556e887b9c90 2024-05-20T20:06:37.950 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 -- 172.21.15.28:0/2729453577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 msgr2=0x556e887c42b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.950 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c42b0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f5fc40057c0 tx=0x7f5fc4005a00 comp rx=0 tx=0).stop 2024-05-20T20:06:37.950 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 -- 172.21.15.28:0/2729453577 shutdown_connections 2024-05-20T20:06:37.950 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887ba5b0 0x556e887dc4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.950 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887c4e70 0x556e887dca00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 --2- 172.21.15.28:0/2729453577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c42b0 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 -- 172.21.15.28:0/2729453577 >> 172.21.15.28:0/2729453577 conn(0x556e887b61a0 msgr2=0x556e887b6580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 -- 172.21.15.28:0/2729453577 shutdown_connections 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.948+0000 7f5fdb193740 1 -- 172.21.15.28:0/2729453577 wait complete. 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 -- start start 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 Processor -- start 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c7cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887ba5b0 0x556e887c81f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c8730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd01b1cd0 con 0x556e887ba5b0 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd01b1600 con 0x556e887c4e70 2024-05-20T20:06:37.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5fd01ae840 con 0x556e887b9c90 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.949+0000 7f5fdb193740 10 client.0 _ll_register_callbacks cb 0x7f5fc80a9970 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887ba5b0 0x556e887c81f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c7cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887ba5b0 0x556e887c81f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47666/0 (socket says 172.21.15.28:47666) 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 -- 172.21.15.28:0/426608296 learned_addr learned my addr 172.21.15.28:0/426608296 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd6200640 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c8730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 -- 172.21.15.28:0/426608296 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 msgr2=0x556e887c7cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c7cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 -- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 msgr2=0x556e887c8730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c8730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd4e00640 1 -- 172.21.15.28:0/426608296 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556e8883d550 con 0x556e887ba5b0 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.950+0000 7f5fd5800640 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c7cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:37.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fd4e00640 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887ba5b0 0x556e887c81f0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f5fbc009780 tx=0x7f5fbc000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2f18ef1c5f916759 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:37.953 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fb7400640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:37.954 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fb7400640 1 -- 172.21.15.28:0/426608296 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5fbc037040 con 0x556e887ba5b0 2024-05-20T20:06:37.954 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fb7400640 1 -- 172.21.15.28:0/426608296 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5fbc0035f0 con 0x556e887ba5b0 2024-05-20T20:06:37.954 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5fc4005b30 con 0x556e887ba5b0 2024-05-20T20:06:37.954 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fb7400640 1 -- 172.21.15.28:0/426608296 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5fbc0308e0 con 0x556e887ba5b0 2024-05-20T20:06:37.954 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fdb193740 10 client.15324 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:37.954 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.951+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556e8883e870 con 0x556e887ba5b0 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fb7400640 1 -- 172.21.15.28:0/426608296 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5fc4005b30 con 0x556e887ba5b0 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fb7400640 1 -- 172.21.15.28:0/426608296 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5fbc032880 con 0x556e887ba5b0 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fdb193740 10 client.15324 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fdb193740 10 client.15324 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fdb193740 20 client.15324 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fdb193740 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e8883f020 0x556e88841440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fdb193740 4 client.15324 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x556e887262c0 con 0x556e8883f020 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.952+0000 7f5fd5800640 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e8883f020 0x556e88841440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.953+0000 7f5fd5800640 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e8883f020 0x556e88841440 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7eeaeff268b43e07 server_cookie=3956a5a7e9d52fd in_seq=0 out_seq=0 2024-05-20T20:06:37.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.953+0000 7f5fb7400640 10 client.15324 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fb7400640 1 -- 172.21.15.28:0/426608296 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x556e887262c0 con 0x556e8883f020 2024-05-20T20:06:37.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fb7400640 10 client.15324 handle_command_reply: tid=0 2024-05-20T20:06:37.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 client.15324 shutdown 2024-05-20T20:06:37.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887ba5b0 msgr2=0x556e887c81f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887ba5b0 0x556e887c81f0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f5fbc009780 tx=0x7f5fbc000bc0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 shutdown_connections 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556e8883f020 0x556e88841440 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556e887c4e70 0x556e887c8730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556e887ba5b0 0x556e887c81f0 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 --2- 172.21.15.28:0/426608296 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556e887b9c90 0x556e887c7cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fb7400640 0 client.15324 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 >> 172.21.15.28:0/426608296 conn(0x556e887b61a0 msgr2=0x556e887bb640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 shutdown_connections 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.962+0000 7f5fdb193740 1 -- 172.21.15.28:0/426608296 wait complete. 2024-05-20T20:06:37.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.963+0000 7f5fdb193740 20 client.15324 trim_cache size 0 max 16384 2024-05-20T20:06:37.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:06:37.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:06:37.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:06:37.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.963+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5fb80b2bb0 msgr2=0x7f5fb80b5070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.963+0000 7f5fd8200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5fb80b2bb0 0x7f5fb80b5070 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5fbc009830 tx=0x7f5fbc030340 comp rx=0 tx=0).stop 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.963+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd010f410 msgr2=0x7f5fd019fcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.963+0000 7f5fd8200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd010f410 0x7f5fd019fcb0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f5fc800eb40 tx=0x7f5fc800c2d0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.963+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 shutdown_connections 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.963+0000 7f5fd8200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5fd010f410 0x7f5fd019fcb0 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.964+0000 7f5fd8200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5fb80b2bb0 0x7f5fb80b5070 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.964+0000 7f5fd8200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5fd0102180 0x7f5fd019b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.964+0000 7f5fd8200640 1 --2- 172.21.15.28:0/1293343569 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5fd0101860 0x7f5fd019b3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.964+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 >> 172.21.15.28:0/1293343569 conn(0x7f5fd0078ad0 msgr2=0x7f5fd0110960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.964+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 shutdown_connections 2024-05-20T20:06:37.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:37.964+0000 7f5fd8200640 1 -- 172.21.15.28:0/1293343569 wait complete. 2024-05-20T20:06:39.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:38 smithi028 ceph-mon[31178]: pgmap v33: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 2.0 KiB/s wr, 10 op/s 2024-05-20T20:06:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:38 smithi154 ceph-mon[36314]: pgmap v33: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 2.0 KiB/s wr, 10 op/s 2024-05-20T20:06:39.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:38 smithi033 ceph-mon[36179]: pgmap v33: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 2.0 KiB/s wr, 10 op/s 2024-05-20T20:06:39.752 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:06:39.752 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:06:40.238 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:40.893 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:40 smithi028 ceph-mon[31178]: pgmap v34: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 3.2 KiB/s wr, 16 op/s 2024-05-20T20:06:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 -- 172.21.15.28:0/306987131 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07181035f0 msgr2=0x7f0718103a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 --2- 172.21.15.28:0/306987131 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07181035f0 0x7f0718103a50 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f0704009990 tx=0x7f0704009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 -- 172.21.15.28:0/306987131 shutdown_connections 2024-05-20T20:06:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 --2- 172.21.15.28:0/306987131 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0718103f90 0x7f0718110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 --2- 172.21.15.28:0/306987131 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0718108500 0x7f07181088e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 --2- 172.21.15.28:0/306987131 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07181035f0 0x7f0718103a50 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 -- 172.21.15.28:0/306987131 >> 172.21.15.28:0/306987131 conn(0x7f071807a6a0 msgr2=0x7f071807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.951+0000 7f071f800700 1 -- 172.21.15.28:0/306987131 shutdown_connections 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.952+0000 7f071f800700 1 -- 172.21.15.28:0/306987131 wait complete. 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 Processor -- start 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 -- start start 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07181035f0 0x7f071819fa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0718103f90 0x7f071819ffa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0718108500 0x7f0718199b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0718113790 con 0x7f0718108500 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0718113610 con 0x7f07181035f0 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.953+0000 7f071f800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0718113910 con 0x7f0718103f90 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0718108500 0x7f0718199b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07181035f0 0x7f071819fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f0717e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0718103f90 0x7f071819ffa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0718108500 0x7f0718199b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47686/0 (socket says 172.21.15.28:47686) 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07181035f0 0x7f071819fa60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44924/0 (socket says 172.21.15.28:44924) 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 -- 172.21.15.28:0/526598154 learned_addr learned my addr 172.21.15.28:0/526598154 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 -- 172.21.15.28:0/526598154 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0718103f90 msgr2=0x7f071819ffa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0718103f90 0x7f071819ffa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 -- 172.21.15.28:0/526598154 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07181035f0 msgr2=0x7f071819fa60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07181035f0 0x7f071819fa60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.954+0000 7f071d800700 1 -- 172.21.15.28:0/526598154 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f071819a410 con 0x7f0718108500 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.955+0000 7f071d800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0718108500 0x7f0718199b30 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f071000c0f0 tx=0x7f071000c400 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.955+0000 7f0715600700 1 -- 172.21.15.28:0/526598154 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0710018020 con 0x7f0718108500 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.955+0000 7f0715600700 1 -- 172.21.15.28:0/526598154 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f071000ca00 con 0x7f0718108500 2024-05-20T20:06:40.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.955+0000 7f0715600700 1 -- 172.21.15.28:0/526598154 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0710004670 con 0x7f0718108500 2024-05-20T20:06:40.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.955+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f071819a700 con 0x7f0718108500 2024-05-20T20:06:40.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.955+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f07181a69e0 con 0x7f0718108500 2024-05-20T20:06:40.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.959+0000 7f0715600700 1 -- 172.21.15.28:0/526598154 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f0710012050 con 0x7f0718108500 2024-05-20T20:06:40.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.960+0000 7f072a121b80 1 Processor -- start 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.960+0000 7f072a121b80 1 -- start start 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.960+0000 7f072a121b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a19a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.960+0000 7f072a121b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a81959bad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.960+0000 7f072a121b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959c570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f072a121b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a8194f9df0 con 0x55a8195a1ee0 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f072a121b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a8194e9990 con 0x55a8195a15c0 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f072a121b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a8194ed9d0 con 0x55a81959c0f0 2024-05-20T20:06:40.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f0715600700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07000b2c20 0x7f07000b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f0717e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a19a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f071ce00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a81959bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f0717e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a19a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44932/0 (socket says 172.21.15.28:44932) 2024-05-20T20:06:40.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f0717e00700 1 -- 172.21.15.28:0/4167846847 learned_addr learned my addr 172.21.15.28:0/4167846847 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:40.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f0715600700 1 -- 172.21.15.28:0/526598154 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f07100e23b0 con 0x7f0718108500 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f0717e00700 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959c570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.961+0000 7f071d800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07000b2c20 0x7f07000b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f071ce00700 1 -- 172.21.15.28:0/4167846847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 msgr2=0x55a81959c570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f071ce00700 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959c570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f071ce00700 1 -- 172.21.15.28:0/4167846847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 msgr2=0x55a8195a19a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f071ce00700 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a19a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f071ce00700 1 -- 172.21.15.28:0/4167846847 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a8194b23d0 con 0x55a8195a1ee0 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f0717e00700 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a19a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f071ce00700 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a81959bad0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f070c009c60 tx=0x7f070c009f70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ac4104cf10440eb7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f071d800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07000b2c20 0x7f07000b50e0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f07100df930 tx=0x7f0710005ae0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f070a000700 1 -- 172.21.15.28:0/4167846847 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f070c011cb0 con 0x55a8195a1ee0 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f070a000700 1 -- 172.21.15.28:0/4167846847 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f070c011e50 con 0x55a8195a1ee0 2024-05-20T20:06:40.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.962+0000 7f070a000700 1 -- 172.21.15.28:0/4167846847 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f070c019420 con 0x55a8195a1ee0 2024-05-20T20:06:40.965 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 -- 172.21.15.28:0/4167846847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 msgr2=0x55a81959bad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a81959bad0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f070c009c60 tx=0x7f070c009f70 comp rx=0 tx=0).stop 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 -- 172.21.15.28:0/4167846847 shutdown_connections 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959c570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a19a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 --2- 172.21.15.28:0/4167846847 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a81959bad0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 -- 172.21.15.28:0/4167846847 >> 172.21.15.28:0/4167846847 conn(0x55a8194c9b90 msgr2=0x55a819596640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.963+0000 7f072a121b80 1 -- 172.21.15.28:0/4167846847 shutdown_connections 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.964+0000 7f072a121b80 1 -- 172.21.15.28:0/4167846847 wait complete. 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.964+0000 7f072a121b80 1 -- start start 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.964+0000 7f072a121b80 1 Processor -- start 2024-05-20T20:06:40.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f072a121b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195a7b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f072a121b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195a4ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f072a121b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195a5400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f072a121b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a8193eff50 con 0x55a8195a15c0 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f072a121b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a81948a710 con 0x55a81959c0f0 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f072a121b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a8195a8990 con 0x55a8195a1ee0 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f0717e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195a7b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f072a121b80 10 client.0 _ll_register_callbacks cb 0x55a819493c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f0717e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195a7b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44940/0 (socket says 172.21.15.28:44940) 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f0717e00700 1 -- 172.21.15.28:0/2381417914 learned_addr learned my addr 172.21.15.28:0/2381417914 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f071ce00700 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195a4ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.965+0000 7f0717e00700 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195a5400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f0717e00700 1 -- 172.21.15.28:0/2381417914 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 msgr2=0x55a8195a5400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f0717e00700 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195a5400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f0717e00700 1 -- 172.21.15.28:0/2381417914 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 msgr2=0x55a8195a4ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:40.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f0717e00700 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195a4ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f0717e00700 1 -- 172.21.15.28:0/2381417914 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a819439880 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f0717e00700 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195a7b20 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f0704000c00 tx=0x7f0704009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6e4423813ddeffa3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f06fea00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f06fea00700 1 -- 172.21.15.28:0/2381417914 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0704005230 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f070c011e50 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.966+0000 7f06fea00700 1 -- 172.21.15.28:0/2381417914 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f07040053d0 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f06fea00700 1 -- 172.21.15.28:0/2381417914 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0704023930 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f072a121b80 10 client.25478 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a8194ba010 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f06fea00700 1 -- 172.21.15.28:0/2381417914 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f070c011e50 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f06fea00700 1 -- 172.21.15.28:0/2381417914 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0704032050 con 0x55a81959c0f0 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f072a121b80 10 client.25478 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f072a121b80 10 client.25478 resolve_mds: resolved name 'e' to daemon mds.e 2024-05-20T20:06:40.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.967+0000 7f072a121b80 20 client.25478 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:40.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.968+0000 7f072a121b80 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a8195b5370 0x55a8195b67e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.968+0000 7f072a121b80 4 client.25478 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:40.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.968+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a8194fbd40 con 0x55a8195b5370 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.968+0000 7f071ce00700 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a8195b5370 0x55a8195b67e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.969+0000 7f071ce00700 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a8195b5370 0x55a8195b67e0 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=e0ceeb055e17372d server_cookie=ec8bdc87f24c646 in_seq=0 out_seq=0 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.969+0000 7f06fea00700 10 client.25478 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.969+0000 7f06fea00700 1 -- 172.21.15.28:0/2381417914 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a8194fbd40 con 0x55a8195b5370 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.969+0000 7f06fea00700 10 client.25478 handle_command_reply: tid=0 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.969+0000 7f072a121b80 1 client.25478 shutdown 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 msgr2=0x55a8195a7b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195a7b20 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f0704000c00 tx=0x7f0704009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 shutdown_connections 2024-05-20T20:06:40.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a8195b5370 0x55a8195b67e0 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195a5400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195a7b20 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 --2- 172.21.15.28:0/2381417914 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195a4ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f06fea00700 0 client.25478 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 >> 172.21.15.28:0/2381417914 conn(0x55a8194c9b90 msgr2=0x55a819595d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 shutdown_connections 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 1 -- 172.21.15.28:0/2381417914 wait complete. 2024-05-20T20:06:40.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.970+0000 7f072a121b80 20 client.25478 trim_cache size 0 max 16384 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.994+0000 7f072a121b80 1 Processor -- start 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.994+0000 7f072a121b80 1 -- start start 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f072a121b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959bad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f072a121b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a53b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f072a121b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a819592990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f072a121b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a8195b4b20 con 0x55a8195a1ee0 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f072a121b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a8195b51f0 con 0x55a8195a15c0 2024-05-20T20:06:40.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f072a121b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a8195b58c0 con 0x55a81959c0f0 2024-05-20T20:06:40.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f0717e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f0717e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a819592990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f0717e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959bad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38656/0 (socket says 172.21.15.28:38656) 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f0717e00700 1 -- 172.21.15.28:0/655393065 learned_addr learned my addr 172.21.15.28:0/655393065 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.995+0000 7f071ce00700 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a53b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0717e00700 1 -- 172.21.15.28:0/655393065 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 msgr2=0x55a8195a53b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0717e00700 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a53b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0717e00700 1 -- 172.21.15.28:0/655393065 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 msgr2=0x55a819592990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0717e00700 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a819592990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0717e00700 1 -- 172.21.15.28:0/655393065 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a8195a3450 con 0x55a81959c0f0 2024-05-20T20:06:40.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0717e00700 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959bad0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f0704006010 tx=0x7f0704009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=10a1b6d60f4eae92 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:40.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0709600700 1 -- 172.21.15.28:0/655393065 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f07040095a0 con 0x55a81959c0f0 2024-05-20T20:06:40.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0709600700 1 -- 172.21.15.28:0/655393065 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0704005110 con 0x55a81959c0f0 2024-05-20T20:06:40.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.996+0000 7f0709600700 1 -- 172.21.15.28:0/655393065 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f070400f670 con 0x55a81959c0f0 2024-05-20T20:06:41.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 -- 172.21.15.28:0/655393065 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 msgr2=0x55a81959bad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959bad0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f0704006010 tx=0x7f0704009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 -- 172.21.15.28:0/655393065 shutdown_connections 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a1ee0 0x55a819592990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a81959c0f0 0x55a81959bad0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 --2- 172.21.15.28:0/655393065 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8195a15c0 0x55a8195a53b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 -- 172.21.15.28:0/655393065 >> 172.21.15.28:0/655393065 conn(0x55a8194c9b90 msgr2=0x55a8195a4ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 -- 172.21.15.28:0/655393065 shutdown_connections 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.997+0000 7f072a121b80 1 -- 172.21.15.28:0/655393065 wait complete. 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.998+0000 7f072a121b80 1 -- start start 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.998+0000 7f072a121b80 1 Processor -- start 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f072a121b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195bfc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:41.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f072a121b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195c01a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f072a121b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195bac10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f072a121b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a8195b6660 con 0x55a8195a15c0 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f072a121b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a8195b5f90 con 0x55a81959c0f0 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f072a121b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a8195b6d30 con 0x55a8195a1ee0 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f072a121b80 10 client.0 _ll_register_callbacks cb 0x55a819493c20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f071ce00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195c01a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f0717e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195bfc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f071ce00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195c01a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47734/0 (socket says 172.21.15.28:47734) 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f071ce00700 1 -- 172.21.15.28:0/999097118 learned_addr learned my addr 172.21.15.28:0/999097118 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f0717e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195bfc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44958/0 (socket says 172.21.15.28:44958) 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f0717e00700 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195bac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:40.999+0000 7f071ce00700 1 -- 172.21.15.28:0/999097118 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 msgr2=0x55a8195bac10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f071ce00700 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195bac10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f071ce00700 1 -- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 msgr2=0x55a8195bfc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:41.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f071ce00700 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195bfc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f071ce00700 1 -- 172.21.15.28:0/999097118 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a8195bb250 con 0x55a8195a15c0 2024-05-20T20:06:41.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f0717e00700 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195bfc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:41.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f071ce00700 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195c01a0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f070c011720 tx=0x7f070c003680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a207afc2c52c091d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:41.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f06fea00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f06fea00700 1 -- 172.21.15.28:0/999097118 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f070c031060 con 0x55a8195a15c0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f06fea00700 1 -- 172.21.15.28:0/999097118 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f070c003de0 con 0x55a8195a15c0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.000+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0704005110 con 0x55a8195a15c0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f06fea00700 1 -- 172.21.15.28:0/999097118 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f070c011ab0 con 0x55a8195a15c0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 10 client.15342 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a819621400 con 0x55a8195a15c0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f06fea00700 1 -- 172.21.15.28:0/999097118 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0704005110 con 0x55a8195a15c0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f06fea00700 1 -- 172.21.15.28:0/999097118 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f070c02b450 con 0x55a8195a15c0 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 10 client.15342 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:41.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 10 client.15342 resolve_mds: resolved name 'e' to daemon mds.e 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 20 client.15342 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a819625d70 0x55a819624a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 4 client.15342 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.001+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "status"}) -- 0x55a8194f5520 con 0x55a819625d70 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.002+0000 7f0717e00700 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a819625d70 0x55a819624a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.002+0000 7f0717e00700 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a819625d70 0x55a819624a80 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=72cbfb9d90ba1206 server_cookie=bd86b0a8355819ce in_seq=0 out_seq=0 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.002+0000 7f06fea00700 10 client.15342 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.002+0000 7f06fea00700 1 -- 172.21.15.28:0/999097118 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55a8194f5520 con 0x55a819625d70 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.002+0000 7f06fea00700 10 client.15342 handle_command_reply: tid=0 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.002+0000 7f072a121b80 1 client.15342 shutdown 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 msgr2=0x55a8195c01a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195c01a0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f070c011720 tx=0x7f070c003680 comp rx=0 tx=0).stop 2024-05-20T20:06:41.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 shutdown_connections 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55a819625d70 0x55a819624a80 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8195a1ee0 0x55a8195bac10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a81959c0f0 0x55a8195bfc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 --2- 172.21.15.28:0/999097118 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8195a15c0 0x55a8195c01a0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 >> 172.21.15.28:0/999097118 conn(0x55a8194c9b90 msgr2=0x55a8195a22c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f06fea00700 0 client.15342 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 shutdown_connections 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 1 -- 172.21.15.28:0/999097118 wait complete. 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.003+0000 7f072a121b80 20 client.15342 trim_cache size 0 max 16384 2024-05-20T20:06:41.007 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 3, 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24482, 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 296.37091507999997, 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 29, 2024-05-20T20:06:41.008 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 81, 2024-05-20T20:06:41.009 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 80, 2024-05-20T20:06:41.009 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 362.41712445899998 2024-05-20T20:06:41.009 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:06:41.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07000b2c20 msgr2=0x7f07000b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:41.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07000b2c20 0x7f07000b50e0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f07100df930 tx=0x7f0710005ae0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0718108500 msgr2=0x7f0718199b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0718108500 0x7f0718199b30 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f071000c0f0 tx=0x7f071000c400 comp rx=0 tx=0).stop 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 shutdown_connections 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0718108500 0x7f0718199b30 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07000b2c20 0x7f07000b50e0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07181035f0 0x7f071819fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 --2- 172.21.15.28:0/526598154 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0718103f90 0x7f071819ffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 >> 172.21.15.28:0/526598154 conn(0x7f071807a6a0 msgr2=0x7f07181017a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 shutdown_connections 2024-05-20T20:06:41.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:41.004+0000 7f071f800700 1 -- 172.21.15.28:0/526598154 wait complete. 2024-05-20T20:06:41.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:40 smithi154 ceph-mon[36314]: pgmap v34: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 3.2 KiB/s wr, 16 op/s 2024-05-20T20:06:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:40 smithi033 ceph-mon[36179]: pgmap v34: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 3.2 KiB/s wr, 16 op/s 2024-05-20T20:06:42.725 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.i status 2024-05-20T20:06:43.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:42 smithi154 ceph-mon[36314]: pgmap v35: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:42 smithi033 ceph-mon[36179]: pgmap v35: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:43.203 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:43.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:42 smithi028 ceph-mon[31178]: pgmap v35: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 -- 172.21.15.28:0/3348477521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 msgr2=0x7f62b8103a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 --2- 172.21.15.28:0/3348477521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b8103a10 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f62a4009990 tx=0x7f62a4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 -- 172.21.15.28:0/3348477521 shutdown_connections 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 --2- 172.21.15.28:0/3348477521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62b8103f50 0x7f62b8110430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 --2- 172.21.15.28:0/3348477521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f62b81096a0 0x7f62b8109a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 --2- 172.21.15.28:0/3348477521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b8103a10 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 -- 172.21.15.28:0/3348477521 >> 172.21.15.28:0/3348477521 conn(0x7f62b807a6f0 msgr2=0x7f62b807ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:44.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 -- 172.21.15.28:0/3348477521 shutdown_connections 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.939+0000 7f62bd600700 1 -- 172.21.15.28:0/3348477521 wait complete. 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 Processor -- start 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 -- start start 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b819bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62b8103f50 0x7f62b819c410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f62b81096a0 0x7f62b81a07a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f62b8113780 con 0x7f62b8103590 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f62b8113600 con 0x7f62b81096a0 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62bd600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f62b8113900 con 0x7f62b8103f50 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.941+0000 7f62b6a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b819bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62b8103f50 0x7f62b819c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b819bed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47746/0 (socket says 172.21.15.28:47746) 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6a00700 1 -- 172.21.15.28:0/3831006264 learned_addr learned my addr 172.21.15.28:0/3831006264 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b7400700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f62b81096a0 0x7f62b81a07a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:44.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6a00700 1 -- 172.21.15.28:0/3831006264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62b8103f50 msgr2=0x7f62b819c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6a00700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62b8103f50 0x7f62b819c410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6a00700 1 -- 172.21.15.28:0/3831006264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f62b81096a0 msgr2=0x7f62b81a07a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6a00700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f62b81096a0 0x7f62b81a07a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6a00700 1 -- 172.21.15.28:0/3831006264 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f62b81a0ee0 con 0x7f62b8103590 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b6000700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62b8103f50 0x7f62b819c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.942+0000 7f62b7400700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f62b81096a0 0x7f62b81a07a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.943+0000 7f62b6a00700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b819bed0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f62a800eb30 tx=0x7f62a800ee40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.943+0000 7f62af400700 1 -- 172.21.15.28:0/3831006264 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f62a800cbd0 con 0x7f62b8103590 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.943+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f62b81a11d0 con 0x7f62b8103590 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.943+0000 7f62af400700 1 -- 172.21.15.28:0/3831006264 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f62a80044d0 con 0x7f62b8103590 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.943+0000 7f62af400700 1 -- 172.21.15.28:0/3831006264 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f62a8003e30 con 0x7f62b8103590 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.943+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f62b81a8a50 con 0x7f62b8103590 2024-05-20T20:06:44.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.944+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f62b806a230 con 0x7f62b8103590 2024-05-20T20:06:44.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.947+0000 7f62af400700 1 -- 172.21.15.28:0/3831006264 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f62a8004000 con 0x7f62b8103590 2024-05-20T20:06:44.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.948+0000 7f62af400700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f629c0b2c20 0x7f629c0b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:44.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.949+0000 7f62af400700 1 -- 172.21.15.28:0/3831006264 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f62a80e2ce0 con 0x7f62b8103590 2024-05-20T20:06:44.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.949+0000 7f62b6000700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f629c0b2c20 0x7f629c0b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:44.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.950+0000 7f62b6000700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f629c0b2c20 0x7f629c0b50e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f62b81108a0 tx=0x7f62a400b260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:44.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:43.952+0000 7f62af400700 1 -- 172.21.15.28:0/3831006264 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f62a8053df0 con 0x7f62b8103590 2024-05-20T20:06:44.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.246+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f62b819d230 con 0x7f62b8103590 2024-05-20T20:06:44.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.247+0000 7f62af400700 1 -- 172.21.15.28:0/3831006264 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f62a80042f0 con 0x7f62b8103590 2024-05-20T20:06:44.249 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:06:44.249 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:06:44.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f629c0b2c20 msgr2=0x7f629c0b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:44.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f629c0b2c20 0x7f629c0b50e0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f62b81108a0 tx=0x7f62a400b260 comp rx=0 tx=0).stop 2024-05-20T20:06:44.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 msgr2=0x7f62b819bed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:44.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b819bed0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f62a800eb30 tx=0x7f62a800ee40 comp rx=0 tx=0).stop 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 shutdown_connections 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f629c0b2c20 0x7f629c0b50e0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f62b81096a0 0x7f62b81a07a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f62b8103590 0x7f62b819bed0 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 --2- 172.21.15.28:0/3831006264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62b8103f50 0x7f62b819c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 >> 172.21.15.28:0/3831006264 conn(0x7f62b807a6f0 msgr2=0x7f62b8101e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 shutdown_connections 2024-05-20T20:06:44.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:44.251+0000 7f62bd600700 1 -- 172.21.15.28:0/3831006264 wait complete. 2024-05-20T20:06:44.254 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:06:45.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:44 smithi154 ceph-mon[36314]: pgmap v36: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:06:45.291 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:44 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3831006264' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:45.292 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:44 smithi033 ceph-mon[36179]: pgmap v36: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:06:45.292 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:44 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3831006264' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:45.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:44 smithi028 ceph-mon[31178]: pgmap v36: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:06:45.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:44 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3831006264' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:06:45.969 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:06:46.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3987200231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41037f0 msgr2=0x7f9fb4103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3987200231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41037f0 0x7f9fb4103bd0 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40099c0 tx=0x7f9fa4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3987200231 shutdown_connections 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3987200231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb41088f0 0x7f9fb4110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3987200231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41037f0 0x7f9fb4103bd0 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3987200231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41041a0 0x7f9fb41081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3987200231 >> 172.21.15.28:0/3987200231 conn(0x7f9fb407a6a0 msgr2=0x7f9fb407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3987200231 shutdown_connections 2024-05-20T20:06:46.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.542+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3987200231 wait complete. 2024-05-20T20:06:46.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 Processor -- start 2024-05-20T20:06:46.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 -- start start 2024-05-20T20:06:46.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41037f0 0x7f9fb4075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41041a0 0x7f9fb4075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb41088f0 0x7f9fb40762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb4113730 con 0x7f9fb41041a0 2024-05-20T20:06:46.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb41135b0 con 0x7f9fb41088f0 2024-05-20T20:06:46.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.544+0000 7f9fbaa00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9fb41138b0 con 0x7f9fb41037f0 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41041a0 0x7f9fb4075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41037f0 0x7f9fb4075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41037f0 0x7f9fb4075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38702/0 (socket says 172.21.15.28:38702) 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 -- 172.21.15.28:0/3532208354 learned_addr learned my addr 172.21.15.28:0/3532208354 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb8a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb41088f0 0x7f9fb40762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 -- 172.21.15.28:0/3532208354 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb41088f0 msgr2=0x7f9fb40762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb41088f0 0x7f9fb40762c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 -- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41041a0 msgr2=0x7f9fb4075d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41041a0 0x7f9fb4075d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.545+0000 7f9fb3e00700 1 -- 172.21.15.28:0/3532208354 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9fb41a8100 con 0x7f9fb41037f0 2024-05-20T20:06:46.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.546+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41037f0 0x7f9fb4075840 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f9fa4006010 tx=0x7f9fa4005280 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:46.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.546+0000 7f9fb0c00700 1 -- 172.21.15.28:0/3532208354 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa4026070 con 0x7f9fb41037f0 2024-05-20T20:06:46.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.546+0000 7f9fb0c00700 1 -- 172.21.15.28:0/3532208354 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9fa4005680 con 0x7f9fb41037f0 2024-05-20T20:06:46.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.546+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9fb41a8390 con 0x7f9fb41037f0 2024-05-20T20:06:46.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.546+0000 7f9fb0c00700 1 -- 172.21.15.28:0/3532208354 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa400fb20 con 0x7f9fb41037f0 2024-05-20T20:06:46.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.546+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9fb41a8840 con 0x7f9fb41037f0 2024-05-20T20:06:46.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.550+0000 7f9fb0c00700 1 -- 172.21.15.28:0/3532208354 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f9fa4029070 con 0x7f9fb41037f0 2024-05-20T20:06:46.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.552+0000 7f9fb0c00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9f980b2c20 0x7f9f980b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.552+0000 7f9fb0c00700 1 -- 172.21.15.28:0/3532208354 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f9fa40e2e70 con 0x7f9fb41037f0 2024-05-20T20:06:46.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.552+0000 7f9fb3400700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9f980b2c20 0x7f9f980b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.553+0000 7f9fb3400700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9f980b2c20 0x7f9f980b50e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f9fb41030f0 tx=0x7f9fa0009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:46.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.553+0000 7f9fc529cb80 1 Processor -- start 2024-05-20T20:06:46.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.554+0000 7f9fc529cb80 1 -- start start 2024-05-20T20:06:46.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.554+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67cfc9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.554+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfcee0 0x557f67cf6ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.554+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67cf7570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.554+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557f67c54df0 con 0x557f67cfcee0 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.554+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557f67c44990 con 0x557f67cfc5c0 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.554+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557f67c489d0 con 0x557f67cf70f0 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67cfc9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67cfc9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44992/0 (socket says 172.21.15.28:44992) 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 -- 172.21.15.28:0/417569981 learned_addr learned my addr 172.21.15.28:0/417569981 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb8a00700 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67cf7570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3400700 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfcee0 0x557f67cf6ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 -- 172.21.15.28:0/417569981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 msgr2=0x557f67cf7570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67cf7570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 -- 172.21.15.28:0/417569981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfcee0 msgr2=0x557f67cf6ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfcee0 0x557f67cf6ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.555+0000 7f9fb3e00700 1 -- 172.21.15.28:0/417569981 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557f67c0d3d0 con 0x557f67cfc5c0 2024-05-20T20:06:46.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.556+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67cfc9a0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40a9f40 tx=0x7f9fa40a9b40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f3cd6c8a17efca1b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:46.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.556+0000 7f9fa9600700 1 -- 172.21.15.28:0/417569981 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa40ae360 con 0x557f67cfc5c0 2024-05-20T20:06:46.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.556+0000 7f9fa9600700 1 -- 172.21.15.28:0/417569981 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9fa40b2d80 con 0x557f67cfc5c0 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.556+0000 7f9fa9600700 1 -- 172.21.15.28:0/417569981 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa40aa470 con 0x557f67cfc5c0 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 -- 172.21.15.28:0/417569981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 msgr2=0x557f67cfc9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67cfc9a0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40a9f40 tx=0x7f9fa40a9b40 comp rx=0 tx=0).stop 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 -- 172.21.15.28:0/417569981 shutdown_connections 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67cf7570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67cfc9a0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/417569981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfcee0 0x557f67cf6ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 -- 172.21.15.28:0/417569981 >> 172.21.15.28:0/417569981 conn(0x557f67c24b90 msgr2=0x557f67cf1640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 -- 172.21.15.28:0/417569981 shutdown_connections 2024-05-20T20:06:46.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.557+0000 7f9fc529cb80 1 -- 172.21.15.28:0/417569981 wait complete. 2024-05-20T20:06:46.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.558+0000 7f9fc529cb80 1 -- start start 2024-05-20T20:06:46.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.558+0000 7f9fc529cb80 1 Processor -- start 2024-05-20T20:06:46.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cf70f0 0x557f67d02be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67cfffd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d00510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557f67b4af50 con 0x557f67cfc5c0 2024-05-20T20:06:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557f67be5710 con 0x557f67cf70f0 2024-05-20T20:06:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557f67d03750 con 0x557f67cfcee0 2024-05-20T20:06:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fc529cb80 10 client.0 _ll_register_callbacks cb 0x557f67beec20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cf70f0 0x557f67d02be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fb3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67cfffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cf70f0 0x557f67d02be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45000/0 (socket says 172.21.15.28:45000) 2024-05-20T20:06:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.559+0000 7f9fb3e00700 1 -- 172.21.15.28:0/450133443 learned_addr learned my addr 172.21.15.28:0/450133443 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:46.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9fb8a00700 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d00510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9fb3e00700 1 -- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 msgr2=0x557f67d00510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d00510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9fb3e00700 1 -- 172.21.15.28:0/450133443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 msgr2=0x557f67cfffd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67cfffd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9fb3e00700 1 -- 172.21.15.28:0/450133443 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557f67b94880 con 0x557f67cf70f0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cf70f0 0x557f67d02be0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40a9cb0 tx=0x7f9fa40ae3d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=59ae0130f6e9cdc0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.560+0000 7f9f9e000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.561+0000 7f9f9e000700 1 -- 172.21.15.28:0/450133443 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa40c8020 con 0x557f67cf70f0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.561+0000 7f9f9e000700 1 -- 172.21.15.28:0/450133443 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f9fa40aa210 con 0x557f67cf70f0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.561+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9fa40b2d80 con 0x557f67cf70f0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.561+0000 7f9f9e000700 1 -- 172.21.15.28:0/450133443 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f9fa40aa410 con 0x557f67cf70f0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.561+0000 7f9fc529cb80 10 client.25502 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.561+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557f67c15010 con 0x557f67cf70f0 2024-05-20T20:06:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.561+0000 7f9f9e000700 1 -- 172.21.15.28:0/450133443 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9fa40b2d80 con 0x557f67cf70f0 2024-05-20T20:06:46.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.562+0000 7f9f9e000700 1 -- 172.21.15.28:0/450133443 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9fa408b050 con 0x557f67cf70f0 2024-05-20T20:06:46.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.562+0000 7f9fc529cb80 10 client.25502 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:46.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.562+0000 7f9fc529cb80 10 client.25502 resolve_mds: resolved name 'i' to daemon mds.i 2024-05-20T20:06:46.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.562+0000 7f9fc529cb80 20 client.25502 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:46.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.562+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d10010 0x557f67d11480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.562+0000 7f9fc529cb80 4 client.25502 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:46.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.562+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557f67c56d40 con 0x557f67d10010 2024-05-20T20:06:46.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.563+0000 7f9fb3400700 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d10010 0x557f67d11480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.563+0000 7f9fb3400700 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d10010 0x557f67d11480 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=344b1f618ec981d server_cookie=99f379da17be3079 in_seq=0 out_seq=0 2024-05-20T20:06:46.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.563+0000 7f9f9e000700 10 client.25502 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9f9e000700 1 -- 172.21.15.28:0/450133443 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x557f67c56d40 con 0x557f67d10010 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9f9e000700 10 client.25502 handle_command_reply: tid=0 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 client.25502 shutdown 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cf70f0 msgr2=0x557f67d02be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cf70f0 0x557f67d02be0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9fa40a9cb0 tx=0x7f9fa40ae3d0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 shutdown_connections 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d00510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cf70f0 0x557f67d02be0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d10010 0x557f67d11480 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/450133443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67cfffd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 >> 172.21.15.28:0/450133443 conn(0x557f67c24b90 msgr2=0x557f67cf0d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:46.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.564+0000 7f9f9e000700 0 client.25502 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:06:46.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.565+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 shutdown_connections 2024-05-20T20:06:46.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.565+0000 7f9fc529cb80 1 -- 172.21.15.28:0/450133443 wait complete. 2024-05-20T20:06:46.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.565+0000 7f9fc529cb80 20 client.25502 trim_cache size 0 max 16384 2024-05-20T20:06:46.572 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:46.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 Processor -- start 2024-05-20T20:06:46.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 -- start start 2024-05-20T20:06:46.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cf70f0 0x557f67cf6ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67d0f940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d004c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557f67d0f7c0 con 0x557f67cf70f0 2024-05-20T20:06:46.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557f67d0fe90 con 0x557f67cfc5c0 2024-05-20T20:06:46.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.589+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557f67d10560 con 0x557f67cfcee0 2024-05-20T20:06:46.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb8a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d004c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cf70f0 0x557f67cf6ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67d0f940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb8a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d004c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38728/0 (socket says 172.21.15.28:38728) 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb8a00700 1 -- 172.21.15.28:0/2360155416 learned_addr learned my addr 172.21.15.28:0/2360155416 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb3400700 1 -- 172.21.15.28:0/2360155416 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 msgr2=0x557f67d004c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb3400700 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d004c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb3400700 1 -- 172.21.15.28:0/2360155416 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cf70f0 msgr2=0x557f67cf6ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb3400700 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cf70f0 0x557f67cf6ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb3400700 1 -- 172.21.15.28:0/2360155416 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557f67d02820 con 0x557f67cfc5c0 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.590+0000 7f9fb8a00700 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d004c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:46.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.591+0000 7f9fb3400700 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67d0f940 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x557f67d032b0 tx=0x7f9fa0014000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6fb5254cfc74ea16 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:46.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.591+0000 7f9fa8c00700 1 -- 172.21.15.28:0/2360155416 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa0003880 con 0x557f67cfc5c0 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.591+0000 7f9fa8c00700 1 -- 172.21.15.28:0/2360155416 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9fa0011b20 con 0x557f67cfc5c0 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.591+0000 7f9fa8c00700 1 -- 172.21.15.28:0/2360155416 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa002acf0 con 0x557f67cfc5c0 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 -- 172.21.15.28:0/2360155416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 msgr2=0x557f67d0f940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67d0f940 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x557f67d032b0 tx=0x7f9fa0014000 comp rx=0 tx=0).stop 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 -- 172.21.15.28:0/2360155416 shutdown_connections 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cfcee0 0x557f67d004c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cf70f0 0x557f67cf6ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/2360155416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfc5c0 0x557f67d0f940 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 -- 172.21.15.28:0/2360155416 >> 172.21.15.28:0/2360155416 conn(0x557f67c24b90 msgr2=0x557f67cffdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 -- 172.21.15.28:0/2360155416 shutdown_connections 2024-05-20T20:06:46.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.592+0000 7f9fc529cb80 1 -- 172.21.15.28:0/2360155416 wait complete. 2024-05-20T20:06:46.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.593+0000 7f9fc529cb80 1 -- start start 2024-05-20T20:06:46.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.593+0000 7f9fc529cb80 1 Processor -- start 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67d1aae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67d1b020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fc529cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfcee0 0x557f67d158b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557f67d11300 con 0x557f67cfc5c0 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557f67d10c30 con 0x557f67cfcee0 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fc529cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557f67d119d0 con 0x557f67cf70f0 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fc529cb80 10 client.0 _ll_register_callbacks cb 0x557f67beec20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67d1aae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fb3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67d1aae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38730/0 (socket says 172.21.15.28:38730) 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fb3e00700 1 -- 172.21.15.28:0/504164896 learned_addr learned my addr 172.21.15.28:0/504164896 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fb3400700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67d1b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.594+0000 7f9fb8a00700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfcee0 0x557f67d158b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb3e00700 1 -- 172.21.15.28:0/504164896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfcee0 msgr2=0x557f67d158b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfcee0 0x557f67d158b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb3e00700 1 -- 172.21.15.28:0/504164896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 msgr2=0x557f67d1b020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67d1b020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb3e00700 1 -- 172.21.15.28:0/504164896 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557f67d160b0 con 0x557f67cf70f0 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb3400700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67d1b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb8a00700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfcee0 0x557f67d158b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fb3e00700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67d1aae0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f9fa4005b40 tx=0x7f9fa40e0500 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2d16f799c957d06b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9f9e000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:06:46.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9f9e000700 1 -- 172.21.15.28:0/504164896 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9fa40c8070 con 0x557f67cf70f0 2024-05-20T20:06:46.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9f9e000700 1 -- 172.21.15.28:0/504164896 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f9fa40aa210 con 0x557f67cf70f0 2024-05-20T20:06:46.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.595+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9fa0011b20 con 0x557f67cf70f0 2024-05-20T20:06:46.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.596+0000 7f9f9e000700 1 -- 172.21.15.28:0/504164896 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f9fa40a5e70 con 0x557f67cf70f0 2024-05-20T20:06:46.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.596+0000 7f9fc529cb80 10 client.25480 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:46.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.596+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557f67d160b0 con 0x557f67cf70f0 2024-05-20T20:06:46.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.596+0000 7f9f9e000700 1 -- 172.21.15.28:0/504164896 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9fa0011b20 con 0x557f67cf70f0 2024-05-20T20:06:46.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.596+0000 7f9f9e000700 1 -- 172.21.15.28:0/504164896 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9fa408b070 con 0x557f67cf70f0 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.597+0000 7f9fc529cb80 10 client.25480 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.597+0000 7f9fc529cb80 10 client.25480 resolve_mds: resolved name 'i' to daemon mds.i 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.597+0000 7f9fc529cb80 20 client.25480 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.597+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d80a40 0x557f67d7f750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.597+0000 7f9fc529cb80 4 client.25480 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.597+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "status"}) -- 0x557f67c50520 con 0x557f67d80a40 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.597+0000 7f9fb3400700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d80a40 0x557f67d7f750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fb3400700 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d80a40 0x557f67d7f750 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f5dab22c7a6814af server_cookie=b6ed241f3a0b586f in_seq=0 out_seq=0 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9f9e000700 10 client.25480 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9f9e000700 1 -- 172.21.15.28:0/504164896 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x557f67c50520 con 0x557f67d80a40 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9f9e000700 10 client.25480 handle_command_reply: tid=0 2024-05-20T20:06:46.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 client.25480 shutdown 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 msgr2=0x557f67d1aae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67d1aae0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f9fa4005b40 tx=0x7f9fa40e0500 comp rx=0 tx=0).stop 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 shutdown_connections 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557f67cfcee0 0x557f67d158b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557f67cf70f0 0x557f67d1aae0 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x557f67d80a40 0x557f67d7f750 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 --2- 172.21.15.28:0/504164896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557f67cfc5c0 0x557f67d1b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 >> 172.21.15.28:0/504164896 conn(0x557f67c24b90 msgr2=0x557f67cfd2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.598+0000 7f9f9e000700 0 client.25480 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.599+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 shutdown_connections 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.599+0000 7f9fc529cb80 1 -- 172.21.15.28:0/504164896 wait complete. 2024-05-20T20:06:46.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.599+0000 7f9fc529cb80 20 client.25480 trim_cache size 0 max 16384 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 2, 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24493, 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 302.97816560699999, 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 29, 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 81, 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 80, 2024-05-20T20:06:46.604 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 356.28898218900002 2024-05-20T20:06:46.605 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:06:46.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9f980b2c20 msgr2=0x7f9f980b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9f980b2c20 0x7f9f980b50e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f9fb41030f0 tx=0x7f9fa0009260 comp rx=0 tx=0).stop 2024-05-20T20:06:46.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41037f0 msgr2=0x7f9fb4075840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:46.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41037f0 0x7f9fb4075840 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f9fa4006010 tx=0x7f9fa4005280 comp rx=0 tx=0).stop 2024-05-20T20:06:46.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 shutdown_connections 2024-05-20T20:06:46.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9f980b2c20 0x7f9f980b50e0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9fb41088f0 0x7f9fb40762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9fb41037f0 0x7f9fb4075840 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 --2- 172.21.15.28:0/3532208354 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9fb41041a0 0x7f9fb4075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 >> 172.21.15.28:0/3532208354 conn(0x7f9fb407a6a0 msgr2=0x7f9fb4101840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 shutdown_connections 2024-05-20T20:06:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:46.600+0000 7f9fbaa00700 1 -- 172.21.15.28:0/3532208354 wait complete. 2024-05-20T20:06:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:46 smithi028 ceph-mon[31178]: pgmap v37: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:46 smithi154 ceph-mon[36314]: pgmap v37: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:47.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:46 smithi033 ceph-mon[36179]: pgmap v37: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:48.836 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.h status 2024-05-20T20:06:49.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:48 smithi028 ceph-mon[31178]: pgmap v38: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:49.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:48 smithi154 ceph-mon[36314]: pgmap v38: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:48 smithi033 ceph-mon[36179]: pgmap v38: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 1.2 KiB/s wr, 10 op/s 2024-05-20T20:06:49.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 -- 172.21.15.28:0/3103628149 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac1095a0 msgr2=0x7f15ac109980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 --2- 172.21.15.28:0/3103628149 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac1095a0 0x7f15ac109980 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f15a00098f0 tx=0x7f15a0009c00 comp rx=0 tx=0).stop 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 -- 172.21.15.28:0/3103628149 shutdown_connections 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 --2- 172.21.15.28:0/3103628149 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15ac06c470 0x7f15ac1103c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 --2- 172.21.15.28:0/3103628149 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac1095a0 0x7f15ac109980 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 --2- 172.21.15.28:0/3103628149 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15ac06bab0 0x7f15ac06bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 -- 172.21.15.28:0/3103628149 >> 172.21.15.28:0/3103628149 conn(0x7f15ac0793b0 msgr2=0x7f15ac0797c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 -- 172.21.15.28:0/3103628149 shutdown_connections 2024-05-20T20:06:49.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.410+0000 7f15b3600700 1 -- 172.21.15.28:0/3103628149 wait complete. 2024-05-20T20:06:49.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.411+0000 7f15b3600700 1 Processor -- start 2024-05-20T20:06:49.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.411+0000 7f15b3600700 1 -- start start 2024-05-20T20:06:49.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b3600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac06bab0 0x7f15ac19bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b3600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15ac06c470 0x7f15ac19c410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b3600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15ac1095a0 0x7f15ac1a07a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b3600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac113720 con 0x7f15ac1095a0 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b3600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1135a0 con 0x7f15ac06c470 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b3600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1138a0 con 0x7f15ac06bab0 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac06bab0 0x7f15ac19bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15abe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15ac06c470 0x7f15ac19c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac06bab0 0x7f15ac19bed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55210/0 (socket says 172.21.15.28:55210) 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b0c00700 1 -- 172.21.15.28:0/498009253 learned_addr learned my addr 172.21.15.28:0/498009253 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:49.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.412+0000 7f15b1600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15ac1095a0 0x7f15ac1a07a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.413+0000 7f15b0c00700 1 -- 172.21.15.28:0/498009253 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15ac06c470 msgr2=0x7f15ac19c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.413+0000 7f15b0c00700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15ac06c470 0x7f15ac19c410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.413+0000 7f15b0c00700 1 -- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15ac1095a0 msgr2=0x7f15ac1a07a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.413+0000 7f15b0c00700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15ac1095a0 0x7f15ac1a07a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.413+0000 7f15b0c00700 1 -- 172.21.15.28:0/498009253 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f15ac1a0ee0 con 0x7f15ac06bab0 2024-05-20T20:06:49.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.413+0000 7f15b1600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15ac1095a0 0x7f15ac1a07a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:49.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.413+0000 7f15b0c00700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac06bab0 0x7f15ac19bed0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f15a0009230 tx=0x7f15a000fa70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:49.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.414+0000 7f15a9600700 1 -- 172.21.15.28:0/498009253 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f15a0005140 con 0x7f15ac06bab0 2024-05-20T20:06:49.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.414+0000 7f15a9600700 1 -- 172.21.15.28:0/498009253 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f15a000b560 con 0x7f15ac06bab0 2024-05-20T20:06:49.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.414+0000 7f15a9600700 1 -- 172.21.15.28:0/498009253 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f15a0003970 con 0x7f15ac06bab0 2024-05-20T20:06:49.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.414+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f15ac1a1170 con 0x7f15ac06bab0 2024-05-20T20:06:49.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.414+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f15ac1a8a50 con 0x7f15ac06bab0 2024-05-20T20:06:49.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.419+0000 7f15be077b80 1 Processor -- start 2024-05-20T20:06:49.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.419+0000 7f15be077b80 1 -- start start 2024-05-20T20:06:49.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.419+0000 7f15be077b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4ff290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.419+0000 7f15be077b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4f9260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.419+0000 7f15be077b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15be077b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55aa4a457e40 con 0x55aa4a4ff7d0 2024-05-20T20:06:49.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15be077b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55aa4a4479e0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15be077b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55aa4a44ba20 con 0x55aa4a4f9880 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15b1600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4ff290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15b1600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4ff290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57422/0 (socket says 172.21.15.28:57422) 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15b1600700 1 -- 172.21.15.28:0/52721904 learned_addr learned my addr 172.21.15.28:0/52721904 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15a9600700 1 -- 172.21.15.28:0/498009253 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f15a000ba50 con 0x7f15ac06bab0 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15abe00700 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15b1600700 1 -- 172.21.15.28:0/52721904 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 msgr2=0x55aa4a4f9ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15b1600700 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.420+0000 7f15b1600700 1 -- 172.21.15.28:0/52721904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 msgr2=0x55aa4a4f9260 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.421+0000 7f15b1600700 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4f9260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.421+0000 7f15b1600700 1 -- 172.21.15.28:0/52721904 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55aa4a410420 con 0x55aa4a4feeb0 2024-05-20T20:06:49.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.421+0000 7f15b1600700 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4ff290 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f15a4003eb0 tx=0x7f15a4009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=17c66652e5b66708 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:49.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.421+0000 7f159e000700 1 -- 172.21.15.28:0/52721904 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f15a4019020 con 0x55aa4a4feeb0 2024-05-20T20:06:49.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.421+0000 7f159e000700 1 -- 172.21.15.28:0/52721904 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f15a40092d0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.421+0000 7f159e000700 1 -- 172.21.15.28:0/52721904 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f15a400e8b0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.422+0000 7f15be077b80 1 -- 172.21.15.28:0/52721904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 msgr2=0x55aa4a4ff290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.422+0000 7f15be077b80 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4ff290 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f15a4003eb0 tx=0x7f15a4009920 comp rx=0 tx=0).stop 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.422+0000 7f15a9600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f15940b2c20 0x7f15940b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15be077b80 1 -- 172.21.15.28:0/52721904 shutdown_connections 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15be077b80 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15be077b80 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4ff290 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15be077b80 1 --2- 172.21.15.28:0/52721904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4f9260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15be077b80 1 -- 172.21.15.28:0/52721904 >> 172.21.15.28:0/52721904 conn(0x55aa4a427be0 msgr2=0x55aa4a4f4010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:49.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15a9600700 1 -- 172.21.15.28:0/498009253 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f15a00e3810 con 0x7f15ac06bab0 2024-05-20T20:06:49.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15b0c00700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f15940b2c20 0x7f15940b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15be077b80 1 -- 172.21.15.28:0/52721904 shutdown_connections 2024-05-20T20:06:49.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.423+0000 7f15be077b80 1 -- 172.21.15.28:0/52721904 wait complete. 2024-05-20T20:06:49.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.424+0000 7f15be077b80 1 -- start start 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.424+0000 7f15b0c00700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f15940b2c20 0x7f15940b50e0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f15a00e0d90 tx=0x7f15a00095b0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.424+0000 7f15be077b80 1 Processor -- start 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15be077b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a505450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15be077b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a5027f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15be077b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a502d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15be077b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55aa4a34dfa0 con 0x55aa4a4ff7d0 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15be077b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55aa4a3e8760 con 0x55aa4a4feeb0 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15be077b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55aa4a506240 con 0x55aa4a4f9880 2024-05-20T20:06:49.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15be077b80 10 client.0 _ll_register_callbacks cb 0x55aa4a3f1c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:49.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15abe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a5027f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15abe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a5027f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57428/0 (socket says 172.21.15.28:57428) 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15abe00700 1 -- 172.21.15.28:0/3752609501 learned_addr learned my addr 172.21.15.28:0/3752609501 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15b1600700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a505450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15b1600700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a502d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15abe00700 1 -- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 msgr2=0x55aa4a505450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15abe00700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a505450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.425+0000 7f15abe00700 1 -- 172.21.15.28:0/3752609501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 msgr2=0x55aa4a502d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f15abe00700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a502d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f15abe00700 1 -- 172.21.15.28:0/3752609501 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55aa4a3978d0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f15b1600700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a502d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f15abe00700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a5027f0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f1598012700 tx=0x7f1598012ac0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d716ff207136ce03 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f1592a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:49.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f1592a00700 1 -- 172.21.15.28:0/3752609501 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f159800f540 con 0x55aa4a4feeb0 2024-05-20T20:06:49.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f15a40092d0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.426+0000 7f1592a00700 1 -- 172.21.15.28:0/3752609501 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f159800f6e0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.427+0000 7f1592a00700 1 -- 172.21.15.28:0/3752609501 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f159800f8e0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.427+0000 7f15be077b80 10 client.25520 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.427+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55aa4a418060 con 0x55aa4a4feeb0 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f1592a00700 1 -- 172.21.15.28:0/3752609501 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f15a40092d0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f1592a00700 1 -- 172.21.15.28:0/3752609501 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1598010b10 con 0x55aa4a4feeb0 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f15be077b80 10 client.25520 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f15be077b80 10 client.25520 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f15be077b80 20 client.25520 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:49.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f15be077b80 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a512e50 0x55aa4a515270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f15be077b80 4 client.25520 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55aa4a459d90 con 0x55aa4a512e50 2024-05-20T20:06:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.428+0000 7f15b1600700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a512e50 0x55aa4a515270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.429+0000 7f15b1600700 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a512e50 0x55aa4a515270 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=bb944141964270ae server_cookie=a347a50c5c2d8e24 in_seq=0 out_seq=0 2024-05-20T20:06:49.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.429+0000 7f1592a00700 10 client.25520 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:49.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f1592a00700 1 -- 172.21.15.28:0/3752609501 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55aa4a459d90 con 0x55aa4a512e50 2024-05-20T20:06:49.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f1592a00700 10 client.25520 handle_command_reply: tid=0 2024-05-20T20:06:49.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 client.25520 shutdown 2024-05-20T20:06:49.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 msgr2=0x55aa4a5027f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a5027f0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f1598012700 tx=0x7f1598012ac0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 shutdown_connections 2024-05-20T20:06:49.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a502d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a512e50 0x55aa4a515270 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4f9880 0x55aa4a505450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 --2- 172.21.15.28:0/3752609501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4feeb0 0x55aa4a5027f0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f1592a00700 0 client.25520 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.430+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 >> 172.21.15.28:0/3752609501 conn(0x55aa4a427be0 msgr2=0x55aa4a4f8250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.431+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 shutdown_connections 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.431+0000 7f15be077b80 1 -- 172.21.15.28:0/3752609501 wait complete. 2024-05-20T20:06:49.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.431+0000 7f15be077b80 20 client.25520 trim_cache size 0 max 16384 2024-05-20T20:06:49.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.449+0000 7f15be077b80 1 Processor -- start 2024-05-20T20:06:49.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.449+0000 7f15be077b80 1 -- start start 2024-05-20T20:06:49.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.449+0000 7f15be077b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15be077b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4f40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15be077b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4ffc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15be077b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1affd0 con 0x55aa4a4ff7d0 2024-05-20T20:06:49.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15be077b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1b1b10 con 0x55aa4a4f9880 2024-05-20T20:06:49.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15be077b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1b06a0 con 0x55aa4a4feeb0 2024-05-20T20:06:49.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15b1600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4ffc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15abe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4f40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15b1600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4ffc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57704/0 (socket says 172.21.15.28:57704) 2024-05-20T20:06:49.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15b1600700 1 -- 172.21.15.28:0/3787279238 learned_addr learned my addr 172.21.15.28:0/3787279238 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:49.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15b1600700 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15abe00700 1 -- 172.21.15.28:0/3787279238 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 msgr2=0x55aa4a4f9260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:49.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15abe00700 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15abe00700 1 -- 172.21.15.28:0/3787279238 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 msgr2=0x55aa4a4ffc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:49.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15abe00700 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4ffc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.450+0000 7f15abe00700 1 -- 172.21.15.28:0/3787279238 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55aa4a4ec190 con 0x55aa4a4feeb0 2024-05-20T20:06:49.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.451+0000 7f15abe00700 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4f40f0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f15980126d0 tx=0x7f15980058e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e336d0ccbab4bbd0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:49.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.451+0000 7f159d600700 1 -- 172.21.15.28:0/3787279238 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f159800f600 con 0x55aa4a4feeb0 2024-05-20T20:06:49.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.451+0000 7f159d600700 1 -- 172.21.15.28:0/3787279238 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f159801c540 con 0x55aa4a4feeb0 2024-05-20T20:06:49.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.451+0000 7f159d600700 1 -- 172.21.15.28:0/3787279238 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f159801ce60 con 0x55aa4a4feeb0 2024-05-20T20:06:49.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 -- 172.21.15.28:0/3787279238 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 msgr2=0x55aa4a4f40f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4f40f0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f15980126d0 tx=0x7f15980058e0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 -- 172.21.15.28:0/3787279238 shutdown_connections 2024-05-20T20:06:49.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a4ffc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a4f9260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 --2- 172.21.15.28:0/3787279238 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a4f40f0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 -- 172.21.15.28:0/3787279238 >> 172.21.15.28:0/3787279238 conn(0x55aa4a427be0 msgr2=0x55aa4a4f39e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:49.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 -- 172.21.15.28:0/3787279238 shutdown_connections 2024-05-20T20:06:49.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.452+0000 7f15be077b80 1 -- 172.21.15.28:0/3787279238 wait complete. 2024-05-20T20:06:49.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.453+0000 7f15be077b80 1 -- start start 2024-05-20T20:06:49.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.453+0000 7f15be077b80 1 Processor -- start 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15be077b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a50b500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15be077b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a50ba40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15be077b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a5084d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15be077b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1b1440 con 0x55aa4a4ff7d0 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15be077b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1b0d70 con 0x55aa4a4f9880 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15be077b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f15ac1adf70 con 0x55aa4a4feeb0 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15be077b80 10 client.0 _ll_register_callbacks cb 0x55aa4a3f1c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:49.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a50b500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15abe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a50ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a50b500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57440/0 (socket says 172.21.15.28:57440) 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 -- 172.21.15.28:0/1154208247 learned_addr learned my addr 172.21.15.28:0/1154208247 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a5084d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 -- 172.21.15.28:0/1154208247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 msgr2=0x55aa4a50ba40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a50ba40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 -- 172.21.15.28:0/1154208247 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 msgr2=0x55aa4a5084d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a5084d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.454+0000 7f15b1600700 1 -- 172.21.15.28:0/1154208247 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55aa4a5107d0 con 0x55aa4a4f9880 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.455+0000 7f15b1600700 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a50b500 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f15a4009fd0 tx=0x7f15a4022040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=da8e7b00962481f7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:49.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.455+0000 7f1592a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.455+0000 7f1592a00700 1 -- 172.21.15.28:0/1154208247 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f15a4019040 con 0x55aa4a4f9880 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.455+0000 7f1592a00700 1 -- 172.21.15.28:0/1154208247 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f15a400eb00 con 0x55aa4a4f9880 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.455+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f159801c540 con 0x55aa4a4f9880 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.456+0000 7f1592a00700 1 -- 172.21.15.28:0/1154208247 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f15a400ed00 con 0x55aa4a4f9880 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.456+0000 7f15be077b80 10 client.25526 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.456+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55aa4a57de30 con 0x55aa4a4f9880 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.456+0000 7f1592a00700 1 -- 172.21.15.28:0/1154208247 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f159801c540 con 0x55aa4a4f9880 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.456+0000 7f1592a00700 1 -- 172.21.15.28:0/1154208247 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f15a4015450 con 0x55aa4a4f9880 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.457+0000 7f15be077b80 10 client.25526 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.457+0000 7f15be077b80 10 client.25526 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:06:49.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.457+0000 7f15be077b80 20 client.25526 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.457+0000 7f15be077b80 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a57e240 0x55aa4a580630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.457+0000 7f15be077b80 4 client.25526 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.457+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55aa4a453570 con 0x55aa4a57e240 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.457+0000 7f15abe00700 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a57e240 0x55aa4a580630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15abe00700 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a57e240 0x55aa4a580630 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=3f5220aad475b6ae server_cookie=471b347779baf61 in_seq=0 out_seq=0 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f1592a00700 10 client.25526 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f1592a00700 1 -- 172.21.15.28:0/1154208247 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55aa4a453570 con 0x55aa4a57e240 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f1592a00700 10 client.25526 handle_command_reply: tid=0 2024-05-20T20:06:49.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 client.25526 shutdown 2024-05-20T20:06:49.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 msgr2=0x55aa4a50b500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a50b500 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f15a4009fd0 tx=0x7f15a4022040 comp rx=0 tx=0).stop 2024-05-20T20:06:49.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 shutdown_connections 2024-05-20T20:06:49.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55aa4a4ff7d0 0x55aa4a5084d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55aa4a4f9880 0x55aa4a50b500 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55aa4a57e240 0x55aa4a580630 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 --2- 172.21.15.28:0/1154208247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55aa4a4feeb0 0x55aa4a50ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f1592a00700 0 client.25526 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:06:49.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.458+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 >> 172.21.15.28:0/1154208247 conn(0x55aa4a427be0 msgr2=0x55aa4a4ffc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:49.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.459+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 shutdown_connections 2024-05-20T20:06:49.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.459+0000 7f15be077b80 1 -- 172.21.15.28:0/1154208247 wait complete. 2024-05-20T20:06:49.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.459+0000 7f15be077b80 20 client.25526 trim_cache size 0 max 16384 2024-05-20T20:06:49.462 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:06:49.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f15940b2c20 msgr2=0x7f15940b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f15940b2c20 0x7f15940b50e0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f15a00e0d90 tx=0x7f15a00095b0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac06bab0 msgr2=0x7f15ac19bed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:49.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac06bab0 0x7f15ac19bed0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f15a0009230 tx=0x7f15a000fa70 comp rx=0 tx=0).stop 2024-05-20T20:06:49.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 shutdown_connections 2024-05-20T20:06:49.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15ac1095a0 0x7f15ac1a07a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15ac06bab0 0x7f15ac19bed0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f15940b2c20 0x7f15940b50e0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 --2- 172.21.15.28:0/498009253 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15ac06c470 0x7f15ac19c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:49.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 >> 172.21.15.28:0/498009253 conn(0x7f15ac0793b0 msgr2=0x7f15ac100eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:49.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 shutdown_connections 2024-05-20T20:06:49.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:49.460+0000 7f15b3600700 1 -- 172.21.15.28:0/498009253 wait complete. 2024-05-20T20:06:49.763 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:50.447 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:06:51.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:50 smithi028 ceph-mon[31178]: pgmap v39: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:06:51.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.105+0000 7f2374a00640 1 -- 172.21.15.28:0/2106683656 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2370104a90 msgr2=0x7f2370104f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.105+0000 7f2374a00640 1 --2- 172.21.15.28:0/2106683656 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2370104a90 0x7f2370104f10 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f2358009900 tx=0x7f235802a400 comp rx=0 tx=0).stop 2024-05-20T20:06:51.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.106+0000 7f2374a00640 1 -- 172.21.15.28:0/2106683656 shutdown_connections 2024-05-20T20:06:51.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.106+0000 7f2374a00640 1 --2- 172.21.15.28:0/2106683656 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2370104a90 0x7f2370104f10 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.106+0000 7f2374a00640 1 --2- 172.21.15.28:0/2106683656 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2370105450 0x7f2370111960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.106+0000 7f2374a00640 1 --2- 172.21.15.28:0/2106683656 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f237010aaa0 0x7f237010ae80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.106+0000 7f2374a00640 1 -- 172.21.15.28:0/2106683656 >> 172.21.15.28:0/2106683656 conn(0x7f2370100560 msgr2=0x7f2370102980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:51.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.106+0000 7f2374a00640 1 -- 172.21.15.28:0/2106683656 shutdown_connections 2024-05-20T20:06:51.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.106+0000 7f2374a00640 1 -- 172.21.15.28:0/2106683656 wait complete. 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 Processor -- start 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 -- start start 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2370104a90 0x7f237019b2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2370105450 0x7f237019b7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f237010aaa0 0x7f237019fb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2370114db0 con 0x7f2370104a90 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2370114c30 con 0x7f237010aaa0 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f2374a00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2370114f30 con 0x7f2370105450 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.107+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2370104a90 0x7f237019b2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2370104a90 0x7f237019b2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57728/0 (socket says 172.21.15.28:57728) 2024-05-20T20:06:51.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 -- 172.21.15.28:0/212251547 learned_addr learned my addr 172.21.15.28:0/212251547 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236f200640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f237010aaa0 0x7f237019fb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f2367e00640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2370105450 0x7f237019b7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 -- 172.21.15.28:0/212251547 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2370105450 msgr2=0x7f237019b7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2370105450 0x7f237019b7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 -- 172.21.15.28:0/212251547 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f237010aaa0 msgr2=0x7f237019fb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f237010aaa0 0x7f237019fb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 -- 172.21.15.28:0/212251547 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f23701a0250 con 0x7f2370104a90 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236f200640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f237010aaa0 0x7f237019fb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.108+0000 7f236e800640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2370104a90 0x7f237019b2b0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f236000ea60 tx=0x7f236000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.109+0000 7f2367400640 1 -- 172.21.15.28:0/212251547 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f236000cde0 con 0x7f2370104a90 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.109+0000 7f2367400640 1 -- 172.21.15.28:0/212251547 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23600035f0 con 0x7f2370104a90 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.109+0000 7f2367400640 1 -- 172.21.15.28:0/212251547 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2360010690 con 0x7f2370104a90 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.109+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f23701a0540 con 0x7f2370104a90 2024-05-20T20:06:51.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.109+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f23701a7dd0 con 0x7f2370104a90 2024-05-20T20:06:51.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.112+0000 7f2378237740 1 Processor -- start 2024-05-20T20:06:51.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2378237740 1 -- start start 2024-05-20T20:06:51.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2367400640 1 -- 172.21.15.28:0/212251547 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f23600037a0 con 0x7f2370104a90 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2378237740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac52230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2378237740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac52770 0x55eb7ac5caf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2378237740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5f420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2378237740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eb7abc2b90 con 0x55eb7ac51e50 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2378237740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eb7abb2730 con 0x55eb7ac5d030 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2378237740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eb7abb6770 con 0x55eb7ac52770 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f236f200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac52230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f236f200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac52230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57744/0 (socket says 172.21.15.28:57744) 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f236f200640 1 -- 172.21.15.28:0/510706839 learned_addr learned my addr 172.21.15.28:0/510706839 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:51.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f2367e00640 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5f420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.113+0000 7f236e800640 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac52770 0x55eb7ac5caf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f2367e00640 1 -- 172.21.15.28:0/510706839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac52770 msgr2=0x55eb7ac5caf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f2367e00640 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac52770 0x55eb7ac5caf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f2367e00640 1 -- 172.21.15.28:0/510706839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 msgr2=0x55eb7ac52230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f2367e00640 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac52230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f2367e00640 1 -- 172.21.15.28:0/510706839 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55eb7ab4b8b0 con 0x55eb7ac5d030 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f236e800640 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac52770 0x55eb7ac5caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f236f200640 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac52230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.114+0000 7f2367e00640 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5f420 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f235802a9f0 tx=0x7f23580040d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4f2c22a8f87fd7a9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.115+0000 7f235f400640 1 -- 172.21.15.28:0/510706839 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2358035040 con 0x55eb7ac5d030 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.115+0000 7f235f400640 1 -- 172.21.15.28:0/510706839 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23580054f0 con 0x55eb7ac5d030 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.115+0000 7f235f400640 1 -- 172.21.15.28:0/510706839 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23580035f0 con 0x55eb7ac5d030 2024-05-20T20:06:51.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.115+0000 7f2367400640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23540b2b80 0x7f23540b5040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.115+0000 7f2378237740 1 -- 172.21.15.28:0/510706839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac5d030 msgr2=0x55eb7ac5f420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.115+0000 7f2378237740 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5f420 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f235802a9f0 tx=0x7f23580040d0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.115+0000 7f2367400640 1 -- 172.21.15.28:0/212251547 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f23600e2a30 con 0x7f2370104a90 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f236f200640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23540b2b80 0x7f23540b5040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 -- 172.21.15.28:0/510706839 shutdown_connections 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5f420 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac52770 0x55eb7ac5caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 --2- 172.21.15.28:0/510706839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac52230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 -- 172.21.15.28:0/510706839 >> 172.21.15.28:0/510706839 conn(0x55eb7ac4e360 msgr2=0x55eb7ac4e740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 -- 172.21.15.28:0/510706839 shutdown_connections 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 -- 172.21.15.28:0/510706839 wait complete. 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f236f200640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23540b2b80 0x7f23540b5040 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f2368009fd0 tx=0x7f2368009500 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:51.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.116+0000 7f2378237740 1 -- start start 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 1 Processor -- start 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac59cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5a270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eb7a898790 con 0x55eb7ac51e50 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eb7ab5abc0 con 0x55eb7ac52770 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eb7ac5fc00 con 0x55eb7ac5d030 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.117+0000 7f2378237740 10 client.0 _ll_register_callbacks cb 0x55eb7ab38400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:51.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57754/0 (socket says 172.21.15.28:57754) 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 -- 172.21.15.28:0/1732874948 learned_addr learned my addr 172.21.15.28:0/1732874948 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f2367e00640 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac59cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236f200640 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5a270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 -- 172.21.15.28:0/1732874948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 msgr2=0x55eb7ac5a270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5a270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 -- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 msgr2=0x55eb7ac59cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac59cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 -- 172.21.15.28:0/1732874948 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55eb7a8c7210 con 0x55eb7ac51e50 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f236e800640 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c9e0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f2360009920 tx=0x7f23600a9430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=92176d42b7b13fc4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.118+0000 7f2353e00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:51.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2353e00640 1 -- 172.21.15.28:0/1732874948 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23600aa1e0 con 0x55eb7ac51e50 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2353e00640 1 -- 172.21.15.28:0/1732874948 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f23600a8d40 con 0x55eb7ac51e50 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f23580054f0 con 0x55eb7ac51e50 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2353e00640 1 -- 172.21.15.28:0/1732874948 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f23600a9ae0 con 0x55eb7ac51e50 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2378237740 10 client.15393 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55eb7a993620 con 0x55eb7ac51e50 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2353e00640 1 -- 172.21.15.28:0/1732874948 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23580054f0 con 0x55eb7ac51e50 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2353e00640 1 -- 172.21.15.28:0/1732874948 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f23600cc0d0 con 0x55eb7ac51e50 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2378237740 10 client.15393 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.119+0000 7f2378237740 10 client.15393 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.120+0000 7f2378237740 20 client.15393 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.120+0000 7f2378237740 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7ac6c950 0x55eb7ac6ed40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.120+0000 7f2378237740 4 client.15393 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:51.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.120+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55eb7abc4ae0 con 0x55eb7ac6c950 2024-05-20T20:06:51.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.120+0000 7f2367e00640 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7ac6c950 0x55eb7ac6ed40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.121+0000 7f2367e00640 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7ac6c950 0x55eb7ac6ed40 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bd5d94da203ba618 server_cookie=56de7f59777cfc13 in_seq=0 out_seq=0 2024-05-20T20:06:51.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.121+0000 7f2353e00640 10 client.15393 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:51.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:50 smithi033 ceph-mon[36179]: pgmap v39: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:06:51.129 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:50 smithi154 ceph-mon[36314]: pgmap v39: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:06:51.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.121+0000 7f2353e00640 1 -- 172.21.15.28:0/1732874948 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55eb7abc4ae0 con 0x55eb7ac6c950 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.121+0000 7f2353e00640 10 client.15393 handle_command_reply: tid=0 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.121+0000 7f2378237740 1 client.15393 shutdown 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 msgr2=0x55eb7ac5c9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c9e0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f2360009920 tx=0x7f23600a9430 comp rx=0 tx=0).stop 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 shutdown_connections 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7ac6c950 0x55eb7ac6ed40 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac59cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac5a270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2353e00640 0 client.15393 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 --2- 172.21.15.28:0/1732874948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c9e0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 >> 172.21.15.28:0/1732874948 conn(0x55eb7ac4e360 msgr2=0x55eb7ac540e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 shutdown_connections 2024-05-20T20:06:51.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 1 -- 172.21.15.28:0/1732874948 wait complete. 2024-05-20T20:06:51.131 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.122+0000 7f2378237740 20 client.15393 trim_cache size 0 max 16384 2024-05-20T20:06:51.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.139+0000 7f2378237740 1 Processor -- start 2024-05-20T20:06:51.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2378237740 1 -- start start 2024-05-20T20:06:51.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2378237740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2378237740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac64e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2378237740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac65390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2378237740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f23701b0730 con 0x55eb7ac51e50 2024-05-20T20:06:51.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2378237740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f23701b2270 con 0x55eb7ac52770 2024-05-20T20:06:51.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2378237740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f23701b0e00 con 0x55eb7ac5d030 2024-05-20T20:06:51.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57756/0 (socket says 172.21.15.28:57756) 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f236e800640 1 -- 172.21.15.28:0/157113376 learned_addr learned my addr 172.21.15.28:0/157113376 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f2367e00640 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac64e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.140+0000 7f236f200640 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac65390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f236e800640 1 -- 172.21.15.28:0/157113376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 msgr2=0x55eb7ac65390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f236e800640 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac65390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f236e800640 1 -- 172.21.15.28:0/157113376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 msgr2=0x55eb7ac64e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f236e800640 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac64e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f236e800640 1 -- 172.21.15.28:0/157113376 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55eb7ac52de0 con 0x55eb7ac51e50 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f2367e00640 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac64e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f236e800640 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c510 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f236000d010 tx=0x7f23600a59c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=baff5ac339451404 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f235ea00640 1 -- 172.21.15.28:0/157113376 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23600a9250 con 0x55eb7ac51e50 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f235ea00640 1 -- 172.21.15.28:0/157113376 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23600aa1e0 con 0x55eb7ac51e50 2024-05-20T20:06:51.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.141+0000 7f235ea00640 1 -- 172.21.15.28:0/157113376 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2360004000 con 0x55eb7ac51e50 2024-05-20T20:06:51.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 -- 172.21.15.28:0/157113376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 msgr2=0x55eb7ac5c510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c510 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f236000d010 tx=0x7f23600a59c0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 -- 172.21.15.28:0/157113376 shutdown_connections 2024-05-20T20:06:51.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac64e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac65390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 --2- 172.21.15.28:0/157113376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac5c510 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 -- 172.21.15.28:0/157113376 >> 172.21.15.28:0/157113376 conn(0x55eb7ac4e360 msgr2=0x55eb7ac67c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 -- 172.21.15.28:0/157113376 shutdown_connections 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.142+0000 7f2378237740 1 -- 172.21.15.28:0/157113376 wait complete. 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.143+0000 7f2378237740 1 -- start start 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.143+0000 7f2378237740 1 Processor -- start 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2378237740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac62d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2378237740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac632c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2378237740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac63800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2378237740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f23701b1ba0 con 0x55eb7ac51e50 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2378237740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f23701b14d0 con 0x55eb7ac52770 2024-05-20T20:06:51.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2378237740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f23701ae710 con 0x55eb7ac5d030 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2378237740 10 client.0 _ll_register_callbacks cb 0x7f23600a5a70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac62d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac62d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57772/0 (socket says 172.21.15.28:57772) 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 -- 172.21.15.28:0/725901182 learned_addr learned my addr 172.21.15.28:0/725901182 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236f200640 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac63800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f2367e00640 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac632c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 -- 172.21.15.28:0/725901182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 msgr2=0x55eb7ac63800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac63800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 -- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 msgr2=0x55eb7ac632c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac632c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.144+0000 7f236e800640 1 -- 172.21.15.28:0/725901182 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55eb7acd66c0 con 0x55eb7ac51e50 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f236e800640 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac62d80 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f236000ea30 tx=0x7f23600a8ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=266bae4e3552f741 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2353e00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2353e00640 1 -- 172.21.15.28:0/725901182 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23600a9250 con 0x55eb7ac51e50 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2353e00640 1 -- 172.21.15.28:0/725901182 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f23600a60d0 con 0x55eb7ac51e50 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f23600aa1e0 con 0x55eb7ac51e50 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2353e00640 1 -- 172.21.15.28:0/725901182 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f23600ade90 con 0x55eb7ac51e50 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2378237740 10 client.15399 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55eb7acd66c0 con 0x55eb7ac51e50 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.145+0000 7f2353e00640 1 -- 172.21.15.28:0/725901182 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23600aa1e0 con 0x55eb7ac51e50 2024-05-20T20:06:51.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2353e00640 1 -- 172.21.15.28:0/725901182 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f23600e0260 con 0x55eb7ac51e50 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2378237740 10 client.15399 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2378237740 10 client.15399 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2378237740 20 client.15399 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2378237740 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7acd7be0 0x55eb7acd9fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2378237740 4 client.15399 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55eb7abbe2c0 con 0x55eb7acd7be0 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.146+0000 7f2367e00640 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7acd7be0 0x55eb7acd9fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.147+0000 7f2367e00640 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7acd7be0 0x55eb7acd9fd0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1e8f63723556b799 server_cookie=fc3775754926e950 in_seq=0 out_seq=0 2024-05-20T20:06:51.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.147+0000 7f2353e00640 10 client.15399 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:51.152 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.151+0000 7f2353e00640 1 -- 172.21.15.28:0/725901182 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55eb7abbe2c0 con 0x55eb7acd7be0 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.151+0000 7f2353e00640 10 client.15399 handle_command_reply: tid=0 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.151+0000 7f2378237740 1 client.15399 shutdown 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.151+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 msgr2=0x55eb7ac62d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.151+0000 7f2378237740 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac62d80 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f236000ea30 tx=0x7f23600a8ff0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 shutdown_connections 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb7acd7be0 0x55eb7acd9fd0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb7ac52770 0x55eb7ac632c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb7ac5d030 0x55eb7ac63800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 --2- 172.21.15.28:0/725901182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb7ac51e50 0x55eb7ac62d80 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 >> 172.21.15.28:0/725901182 conn(0x55eb7ac4e360 msgr2=0x55eb7ac684e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2353e00640 0 client.15399 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:51.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 shutdown_connections 2024-05-20T20:06:51.154 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 1 -- 172.21.15.28:0/725901182 wait complete. 2024-05-20T20:06:51.154 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.152+0000 7f2378237740 20 client.15399 trim_cache size 0 max 16384 2024-05-20T20:06:51.154 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:06:51.154 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:06:51.154 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23540b2b80 msgr2=0x7f23540b5040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23540b2b80 0x7f23540b5040 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f2368009fd0 tx=0x7f2368009500 comp rx=0 tx=0).stop 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2370104a90 msgr2=0x7f237019b2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2370104a90 0x7f237019b2b0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f236000ea60 tx=0x7f236000ef20 comp rx=0 tx=0).stop 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 shutdown_connections 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f237010aaa0 0x7f237019fb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23540b2b80 0x7f23540b5040 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2370105450 0x7f237019b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 --2- 172.21.15.28:0/212251547 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2370104a90 0x7f237019b2b0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:51.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 >> 172.21.15.28:0/212251547 conn(0x7f2370100560 msgr2=0x7f2370100da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:51.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 shutdown_connections 2024-05-20T20:06:51.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:06:51.153+0000 7f2374a00640 1 -- 172.21.15.28:0/212251547 wait complete. 2024-05-20T20:06:51.870 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:52.996 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:52 smithi028 ceph-mon[31178]: pgmap v40: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:06:53.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:52 smithi154 ceph-mon[36314]: pgmap v40: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:06:53.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:52 smithi033 ceph-mon[36179]: pgmap v40: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:06:53.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.176+0000 7f0bb9400700 1 -- 172.21.15.28:0/3429772989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4109640 msgr2=0x7f0bb4109a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.176+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3429772989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4109640 0x7f0bb4109a20 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f0ba40099c0 tx=0x7f0ba4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.177+0000 7f0bb9400700 1 -- 172.21.15.28:0/3429772989 shutdown_connections 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.177+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3429772989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4103f50 0x7f0bb4110410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.177+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3429772989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4109640 0x7f0bb4109a20 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.177+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3429772989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0bb4103590 0x7f0bb4103a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.177+0000 7f0bb9400700 1 -- 172.21.15.28:0/3429772989 >> 172.21.15.28:0/3429772989 conn(0x7f0bb407a6f0 msgr2=0x7f0bb407ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.177+0000 7f0bb9400700 1 -- 172.21.15.28:0/3429772989 shutdown_connections 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.177+0000 7f0bb9400700 1 -- 172.21.15.28:0/3429772989 wait complete. 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 Processor -- start 2024-05-20T20:06:53.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 -- start start 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0bb4103590 0x7f0bb419be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4103f50 0x7f0bb419c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4109640 0x7f0bb41a06e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0bb4113760 con 0x7f0bb4109640 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0bb41135e0 con 0x7f0bb4103f50 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.178+0000 7f0bb9400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0bb41138e0 con 0x7f0bb4103590 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4109640 0x7f0bb41a06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4109640 0x7f0bb41a06e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57784/0 (socket says 172.21.15.28:57784) 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 -- 172.21.15.28:0/3655245969 learned_addr learned my addr 172.21.15.28:0/3655245969 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4103f50 0x7f0bb419c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb2800700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0bb4103590 0x7f0bb419be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 -- 172.21.15.28:0/3655245969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0bb4103590 msgr2=0x7f0bb419be10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0bb4103590 0x7f0bb419be10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 -- 172.21.15.28:0/3655245969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4103f50 msgr2=0x7f0bb419c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4103f50 0x7f0bb419c350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 -- 172.21.15.28:0/3655245969 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0bb41a0e20 con 0x7f0bb4109640 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.179+0000 7f0bb3200700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4109640 0x7f0bb41a06e0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f0bac00c0f0 tx=0x7f0bac00c400 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.180+0000 7f0bab400700 1 -- 172.21.15.28:0/3655245969 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bac018020 con 0x7f0bb4109640 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.180+0000 7f0bab400700 1 -- 172.21.15.28:0/3655245969 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0bac00ca00 con 0x7f0bb4109640 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.180+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0bb41a1110 con 0x7f0bb4109640 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.180+0000 7f0bab400700 1 -- 172.21.15.28:0/3655245969 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bac004670 con 0x7f0bb4109640 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.180+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0bb41a8990 con 0x7f0bb4109640 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.184+0000 7f0bab400700 1 -- 172.21.15.28:0/3655245969 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f0bac012050 con 0x7f0bb4109640 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 Processor -- start 2024-05-20T20:06:53.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 -- start start 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bab400700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b980b2c20 0x7f0b980b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ca9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046c4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c5570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555804622df0 con 0x5558046c50f0 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555804612990 con 0x5558046ca5c0 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.185+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5558046169d0 con 0x5558046caee0 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bab400700 1 -- 172.21.15.28:0/3655245969 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0bac0e2330 con 0x7f0bb4109640 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b980b2c20 0x7f0b980b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ca9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb3200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c5570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ca9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57524/0 (socket says 172.21.15.28:57524) 2024-05-20T20:06:53.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 -- 172.21.15.28:0/3939082532 learned_addr learned my addr 172.21.15.28:0/3939082532 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046c4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 -- 172.21.15.28:0/3939082532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 msgr2=0x5558046c4ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046c4ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 -- 172.21.15.28:0/3939082532 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 msgr2=0x5558046c5570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c5570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb2800700 1 -- 172.21.15.28:0/3939082532 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5558045db3d0 con 0x5558046ca5c0 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046c4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.186+0000 7f0bb3200700 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c5570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.187+0000 7f0bb2800700 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ca9a0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f0ba4015000 tx=0x7f0ba4003680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=547d4d5d8ad092ca server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.187+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b980b2c20 0x7f0b980b50e0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f0ba0009fd0 tx=0x7f0ba00099b0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.187+0000 7f0b9fe00700 1 -- 172.21.15.28:0/3939082532 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ba4028070 con 0x5558046ca5c0 2024-05-20T20:06:53.198 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.187+0000 7f0b9fe00700 1 -- 172.21.15.28:0/3939082532 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0ba40037c0 con 0x5558046ca5c0 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.187+0000 7f0b9fe00700 1 -- 172.21.15.28:0/3939082532 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ba400b9f0 con 0x5558046ca5c0 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/3939082532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 msgr2=0x5558046ca9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ca9a0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f0ba4015000 tx=0x7f0ba4003680 comp rx=0 tx=0).stop 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/3939082532 shutdown_connections 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c5570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ca9a0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/3939082532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046c4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/3939082532 >> 172.21.15.28:0/3939082532 conn(0x5558045f2b90 msgr2=0x5558046bf640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/3939082532 shutdown_connections 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.188+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/3939082532 wait complete. 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.189+0000 7f0bc3e38b80 1 -- start start 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.189+0000 7f0bc3e38b80 1 Processor -- start 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046d0bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046cdf90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046ce4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555804518f50 con 0x5558046c50f0 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5558045b3710 con 0x5558046ca5c0 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5558046d1930 con 0x5558046caee0 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bc3e38b80 10 client.0 _ll_register_callbacks cb 0x5558045bcc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046ce4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb2800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046d0bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046ce4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55344/0 (socket says 172.21.15.28:55344) 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 -- 172.21.15.28:0/1962522921 learned_addr learned my addr 172.21.15.28:0/1962522921 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046cdf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 -- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 msgr2=0x5558046cdf90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046cdf90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 -- 172.21.15.28:0/1962522921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 msgr2=0x5558046d0bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046d0bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb3200700 1 -- 172.21.15.28:0/1962522921 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555804562880 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.190+0000 7f0bb2800700 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046d0bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.191+0000 7f0bb3200700 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046ce4d0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f0bac0a9400 tx=0x7f0bac021000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b7a47be3c0f44578 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.191+0000 7f0b9cc00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.191+0000 7f0b9cc00700 1 -- 172.21.15.28:0/1962522921 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bac00a040 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.191+0000 7f0b9cc00700 1 -- 172.21.15.28:0/1962522921 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0bac0a96d0 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.191+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0ba40037c0 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.191+0000 7f0b9cc00700 1 -- 172.21.15.28:0/1962522921 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0bac01f0b0 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.191+0000 7f0bc3e38b80 10 client.25525 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5558045e3010 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0b9cc00700 1 -- 172.21.15.28:0/1962522921 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0ba40037c0 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0b9cc00700 1 -- 172.21.15.28:0/1962522921 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0bac0b1db0 con 0x5558046caee0 2024-05-20T20:06:53.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0bc3e38b80 10 client.25525 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:53.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0bc3e38b80 10 client.25525 resolve_mds: resolved name 'h' to daemon mds.h 2024-05-20T20:06:53.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0bc3e38b80 20 client.25525 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:53.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5558046de4d0 0x5558046df940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0bc3e38b80 4 client.25525 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:53.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.192+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555804624d40 con 0x5558046de4d0 2024-05-20T20:06:53.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.193+0000 7f0bb2800700 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5558046de4d0 0x5558046df940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.193+0000 7f0bb2800700 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5558046de4d0 0x5558046df940 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ad4e7732bfb00b0e server_cookie=65351b86ee71ba15 in_seq=0 out_seq=0 2024-05-20T20:06:53.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.193+0000 7f0b9cc00700 10 client.25525 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:06:53.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.194+0000 7f0b9cc00700 1 -- 172.21.15.28:0/1962522921 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x555804624d40 con 0x5558046de4d0 2024-05-20T20:06:53.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.194+0000 7f0b9cc00700 10 client.25525 handle_command_reply: tid=0 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 client.25525 shutdown 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 msgr2=0x5558046ce4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046ce4d0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f0bac0a9400 tx=0x7f0bac021000 comp rx=0 tx=0).stop 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 shutdown_connections 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5558046de4d0 0x5558046df940 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046ce4d0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046d0bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1962522921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046cdf90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 >> 172.21.15.28:0/1962522921 conn(0x5558045f2b90 msgr2=0x5558046bed20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0b9cc00700 0 client.25525 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 shutdown_connections 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1962522921 wait complete. 2024-05-20T20:06:53.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.195+0000 7f0bc3e38b80 20 client.25525 trim_cache size 0 max 16384 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.219+0000 7f0bc3e38b80 1 Processor -- start 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bc3e38b80 1 -- start start 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ce480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046bf4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5558046ddc80 con 0x5558046c50f0 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5558046de350 con 0x5558046ca5c0 2024-05-20T20:06:53.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5558046dea20 con 0x5558046caee0 2024-05-20T20:06:53.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bb2800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bb2800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c4ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57820/0 (socket says 172.21.15.28:57820) 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bb2800700 1 -- 172.21.15.28:0/1654693660 learned_addr learned my addr 172.21.15.28:0/1654693660 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ce480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.220+0000 7f0bb3200700 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046bf4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.221+0000 7f0bb1e00700 1 -- 172.21.15.28:0/1654693660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 msgr2=0x5558046bf4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.221+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046bf4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.221+0000 7f0bb1e00700 1 -- 172.21.15.28:0/1654693660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 msgr2=0x5558046c4ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.221+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c4ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.221+0000 7f0bb1e00700 1 -- 172.21.15.28:0/1654693660 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5558046cc500 con 0x5558046ca5c0 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.221+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ce480 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f0ba00097a0 tx=0x7f0ba00047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=33867b9fb6235467 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0b9f400700 1 -- 172.21.15.28:0/1654693660 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ba0004a30 con 0x5558046ca5c0 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0b9f400700 1 -- 172.21.15.28:0/1654693660 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0ba0012c60 con 0x5558046ca5c0 2024-05-20T20:06:53.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0b9f400700 1 -- 172.21.15.28:0/1654693660 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ba0005540 con 0x5558046ca5c0 2024-05-20T20:06:53.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1654693660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 msgr2=0x5558046ce480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ce480 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f0ba00097a0 tx=0x7f0ba00047c0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1654693660 shutdown_connections 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046caee0 0x5558046bf4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046c50f0 0x5558046c4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/1654693660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046ca5c0 0x5558046ce480 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.222+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1654693660 >> 172.21.15.28:0/1654693660 conn(0x5558045f2b90 msgr2=0x5558046cdd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.223+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1654693660 shutdown_connections 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.223+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/1654693660 wait complete. 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.223+0000 7f0bc3e38b80 1 -- start start 2024-05-20T20:06:53.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.223+0000 7f0bc3e38b80 1 Processor -- start 2024-05-20T20:06:53.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046c50f0 0x5558046e93d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046ca5c0 0x5558046e3d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bc3e38b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046caee0 0x5558046e42b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5558046df7c0 con 0x5558046caee0 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5558046df0f0 con 0x5558046c50f0 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bc3e38b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5558046dfe90 con 0x5558046ca5c0 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bc3e38b80 10 client.0 _ll_register_callbacks cb 0x5558045bcc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bb2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046c50f0 0x5558046e93d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bb2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046c50f0 0x5558046e93d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57554/0 (socket says 172.21.15.28:57554) 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bb2800700 1 -- 172.21.15.28:0/162472459 learned_addr learned my addr 172.21.15.28:0/162472459 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bb1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046ca5c0 0x5558046e3d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.224+0000 7f0bb3200700 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046caee0 0x5558046e42b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0bb2800700 1 -- 172.21.15.28:0/162472459 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046ca5c0 msgr2=0x5558046e3d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0bb2800700 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046ca5c0 0x5558046e3d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0bb2800700 1 -- 172.21.15.28:0/162472459 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046caee0 msgr2=0x5558046e42b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:53.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0bb2800700 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046caee0 0x5558046e42b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0bb2800700 1 -- 172.21.15.28:0/162472459 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5558046e4ac0 con 0x5558046c50f0 2024-05-20T20:06:53.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0bb2800700 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046c50f0 0x5558046e93d0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f0ba4015000 tx=0x7f0ba400b6c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b0e1afcb9e34b5f3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:53.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0b9cc00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:53.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.225+0000 7f0b9cc00700 1 -- 172.21.15.28:0/162472459 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ba4028040 con 0x5558046c50f0 2024-05-20T20:06:53.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.226+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0ba0012c60 con 0x5558046c50f0 2024-05-20T20:06:53.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.226+0000 7f0b9cc00700 1 -- 172.21.15.28:0/162472459 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0ba40052b0 con 0x5558046c50f0 2024-05-20T20:06:53.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.226+0000 7f0b9cc00700 1 -- 172.21.15.28:0/162472459 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0ba4040050 con 0x5558046c50f0 2024-05-20T20:06:53.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.226+0000 7f0bc3e38b80 10 client.25562 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:53.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.226+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55580474ac40 con 0x5558046c50f0 2024-05-20T20:06:53.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.226+0000 7f0b9cc00700 1 -- 172.21.15.28:0/162472459 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0ba0012c60 con 0x5558046c50f0 2024-05-20T20:06:53.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0b9cc00700 1 -- 172.21.15.28:0/162472459 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0ba4037070 con 0x5558046c50f0 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0bc3e38b80 10 client.25562 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0bc3e38b80 10 client.25562 resolve_mds: resolved name 'h' to daemon mds.h 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0bc3e38b80 20 client.25562 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55580474f290 0x55580474dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0bc3e38b80 4 client.25562 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "status"}) -- 0x55580461e520 con 0x55580474f290 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.227+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55580474f290 0x55580474dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bb1e00700 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55580474f290 0x55580474dfa0 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=5365dd2fbba2dc05 server_cookie=9ce44c5cad14165c in_seq=0 out_seq=0 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0b9cc00700 10 client.25562 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0b9cc00700 1 -- 172.21.15.28:0/162472459 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x55580461e520 con 0x55580474f290 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0b9cc00700 10 client.25562 handle_command_reply: tid=0 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 client.25562 shutdown 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046c50f0 msgr2=0x5558046e93d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046c50f0 0x5558046e93d0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f0ba4015000 tx=0x7f0ba400b6c0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 shutdown_connections 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55580474f290 0x55580474dfa0 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5558046caee0 0x5558046e42b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5558046c50f0 0x5558046e93d0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0b9cc00700 0 client.25562 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 --2- 172.21.15.28:0/162472459 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5558046ca5c0 0x5558046e3d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.228+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 >> 172.21.15.28:0/162472459 conn(0x5558045f2b90 msgr2=0x5558046cb2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.229+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 shutdown_connections 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.229+0000 7f0bc3e38b80 1 -- 172.21.15.28:0/162472459 wait complete. 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.229+0000 7f0bc3e38b80 20 client.25562 trim_cache size 0 max 16384 2024-05-20T20:06:53.232 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 1, 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24500, 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 310.62184961600002, 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 29, 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 81, 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 80, 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 365.81990038200001 2024-05-20T20:06:53.233 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:06:53.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b980b2c20 msgr2=0x7f0b980b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b980b2c20 0x7f0b980b50e0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f0ba0009fd0 tx=0x7f0ba00099b0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4109640 msgr2=0x7f0bb41a06e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:53.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4109640 0x7f0bb41a06e0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f0bac00c0f0 tx=0x7f0bac00c400 comp rx=0 tx=0).stop 2024-05-20T20:06:53.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 shutdown_connections 2024-05-20T20:06:53.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0bb4109640 0x7f0bb41a06e0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0bb4103590 0x7f0bb419be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b980b2c20 0x7f0b980b50e0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 --2- 172.21.15.28:0/3655245969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0bb4103f50 0x7f0bb419c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:53.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 >> 172.21.15.28:0/3655245969 conn(0x7f0bb407a6f0 msgr2=0x7f0bb4075770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:53.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 shutdown_connections 2024-05-20T20:06:53.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:53.230+0000 7f0bb9400700 1 -- 172.21.15.28:0/3655245969 wait complete. 2024-05-20T20:06:54.624 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.b status 2024-05-20T20:06:55.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:54 smithi154 ceph-mon[36314]: pgmap v41: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:06:55.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:54 smithi033 ceph-mon[36179]: pgmap v41: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:06:55.174 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:54 smithi028 ceph-mon[31178]: pgmap v41: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:06:55.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.243+0000 7fd100800700 1 -- 172.21.15.28:0/4131383031 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8075770 msgr2=0x7fd0f8075b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.243+0000 7fd100800700 1 --2- 172.21.15.28:0/4131383031 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8075770 0x7fd0f8075b50 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec009990 tx=0x7fd0ec009ca0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.243+0000 7fd100800700 1 -- 172.21.15.28:0/4131383031 shutdown_connections 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.243+0000 7fd100800700 1 --2- 172.21.15.28:0/4131383031 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0f8114710 0x7fd0f8116b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.243+0000 7fd100800700 1 --2- 172.21.15.28:0/4131383031 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8075770 0x7fd0f8075b50 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.243+0000 7fd100800700 1 --2- 172.21.15.28:0/4131383031 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8076090 0x7fd0f81141d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.243+0000 7fd100800700 1 -- 172.21.15.28:0/4131383031 >> 172.21.15.28:0/4131383031 conn(0x7fd0f807a6a0 msgr2=0x7fd0f807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.244+0000 7fd100800700 1 -- 172.21.15.28:0/4131383031 shutdown_connections 2024-05-20T20:06:55.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.244+0000 7fd100800700 1 -- 172.21.15.28:0/4131383031 wait complete. 2024-05-20T20:06:55.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.245+0000 7fd100800700 1 Processor -- start 2024-05-20T20:06:55.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.245+0000 7fd100800700 1 -- start start 2024-05-20T20:06:55.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd100800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8075770 0x7fd0f81a02f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd100800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8076090 0x7fd0f81a0830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd100800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0f8114710 0x7fd0f81a4bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd100800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f8119d60 con 0x7fd0f8114710 2024-05-20T20:06:55.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd100800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f8119be0 con 0x7fd0f8076090 2024-05-20T20:06:55.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd100800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f8119ee0 con 0x7fd0f8075770 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8075770 0x7fd0f81a02f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd0fd400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8076090 0x7fd0f81a0830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8075770 0x7fd0f81a02f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55380/0 (socket says 172.21.15.28:55380) 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd0fde00700 1 -- 172.21.15.28:0/4114080712 learned_addr learned my addr 172.21.15.28:0/4114080712 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.246+0000 7fd0fe800700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0f8114710 0x7fd0f81a4bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0fd400700 1 -- 172.21.15.28:0/4114080712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8075770 msgr2=0x7fd0f81a02f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0fd400700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8075770 0x7fd0f81a02f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0fd400700 1 -- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0f8114710 msgr2=0x7fd0f81a4bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0fd400700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0f8114710 0x7fd0f81a4bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0fd400700 1 -- 172.21.15.28:0/4114080712 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd0f81a5300 con 0x7fd0f8076090 2024-05-20T20:06:55.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0fde00700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8075770 0x7fd0f81a02f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:55.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0fd400700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8076090 0x7fd0f81a0830 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fd0e800e9f0 tx=0x7fd0e800ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:55.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0f2a00700 1 -- 172.21.15.28:0/4114080712 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0e8004220 con 0x7fd0f8076090 2024-05-20T20:06:55.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0f2a00700 1 -- 172.21.15.28:0/4114080712 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd0e80043c0 con 0x7fd0f8076090 2024-05-20T20:06:55.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd0f2a00700 1 -- 172.21.15.28:0/4114080712 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0e800ce60 con 0x7fd0f8076090 2024-05-20T20:06:55.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.247+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd0f81a55f0 con 0x7fd0f8076090 2024-05-20T20:06:55.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.248+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd0f81ace70 con 0x7fd0f8076090 2024-05-20T20:06:55.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd0f2a00700 1 -- 172.21.15.28:0/4114080712 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7fd0e8005290 con 0x7fd0f8076090 2024-05-20T20:06:55.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 Processor -- start 2024-05-20T20:06:55.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 -- start start 2024-05-20T20:06:55.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b214290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b20e260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20ece0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563b4b16ce40 con 0x563b4b213eb0 2024-05-20T20:06:55.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563b4b15c9e0 con 0x563b4b2147d0 2024-05-20T20:06:55.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.253+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563b4b160a20 con 0x563b4b20e880 2024-05-20T20:06:55.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b214290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fd400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20ece0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b214290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57868/0 (socket says 172.21.15.28:57868) 2024-05-20T20:06:55.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 -- 172.21.15.28:0/402209322 learned_addr learned my addr 172.21.15.28:0/402209322 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:55.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b20e260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 -- 172.21.15.28:0/402209322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 msgr2=0x563b4b20ece0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20ece0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 -- 172.21.15.28:0/402209322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 msgr2=0x563b4b20e260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b20e260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fe800700 1 -- 172.21.15.28:0/402209322 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563b4b125420 con 0x563b4b213eb0 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fd400700 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20ece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.254+0000 7fd0fde00700 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b20e260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.255+0000 7fd0fe800700 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b214290 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fd0f4011600 tx=0x7fd0f4009920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ecc24bc55cf3bc95 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.255+0000 7fd0f2a00700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd0e00b2ba0 0x7fd0e00b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.255+0000 7fd0e7400700 1 -- 172.21.15.28:0/402209322 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0f4003a00 con 0x563b4b213eb0 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.255+0000 7fd0f2a00700 1 -- 172.21.15.28:0/4114080712 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd0e80e2a00 con 0x7fd0f8076090 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.255+0000 7fd0e7400700 1 -- 172.21.15.28:0/402209322 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd0f401a070 con 0x563b4b213eb0 2024-05-20T20:06:55.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.255+0000 7fd0e7400700 1 -- 172.21.15.28:0/402209322 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0f4007760 con 0x563b4b213eb0 2024-05-20T20:06:55.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.255+0000 7fd0fde00700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd0e00b2ba0 0x7fd0e00b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd10b24bb80 1 -- 172.21.15.28:0/402209322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 msgr2=0x563b4b214290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b214290 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fd0f4011600 tx=0x7fd0f4009920 comp rx=0 tx=0).stop 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd0fde00700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd0e00b2ba0 0x7fd0e00b5060 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec006010 tx=0x7fd0ec00b280 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd10b24bb80 1 -- 172.21.15.28:0/402209322 shutdown_connections 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20ece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b214290 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/402209322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b20e260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.256+0000 7fd10b24bb80 1 -- 172.21.15.28:0/402209322 >> 172.21.15.28:0/402209322 conn(0x563b4b13cbe0 msgr2=0x563b4b209010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.257+0000 7fd10b24bb80 1 -- 172.21.15.28:0/402209322 shutdown_connections 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.257+0000 7fd10b24bb80 1 -- 172.21.15.28:0/402209322 wait complete. 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.257+0000 7fd10b24bb80 1 -- start start 2024-05-20T20:06:55.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 1 Processor -- start 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b21a440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b217830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b217d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563b4b062fa0 con 0x563b4b213eb0 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563b4b0fd760 con 0x563b4b2147d0 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563b4b21b240 con 0x563b4b20e880 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.258+0000 7fd10b24bb80 10 client.0 _ll_register_callbacks cb 0x563b4b106c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:55.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fd400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b217830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b21a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fd400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b217830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57880/0 (socket says 172.21.15.28:57880) 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fd400700 1 -- 172.21.15.28:0/3926556221 learned_addr learned my addr 172.21.15.28:0/3926556221 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fe800700 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b217d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fde00700 1 -- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 msgr2=0x563b4b217d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fde00700 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b217d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fde00700 1 -- 172.21.15.28:0/3926556221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 msgr2=0x563b4b217830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fde00700 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b217830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fde00700 1 -- 172.21.15.28:0/3926556221 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563b4b0ac8d0 con 0x563b4b20e880 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.259+0000 7fd0fd400700 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b217830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd0fde00700 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b21a440 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec000c00 tx=0x7fd0ec009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=439d52190c7dd22e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:55.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd0dbe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:06:55.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd0dbe00700 1 -- 172.21.15.28:0/3926556221 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0ec034060 con 0x563b4b20e880 2024-05-20T20:06:55.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd0dbe00700 1 -- 172.21.15.28:0/3926556221 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd0ec00f460 con 0x563b4b20e880 2024-05-20T20:06:55.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd0f401a070 con 0x563b4b20e880 2024-05-20T20:06:55.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd0dbe00700 1 -- 172.21.15.28:0/3926556221 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd0ec02b040 con 0x563b4b20e880 2024-05-20T20:06:55.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd10b24bb80 10 client.25543 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:55.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.260+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563b4b12d060 con 0x563b4b20e880 2024-05-20T20:06:55.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.261+0000 7fd0dbe00700 1 -- 172.21.15.28:0/3926556221 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd0f401a070 con 0x563b4b20e880 2024-05-20T20:06:55.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.261+0000 7fd0dbe00700 1 -- 172.21.15.28:0/3926556221 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd0ec0246a0 con 0x563b4b20e880 2024-05-20T20:06:55.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.262+0000 7fd10b24bb80 10 client.25543 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.262+0000 7fd10b24bb80 10 client.25543 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.262+0000 7fd10b24bb80 20 client.25543 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.262+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b227d20 0x563b4b22a110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.262+0000 7fd10b24bb80 4 client.25543 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.262+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563b4b16ed90 con 0x563b4b227d20 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.262+0000 7fd0fd400700 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b227d20 0x563b4b22a110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.263+0000 7fd0fd400700 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b227d20 0x563b4b22a110 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=5ae28124ad90a785 server_cookie=30642338baf8ccc4 in_seq=0 out_seq=0 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.263+0000 7fd0dbe00700 10 client.25543 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd0dbe00700 1 -- 172.21.15.28:0/3926556221 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x563b4b16ed90 con 0x563b4b227d20 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd0dbe00700 10 client.25543 handle_command_reply: tid=0 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 client.25543 shutdown 2024-05-20T20:06:55.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 msgr2=0x563b4b21a440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b21a440 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec000c00 tx=0x7fd0ec009310 comp rx=0 tx=0).stop 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 shutdown_connections 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b227d20 0x563b4b22a110 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b217d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b21a440 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3926556221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b217830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.264+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 >> 172.21.15.28:0/3926556221 conn(0x563b4b13cbe0 msgr2=0x563b4b20d250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.265+0000 7fd0dbe00700 0 client.25543 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.265+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 shutdown_connections 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.265+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3926556221 wait complete. 2024-05-20T20:06:55.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.265+0000 7fd10b24bb80 20 client.25543 trim_cache size 0 max 16384 2024-05-20T20:06:55.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.283+0000 7fd10b24bb80 1 Processor -- start 2024-05-20T20:06:55.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.283+0000 7fd10b24bb80 1 -- start start 2024-05-20T20:06:55.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20e260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b21a440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b211950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f81b4470 con 0x563b4b213eb0 2024-05-20T20:06:55.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f81b5fb0 con 0x563b4b2147d0 2024-05-20T20:06:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f81b4b40 con 0x563b4b20e880 2024-05-20T20:06:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd0fd400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b21a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20e260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd0fd400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b21a440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57890/0 (socket says 172.21.15.28:57890) 2024-05-20T20:06:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20e260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55398/0 (socket says 172.21.15.28:55398) 2024-05-20T20:06:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd0fd400700 1 -- 172.21.15.28:0/3094039723 learned_addr learned my addr 172.21.15.28:0/3094039723 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.284+0000 7fd0fe800700 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b211950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0fde00700 1 -- 172.21.15.28:0/3094039723 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 msgr2=0x563b4b211950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0fde00700 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b211950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0fde00700 1 -- 172.21.15.28:0/3094039723 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 msgr2=0x563b4b21a440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0fde00700 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b21a440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0fde00700 1 -- 172.21.15.28:0/3094039723 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563b4b215da0 con 0x563b4b20e880 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0fde00700 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20e260 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec024b10 tx=0x7fd0ec00bad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7a432bfbb6ee7080 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0e6a00700 1 -- 172.21.15.28:0/3094039723 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0ec034070 con 0x563b4b20e880 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0e6a00700 1 -- 172.21.15.28:0/3094039723 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd0ec024640 con 0x563b4b20e880 2024-05-20T20:06:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.285+0000 7fd0e6a00700 1 -- 172.21.15.28:0/3094039723 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0ec0176a0 con 0x563b4b20e880 2024-05-20T20:06:55.288 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.286+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3094039723 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 msgr2=0x563b4b20e260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.288 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.286+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20e260 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec024b10 tx=0x7fd0ec00bad0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.288 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.286+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3094039723 shutdown_connections 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.286+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b2147d0 0x563b4b211950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.286+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b20e260 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.286+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/3094039723 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b213eb0 0x563b4b21a440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.286+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3094039723 >> 172.21.15.28:0/3094039723 conn(0x563b4b13cbe0 msgr2=0x563b4b202680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.287+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3094039723 shutdown_connections 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.287+0000 7fd10b24bb80 1 -- 172.21.15.28:0/3094039723 wait complete. 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.287+0000 7fd10b24bb80 1 -- start start 2024-05-20T20:06:55.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.287+0000 7fd10b24bb80 1 Processor -- start 2024-05-20T20:06:55.290 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:55.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b2204d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b213eb0 0x563b4b220a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd10b24bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b2147d0 0x563b4b22e9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f81b58e0 con 0x563b4b2147d0 2024-05-20T20:06:55.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f81b5210 con 0x563b4b213eb0 2024-05-20T20:06:55.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd10b24bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd0f81b2410 con 0x563b4b20e880 2024-05-20T20:06:55.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd10b24bb80 10 client.0 _ll_register_callbacks cb 0x563b4b106c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b2204d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd0fd400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b213eb0 0x563b4b220a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd0fde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b2204d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55410/0 (socket says 172.21.15.28:55410) 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd0fde00700 1 -- 172.21.15.28:0/1009291151 learned_addr learned my addr 172.21.15.28:0/1009291151 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.288+0000 7fd0fe800700 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b2147d0 0x563b4b22e9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0fd400700 1 -- 172.21.15.28:0/1009291151 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 msgr2=0x563b4b2204d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0fd400700 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b2204d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0fd400700 1 -- 172.21.15.28:0/1009291151 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b2147d0 msgr2=0x563b4b22e9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0fd400700 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b2147d0 0x563b4b22e9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0fd400700 1 -- 172.21.15.28:0/1009291151 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563b4b22ef10 con 0x563b4b213eb0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0fd400700 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b213eb0 0x563b4b220a10 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fd0e808ffd0 tx=0x7fd0e80e4e90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ebfd08aeb29b9f6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0dbe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0dbe00700 1 -- 172.21.15.28:0/1009291151 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd0e808ae00 con 0x563b4b213eb0 2024-05-20T20:06:55.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0dbe00700 1 -- 172.21.15.28:0/1009291151 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd0e80a60a0 con 0x563b4b213eb0 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd0ec024640 con 0x563b4b213eb0 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd0dbe00700 1 -- 172.21.15.28:0/1009291151 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd0e801c040 con 0x563b4b213eb0 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd10b24bb80 10 client.25580 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.289+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563b4b291f10 con 0x563b4b213eb0 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd0dbe00700 1 -- 172.21.15.28:0/1009291151 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd0ec024640 con 0x563b4b213eb0 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd0dbe00700 1 -- 172.21.15.28:0/1009291151 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd0e80cc0b0 con 0x563b4b213eb0 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd10b24bb80 10 client.25580 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd10b24bb80 10 client.25580 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd10b24bb80 20 client.25580 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b292580 0x563b4b2949a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd10b24bb80 4 client.25580 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:06:55.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.290+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x563b4b168570 con 0x563b4b292580 2024-05-20T20:06:55.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.291+0000 7fd0fde00700 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b292580 0x563b4b2949a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:55.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.291+0000 7fd0fde00700 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b292580 0x563b4b2949a0 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=b0ad558dc4e52777 server_cookie=6f3788a890fd7488 in_seq=0 out_seq=0 2024-05-20T20:06:55.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.291+0000 7fd0dbe00700 10 client.25580 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:55.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.291+0000 7fd0dbe00700 1 -- 172.21.15.28:0/1009291151 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x563b4b168570 con 0x563b4b292580 2024-05-20T20:06:55.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.291+0000 7fd0dbe00700 10 client.25580 handle_command_reply: tid=0 2024-05-20T20:06:55.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 client.25580 shutdown 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b213eb0 msgr2=0x563b4b220a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b213eb0 0x563b4b220a10 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fd0e808ffd0 tx=0x7fd0e80e4e90 comp rx=0 tx=0).stop 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 shutdown_connections 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x563b4b292580 0x563b4b2949a0 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563b4b2147d0 0x563b4b22e9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563b4b20e880 0x563b4b2204d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 --2- 172.21.15.28:0/1009291151 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563b4b213eb0 0x563b4b220a10 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 >> 172.21.15.28:0/1009291151 conn(0x563b4b13cbe0 msgr2=0x563b4b211950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd0dbe00700 0 client.25580 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 shutdown_connections 2024-05-20T20:06:55.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.292+0000 7fd10b24bb80 1 -- 172.21.15.28:0/1009291151 wait complete. 2024-05-20T20:06:55.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.294+0000 7fd10b24bb80 20 client.25580 trim_cache size 0 max 16384 2024-05-20T20:06:55.296 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:06:55.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.294+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd0e00b2ba0 msgr2=0x7fd0e00b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.294+0000 7fd100800700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd0e00b2ba0 0x7fd0e00b5060 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fd0ec006010 tx=0x7fd0ec00b280 comp rx=0 tx=0).stop 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.294+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8076090 msgr2=0x7fd0f81a0830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.294+0000 7fd100800700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8076090 0x7fd0f81a0830 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fd0e800e9f0 tx=0x7fd0e800ed00 comp rx=0 tx=0).stop 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 shutdown_connections 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd0f8114710 0x7fd0f81a4bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd0f8075770 0x7fd0f81a02f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd0e00b2ba0 0x7fd0e00b5060 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 --2- 172.21.15.28:0/4114080712 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd0f8076090 0x7fd0f81a0830 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 >> 172.21.15.28:0/4114080712 conn(0x7fd0f807a6a0 msgr2=0x7fd0f81115b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 shutdown_connections 2024-05-20T20:06:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:55.295+0000 7fd100800700 1 -- 172.21.15.28:0/4114080712 wait complete. 2024-05-20T20:06:55.959 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:55 smithi028 ceph-mon[31178]: pgmap v42: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:06:56.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:55 smithi154 ceph-mon[36314]: pgmap v42: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:06:56.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:55 smithi033 ceph-mon[36179]: pgmap v42: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:06:56.401 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:06:57.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:56 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:06:57.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:56 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:06:57.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:56 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:06:57.437 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:06:58.009 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:57 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:06:58.010 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:57 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:06:58.010 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:57 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:06:58.010 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:06:57 smithi028 ceph-mon[31178]: pgmap v43: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:06:58.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:57 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:06:58.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:57 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:06:58.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:57 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:06:58.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:06:57 smithi154 ceph-mon[36314]: pgmap v43: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:06:58.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:57 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:06:58.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:57 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:06:58.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:57 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:06:58.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:06:57 smithi033 ceph-mon[36179]: pgmap v43: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:06:58.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 -- 172.21.15.28:0/3466522492 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 msgr2=0x7f66481081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 --2- 172.21.15.28:0/3466522492 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 0x7f66481081c0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f6638009920 tx=0x7f6638009c30 comp rx=0 tx=0).stop 2024-05-20T20:06:58.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 -- 172.21.15.28:0/3466522492 shutdown_connections 2024-05-20T20:06:58.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 --2- 172.21.15.28:0/3466522492 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f66481088f0 0x7f6648110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 --2- 172.21.15.28:0/3466522492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 0x7f6648103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 --2- 172.21.15.28:0/3466522492 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 0x7f66481081c0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 -- 172.21.15.28:0/3466522492 >> 172.21.15.28:0/3466522492 conn(0x7f664807a6a0 msgr2=0x7f664807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 -- 172.21.15.28:0/3466522492 shutdown_connections 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.698+0000 7f664f400700 1 -- 172.21.15.28:0/3466522492 wait complete. 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.699+0000 7f664f400700 1 Processor -- start 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.700+0000 7f664f400700 1 -- start start 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.700+0000 7f664f400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 0x7f6648075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.700+0000 7f664f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 0x7f6648075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.700+0000 7f664f400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f66481088f0 0x7f66480762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.700+0000 7f664f400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6648113700 con 0x7f66481041a0 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.700+0000 7f664f400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6648113580 con 0x7f66481088f0 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.700+0000 7f664f400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6648113880 con 0x7f66481037f0 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 0x7f6648075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f6647e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 0x7f6648075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 0x7f6648075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60642/0 (socket says 172.21.15.28:60642) 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 -- 172.21.15.28:0/770773819 learned_addr learned my addr 172.21.15.28:0/770773819 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664d400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f66481088f0 0x7f66480762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 -- 172.21.15.28:0/770773819 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f66481088f0 msgr2=0x7f66480762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f66481088f0 0x7f66480762c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 -- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 msgr2=0x7f6648075d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 0x7f6648075d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 -- 172.21.15.28:0/770773819 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f66481a81a0 con 0x7f66481037f0 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.701+0000 7f664ca00700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 0x7f6648075840 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f663c00e9c0 tx=0x7f663c00ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.702+0000 7f6645600700 1 -- 172.21.15.28:0/770773819 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f663c004a30 con 0x7f66481037f0 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.702+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f66481a8490 con 0x7f66481037f0 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.702+0000 7f6645600700 1 -- 172.21.15.28:0/770773819 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f663c00cc70 con 0x7f66481037f0 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.702+0000 7f6645600700 1 -- 172.21.15.28:0/770773819 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f663c0054a0 con 0x7f66481037f0 2024-05-20T20:06:58.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.702+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f66481a89a0 con 0x7f66481037f0 2024-05-20T20:06:58.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.708+0000 7f6645600700 1 -- 172.21.15.28:0/770773819 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f663c004bd0 con 0x7f66481037f0 2024-05-20T20:06:58.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.708+0000 7f6659db0b80 1 Processor -- start 2024-05-20T20:06:58.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.709+0000 7f6659db0b80 1 -- start start 2024-05-20T20:06:58.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.709+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d800b5c0 0x5592d800b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.709+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d8005ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.709+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d80060f0 0x5592d8006570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.709+0000 7f6659db0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5592d7f63df0 con 0x5592d800b5c0 2024-05-20T20:06:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.709+0000 7f6659db0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5592d7f53990 con 0x5592d800bee0 2024-05-20T20:06:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.709+0000 7f6659db0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5592d7f579d0 con 0x5592d80060f0 2024-05-20T20:06:58.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.710+0000 7f6647e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d80060f0 0x5592d8006570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.710+0000 7f6647e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d80060f0 0x5592d8006570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60654/0 (socket says 172.21.15.28:60654) 2024-05-20T20:06:58.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.710+0000 7f6647e00700 1 -- 172.21.15.28:0/984226088 learned_addr learned my addr 172.21.15.28:0/984226088 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:58.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.710+0000 7f664ca00700 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d8005ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6645600700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66300b2ba0 0x7f66300b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6647e00700 1 -- 172.21.15.28:0/984226088 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 msgr2=0x5592d8005ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6647e00700 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d8005ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6647e00700 1 -- 172.21.15.28:0/984226088 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d800b5c0 msgr2=0x5592d800b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6647e00700 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d800b5c0 0x5592d800b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6647e00700 1 -- 172.21.15.28:0/984226088 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5592d7f1c3d0 con 0x5592d80060f0 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6645600700 1 -- 172.21.15.28:0/770773819 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f663c0e29a0 con 0x7f66481037f0 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f6647e00700 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d80060f0 0x5592d8006570 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f6638009920 tx=0x7f663800bf60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=915a7fb7d38673d4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.711+0000 7f664d400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66300b2ba0 0x7f66300b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.712+0000 7f6636000700 1 -- 172.21.15.28:0/984226088 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6638026070 con 0x5592d80060f0 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.712+0000 7f6636000700 1 -- 172.21.15.28:0/984226088 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f663800f910 con 0x5592d80060f0 2024-05-20T20:06:58.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.712+0000 7f6636000700 1 -- 172.21.15.28:0/984226088 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6638003680 con 0x5592d80060f0 2024-05-20T20:06:58.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 -- 172.21.15.28:0/984226088 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d80060f0 msgr2=0x5592d8006570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f664d400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66300b2ba0 0x7f66300b5060 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f6640009ec0 tx=0x7f6640009920 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:58.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d80060f0 0x5592d8006570 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f6638009920 tx=0x7f663800bf60 comp rx=0 tx=0).stop 2024-05-20T20:06:58.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 -- 172.21.15.28:0/984226088 shutdown_connections 2024-05-20T20:06:58.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d80060f0 0x5592d8006570 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d800b5c0 0x5592d800b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 --2- 172.21.15.28:0/984226088 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d8005ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 -- 172.21.15.28:0/984226088 >> 172.21.15.28:0/984226088 conn(0x5592d7f33b90 msgr2=0x5592d8000640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:58.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 -- 172.21.15.28:0/984226088 shutdown_connections 2024-05-20T20:06:58.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.713+0000 7f6659db0b80 1 -- 172.21.15.28:0/984226088 wait complete. 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.714+0000 7f6659db0b80 1 -- start start 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.714+0000 7f6659db0b80 1 Processor -- start 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8011c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d800efe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d800f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6659db0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5592d7e59f50 con 0x5592d80060f0 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6659db0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5592d7ef4710 con 0x5592d800bee0 2024-05-20T20:06:58.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6659db0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5592d8012930 con 0x5592d800b5c0 2024-05-20T20:06:58.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6659db0b80 10 client.0 _ll_register_callbacks cb 0x5592d7efdc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:58.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f664ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8011c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f664d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d800f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f664d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d800f520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:47926/0 (socket says 172.21.15.28:47926) 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f664d400700 1 -- 172.21.15.28:0/52266713 learned_addr learned my addr 172.21.15.28:0/52266713 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.715+0000 7f6647e00700 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d800efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f664ca00700 1 -- 172.21.15.28:0/52266713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 msgr2=0x5592d800efe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f664ca00700 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d800efe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f664ca00700 1 -- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 msgr2=0x5592d800f520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f664ca00700 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d800f520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f664ca00700 1 -- 172.21.15.28:0/52266713 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5592d7ea3880 con 0x5592d80060f0 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f664d400700 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d800f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f664ca00700 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8011c40 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f663c0dfec0 tx=0x7f663c0a9420 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2ee01128b58d6296 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.716+0000 7f662aa00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.717+0000 7f662aa00700 1 -- 172.21.15.28:0/52266713 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f663c0a5040 con 0x5592d80060f0 2024-05-20T20:06:58.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.717+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f663800f910 con 0x5592d80060f0 2024-05-20T20:06:58.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.717+0000 7f662aa00700 1 -- 172.21.15.28:0/52266713 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f663c0b2630 con 0x5592d80060f0 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.717+0000 7f662aa00700 1 -- 172.21.15.28:0/52266713 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f663c0a51e0 con 0x5592d80060f0 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.717+0000 7f6659db0b80 10 client.15444 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.717+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5592d7f24010 con 0x5592d80060f0 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.717+0000 7f662aa00700 1 -- 172.21.15.28:0/52266713 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f663800f910 con 0x5592d80060f0 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f662aa00700 1 -- 172.21.15.28:0/52266713 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f663c0cc0c0 con 0x5592d80060f0 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f6659db0b80 10 client.15444 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f6659db0b80 10 client.15444 resolve_mds: resolved name 'b' to daemon mds.b 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f6659db0b80 20 client.15444 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f6659db0b80 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d801f3c0 0x5592d8020830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f6659db0b80 4 client.15444 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:06:58.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5592d7f65d40 con 0x5592d801f3c0 2024-05-20T20:06:58.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.718+0000 7f6647e00700 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d801f3c0 0x5592d8020830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.719+0000 7f6647e00700 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d801f3c0 0x5592d8020830 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7fb20a1884a38175 server_cookie=2771bd3f9ab8de9d in_seq=0 out_seq=0 2024-05-20T20:06:58.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.719+0000 7f662aa00700 10 client.15444 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f662aa00700 1 -- 172.21.15.28:0/52266713 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5592d7f65d40 con 0x5592d801f3c0 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f662aa00700 10 client.15444 handle_command_reply: tid=0 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 client.15444 shutdown 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 msgr2=0x5592d8011c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8011c40 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f663c0dfec0 tx=0x7f663c0a9420 comp rx=0 tx=0).stop 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 shutdown_connections 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d801f3c0 0x5592d8020830 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d800f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8011c40 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 --2- 172.21.15.28:0/52266713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d800efe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f662aa00700 0 client.15444 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:58.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 >> 172.21.15.28:0/52266713 conn(0x5592d7f33b90 msgr2=0x5592d7fffd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:58.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 shutdown_connections 2024-05-20T20:06:58.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 1 -- 172.21.15.28:0/52266713 wait complete. 2024-05-20T20:06:58.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.720+0000 7f6659db0b80 20 client.15444 trim_cache size 0 max 16384 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.744+0000 7f6659db0b80 1 Processor -- start 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6659db0b80 1 -- start start 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8005ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d80064d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d80004a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6659db0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5592d801eb70 con 0x5592d80060f0 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6659db0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5592d801f240 con 0x5592d800bee0 2024-05-20T20:06:58.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6659db0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5592d801f910 con 0x5592d800b5c0 2024-05-20T20:06:58.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6647e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d80064d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f664ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8005ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6647e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d80064d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60664/0 (socket says 172.21.15.28:60664) 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.745+0000 7f6647e00700 1 -- 172.21.15.28:0/368529672 learned_addr learned my addr 172.21.15.28:0/368529672 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f664d400700 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d80004a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f664ca00700 1 -- 172.21.15.28:0/368529672 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 msgr2=0x5592d80064d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f664ca00700 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d80064d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f664ca00700 1 -- 172.21.15.28:0/368529672 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 msgr2=0x5592d80004a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f664ca00700 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d80004a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f664ca00700 1 -- 172.21.15.28:0/368529672 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5592d800bbf0 con 0x5592d80060f0 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f664ca00700 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8005ad0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f663c0a99e0 tx=0x7f663c0e2c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a8e745c7ab91fc9c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.746+0000 7f6635600700 1 -- 172.21.15.28:0/368529672 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f663c0a5040 con 0x5592d80060f0 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6635600700 1 -- 172.21.15.28:0/368529672 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f663c0c2070 con 0x5592d80060f0 2024-05-20T20:06:58.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6635600700 1 -- 172.21.15.28:0/368529672 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f663c0ade00 con 0x5592d80060f0 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6659db0b80 1 -- 172.21.15.28:0/368529672 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 msgr2=0x5592d8005ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6659db0b80 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8005ad0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f663c0a99e0 tx=0x7f663c0e2c20 comp rx=0 tx=0).stop 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6659db0b80 1 -- 172.21.15.28:0/368529672 shutdown_connections 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6659db0b80 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800bee0 0x5592d80004a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6659db0b80 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d8005ad0 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6659db0b80 1 --2- 172.21.15.28:0/368529672 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800b5c0 0x5592d80064d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.747+0000 7f6659db0b80 1 -- 172.21.15.28:0/368529672 >> 172.21.15.28:0/368529672 conn(0x5592d7f33b90 msgr2=0x5592d801e310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.748+0000 7f6659db0b80 1 -- 172.21.15.28:0/368529672 shutdown_connections 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.748+0000 7f6659db0b80 1 -- 172.21.15.28:0/368529672 wait complete. 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.748+0000 7f6659db0b80 1 -- start start 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.748+0000 7f6659db0b80 1 Processor -- start 2024-05-20T20:06:58.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d7fffc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800b5c0 0x5592d8000160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6659db0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800bee0 0x5592d80006a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6659db0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5592d80206b0 con 0x5592d80060f0 2024-05-20T20:06:58.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6659db0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5592d801ffe0 con 0x5592d800b5c0 2024-05-20T20:06:58.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6659db0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5592d8020d80 con 0x5592d800bee0 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6659db0b80 10 client.0 _ll_register_callbacks cb 0x5592d7efdc20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f664ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d7fffc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f664d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800bee0 0x5592d80006a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f664ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d7fffc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41460/0 (socket says 172.21.15.28:41460) 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f664ca00700 1 -- 172.21.15.28:0/1561479877 learned_addr learned my addr 172.21.15.28:0/1561479877 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.749+0000 7f6647e00700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800b5c0 0x5592d8000160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f664d400700 1 -- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800b5c0 msgr2=0x5592d8000160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f664d400700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800b5c0 0x5592d8000160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f664d400700 1 -- 172.21.15.28:0/1561479877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 msgr2=0x5592d7fffc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f664d400700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d7fffc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f664d400700 1 -- 172.21.15.28:0/1561479877 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5592d80285a0 con 0x5592d800bee0 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f6647e00700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800b5c0 0x5592d8000160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f664ca00700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d7fffc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f664d400700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800bee0 0x5592d80006a0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f6640004b10 tx=0x7f6640004b40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c9ddf0266c0e7e2f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f662aa00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f662aa00700 1 -- 172.21.15.28:0/1561479877 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6640019070 con 0x5592d800bee0 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f663c0c2070 con 0x5592d800bee0 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.750+0000 7f662aa00700 1 -- 172.21.15.28:0/1561479877 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f6640003a00 con 0x5592d800bee0 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f662aa00700 1 -- 172.21.15.28:0/1561479877 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6640003c00 con 0x5592d800bee0 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f6659db0b80 10 client.25567 fetch_fsmap learned FSMap version 29 2024-05-20T20:06:58.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5592d808b090 con 0x5592d800bee0 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f662aa00700 1 -- 172.21.15.28:0/1561479877 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f663c0c2070 con 0x5592d800bee0 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f662aa00700 1 -- 172.21.15.28:0/1561479877 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f664002a420 con 0x5592d800bee0 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f6659db0b80 10 client.25567 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f6659db0b80 10 client.25567 resolve_mds: resolved name 'b' to daemon mds.b 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.751+0000 7f6659db0b80 20 client.25567 populate_metadata read hostname 'smithi028' 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.752+0000 7f6659db0b80 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d808fc20 0x5592d808e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.752+0000 7f6659db0b80 4 client.25567 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "status"}] 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.752+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "status"}) -- 0x5592d7f5f520 con 0x5592d808fc20 2024-05-20T20:06:58.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.752+0000 7f664ca00700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d808fc20 0x5592d808e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:06:58.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.752+0000 7f664ca00700 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d808fc20 0x5592d808e930 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=afde74c4ac110fe3 server_cookie=f17a2975d08b27f1 in_seq=0 out_seq=0 2024-05-20T20:06:58.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.752+0000 7f662aa00700 10 client.25567 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:58.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f662aa00700 1 -- 172.21.15.28:0/1561479877 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+332 (crc 0 0 0) 0x5592d7f5f520 con 0x5592d808fc20 2024-05-20T20:06:58.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f662aa00700 10 client.25567 handle_command_reply: tid=0 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 client.25567 shutdown 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800bee0 msgr2=0x5592d80006a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800bee0 0x5592d80006a0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f6640004b10 tx=0x7f6640004b40 comp rx=0 tx=0).stop 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 shutdown_connections 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5592d808fc20 0x5592d808e930 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5592d800bee0 0x5592d80006a0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5592d80060f0 0x5592d7fffc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 --2- 172.21.15.28:0/1561479877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5592d800b5c0 0x5592d8000160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f662aa00700 0 client.25567 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 >> 172.21.15.28:0/1561479877 conn(0x5592d7f33b90 msgr2=0x5592d800edc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 shutdown_connections 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 1 -- 172.21.15.28:0/1561479877 wait complete. 2024-05-20T20:06:58.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.753+0000 7f6659db0b80 20 client.25567 trim_cache size 0 max 16384 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "cluster_fsid": "3fbec9b2-16e2-11ef-bc9a-c7b262605968", 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "whoami": 0, 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "id": 24542, 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "want_state": "up:active", 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "state": "up:active", 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "fs_name": "cephfs", 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "rank_uptime": 332.23729708500002, 2024-05-20T20:06:58.758 INFO:teuthology.orchestra.run.smithi028.stdout: "mdsmap_epoch": 29, 2024-05-20T20:06:58.759 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch": 81, 2024-05-20T20:06:58.759 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_epoch_barrier": 80, 2024-05-20T20:06:58.759 INFO:teuthology.orchestra.run.smithi028.stdout: "uptime": 355.72986649000001 2024-05-20T20:06:58.759 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:06:58.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66300b2ba0 msgr2=0x7f66300b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66300b2ba0 0x7f66300b5060 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f6640009ec0 tx=0x7f6640009920 comp rx=0 tx=0).stop 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 msgr2=0x7f6648075840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 0x7f6648075840 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f663c00e9c0 tx=0x7f663c00ecd0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 shutdown_connections 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66300b2ba0 0x7f66300b5060 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f66481088f0 0x7f66480762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f66481037f0 0x7f6648075840 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 --2- 172.21.15.28:0/770773819 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f66481041a0 0x7f6648075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 >> 172.21.15.28:0/770773819 conn(0x7f664807a6a0 msgr2=0x7f6648101a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.754+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 shutdown_connections 2024-05-20T20:06:58.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:06:58.755+0000 7f664f400700 1 -- 172.21.15.28:0/770773819 wait complete. 2024-05-20T20:07:00.128 INFO:tasks.quiescer:Ready to start quiesce thrashing; initial delay: 15 sec 2024-05-20T20:07:00.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 -- 172.21.15.28:0/902773221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 msgr2=0x7f238c10ff20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 --2- 172.21.15.28:0/902773221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 0x7f238c10ff20 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f238400b0a0 tx=0x7f238400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 -- 172.21.15.28:0/902773221 shutdown_connections 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 --2- 172.21.15.28:0/902773221 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 0x7f238c10ff20 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 --2- 172.21.15.28:0/902773221 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 0x7f238c104a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 --2- 172.21.15.28:0/902773221 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f238c104f50 0x7f238c10d440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 -- 172.21.15.28:0/902773221 >> 172.21.15.28:0/902773221 conn(0x7f238c0fde60 msgr2=0x7f238c100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 -- 172.21.15.28:0/902773221 shutdown_connections 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.754+0000 7f238be00700 1 -- 172.21.15.28:0/902773221 wait complete. 2024-05-20T20:07:00.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.755+0000 7f238be00700 1 Processor -- start 2024-05-20T20:07:00.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238be00700 1 -- start start 2024-05-20T20:07:00.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 0x7f238c1a1940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f238c104f50 0x7f238c1a1e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 0x7f238c19bb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f238c113660 con 0x7f238c10da60 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238be00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1134e0 con 0x7f238c104630 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238be00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1137e0 con 0x7f238c104f50 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 0x7f238c1a1940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 0x7f238c1a1940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:47962/0 (socket says 172.21.15.28:47962) 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f2383e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f238c104f50 0x7f238c1a1e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.756+0000 7f238aa00700 1 -- 172.21.15.28:0/4271604830 learned_addr learned my addr 172.21.15.28:0/4271604830 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:00.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f238b400700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 0x7f238c19bb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f2383e00700 1 -- 172.21.15.28:0/4271604830 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 msgr2=0x7f238c1a1940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f2383e00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 0x7f238c1a1940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f2383e00700 1 -- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 msgr2=0x7f238c19bb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f2383e00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 0x7f238c19bb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f2383e00700 1 -- 172.21.15.28:0/4271604830 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f238c19c260 con 0x7f238c104f50 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f238aa00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 0x7f238c1a1940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f238b400700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 0x7f238c19bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.757+0000 7f2383e00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f238c104f50 0x7f238c1a1e80 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f237400e9c0 tx=0x7f237400ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.758+0000 7f2383400700 1 -- 172.21.15.28:0/4271604830 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2374004220 con 0x7f238c104f50 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.758+0000 7f2383400700 1 -- 172.21.15.28:0/4271604830 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23740043c0 con 0x7f238c104f50 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.758+0000 7f2383400700 1 -- 172.21.15.28:0/4271604830 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2374003c60 con 0x7f238c104f50 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.758+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f238c19c550 con 0x7f238c104f50 2024-05-20T20:07:00.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.758+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f238c1a8810 con 0x7f238c104f50 2024-05-20T20:07:00.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.762+0000 7f2383400700 1 -- 172.21.15.28:0/4271604830 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f237400c7c0 con 0x7f238c104f50 2024-05-20T20:07:00.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 Processor -- start 2024-05-20T20:07:00.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 -- start start 2024-05-20T20:07:00.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f2383400700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23700b2c20 0x7f23700b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325020 0x56024e325400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325940 0x56024e31f4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e31ff30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56024e27de40 con 0x56024e325940 2024-05-20T20:07:00.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56024e26d9e0 con 0x56024e31fad0 2024-05-20T20:07:00.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.763+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56024e271a20 con 0x56024e325020 2024-05-20T20:07:00.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f2383400700 1 -- 172.21.15.28:0/4271604830 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f23740e2880 con 0x7f238c104f50 2024-05-20T20:07:00.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238b400700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23700b2c20 0x7f23700b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325020 0x56024e325400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325020 0x56024e325400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60700/0 (socket says 172.21.15.28:60700) 2024-05-20T20:07:00.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 -- 172.21.15.28:0/737468167 learned_addr learned my addr 172.21.15.28:0/737468167 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:00.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238b400700 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e31ff30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f2383e00700 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325940 0x56024e31f4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 -- 172.21.15.28:0/737468167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 msgr2=0x56024e31ff30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e31ff30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 -- 172.21.15.28:0/737468167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325940 msgr2=0x56024e31f4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325940 0x56024e31f4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.764+0000 7f238aa00700 1 -- 172.21.15.28:0/737468167 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56024e236420 con 0x56024e325020 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.765+0000 7f238aa00700 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325020 0x56024e325400 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f237c0117e0 tx=0x7f237c011af0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=549b9d9e96a508b3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.765+0000 7f237be00700 1 -- 172.21.15.28:0/737468167 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f237c01e070 con 0x56024e325020 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.765+0000 7f238b400700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23700b2c20 0x7f23700b50e0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f2384009fd0 tx=0x7f238400bf60 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.765+0000 7f237be00700 1 -- 172.21.15.28:0/737468167 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f237c00f930 con 0x56024e325020 2024-05-20T20:07:00.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.765+0000 7f237be00700 1 -- 172.21.15.28:0/737468167 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f237c019a20 con 0x56024e325020 2024-05-20T20:07:00.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 -- 172.21.15.28:0/737468167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325020 msgr2=0x56024e325400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325020 0x56024e325400 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f237c0117e0 tx=0x7f237c011af0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 -- 172.21.15.28:0/737468167 shutdown_connections 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e31ff30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325020 0x56024e325400 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/737468167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325940 0x56024e31f4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 -- 172.21.15.28:0/737468167 >> 172.21.15.28:0/737468167 conn(0x56024e24dbe0 msgr2=0x56024e31a150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 -- 172.21.15.28:0/737468167 shutdown_connections 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.766+0000 7f239c5b6b80 1 -- 172.21.15.28:0/737468167 wait complete. 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.767+0000 7f239c5b6b80 1 -- start start 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.767+0000 7f239c5b6b80 1 Processor -- start 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e32b6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e328ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e329020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56024e173fa0 con 0x56024e325020 2024-05-20T20:07:00.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56024e20e760 con 0x56024e325940 2024-05-20T20:07:00.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56024e32c460 con 0x56024e31fad0 2024-05-20T20:07:00.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f239c5b6b80 10 client.0 _ll_register_callbacks cb 0x56024e217c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e32b6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e32b6f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60716/0 (socket says 172.21.15.28:60716) 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 -- 172.21.15.28:0/812956467 learned_addr learned my addr 172.21.15.28:0/812956467 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238b400700 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e329020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f2383e00700 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e328ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 -- 172.21.15.28:0/812956467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 msgr2=0x56024e329020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e329020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 -- 172.21.15.28:0/812956467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 msgr2=0x56024e328ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e328ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.768+0000 7f238aa00700 1 -- 172.21.15.28:0/812956467 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56024e1bd8d0 con 0x56024e31fad0 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.769+0000 7f238aa00700 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e32b6f0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f237c00fd10 tx=0x7f237c00fdf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4ff37ca7b52d1786 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.769+0000 7f2378c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:07:00.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.769+0000 7f2378c00700 1 -- 172.21.15.28:0/812956467 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f237c027060 con 0x56024e31fad0 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.769+0000 7f2378c00700 1 -- 172.21.15.28:0/812956467 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f237c004030 con 0x56024e31fad0 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.769+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f237c00f930 con 0x56024e31fad0 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f2378c00700 1 -- 172.21.15.28:0/812956467 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f237c01e040 con 0x56024e31fad0 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 10 client.25579 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56024e23e060 con 0x56024e31fad0 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f2378c00700 1 -- 172.21.15.28:0/812956467 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f237c00f930 con 0x56024e31fad0 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f2378c00700 1 -- 172.21.15.28:0/812956467 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f237c021030 con 0x56024e31fad0 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 10 client.25579 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 10 client.25579 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 20 client.25579 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e338f40 0x56024e33b330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 4 client.25579 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.770+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56024e27fd90 con 0x56024e338f40 2024-05-20T20:07:00.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.771+0000 7f2383e00700 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e338f40 0x56024e33b330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.771+0000 7f2383e00700 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e338f40 0x56024e33b330 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=d3a123f1a1fc61e0 server_cookie=238242c26bd139cc in_seq=0 out_seq=0 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.771+0000 7f2378c00700 10 client.25579 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.772+0000 7f2378c00700 1 -- 172.21.15.28:0/812956467 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56024e27fd90 con 0x56024e338f40 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.772+0000 7f2378c00700 10 client.25579 handle_command_reply: tid=0 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.772+0000 7f239c5b6b80 1 client.25579 shutdown 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.772+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 msgr2=0x56024e32b6f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.772+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e32b6f0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f237c00fd10 tx=0x7f237c00fdf0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 shutdown_connections 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e329020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e32b6f0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e338f40 0x56024e33b330 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/812956467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e328ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 >> 172.21.15.28:0/812956467 conn(0x56024e24dbe0 msgr2=0x56024e31e4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:00.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f2378c00700 0 client.25579 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:00.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 shutdown_connections 2024-05-20T20:07:00.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 1 -- 172.21.15.28:0/812956467 wait complete. 2024-05-20T20:07:00.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.773+0000 7f239c5b6b80 20 client.25579 trim_cache size 0 max 16384 2024-05-20T20:07:00.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 Processor -- start 2024-05-20T20:07:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 -- start start 2024-05-20T20:07:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e335500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e31f4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e3231e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1aff00 con 0x56024e325020 2024-05-20T20:07:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1b1a40 con 0x56024e325940 2024-05-20T20:07:00.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1b05d0 con 0x56024e31fad0 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.791+0000 7f2383e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e31f4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f2383e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e31f4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41498/0 (socket says 172.21.15.28:41498) 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f2383e00700 1 -- 172.21.15.28:0/1287623436 learned_addr learned my addr 172.21.15.28:0/1287623436 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238aa00700 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e335500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238b400700 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e3231e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238aa00700 1 -- 172.21.15.28:0/1287623436 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 msgr2=0x56024e3231e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238aa00700 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e3231e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238aa00700 1 -- 172.21.15.28:0/1287623436 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 msgr2=0x56024e31f4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238aa00700 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e31f4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238aa00700 1 -- 172.21.15.28:0/1287623436 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56024e327050 con 0x56024e31fad0 2024-05-20T20:07:00.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f2383e00700 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e31f4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:00.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.792+0000 7f238aa00700 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e335500 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f237c000c00 tx=0x7f237c011790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cdddcd43822a43ab server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:00.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.793+0000 7f237b400700 1 -- 172.21.15.28:0/1287623436 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f237c015070 con 0x56024e31fad0 2024-05-20T20:07:00.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.793+0000 7f237b400700 1 -- 172.21.15.28:0/1287623436 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f237c0054f0 con 0x56024e31fad0 2024-05-20T20:07:00.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.793+0000 7f237b400700 1 -- 172.21.15.28:0/1287623436 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f237c027070 con 0x56024e31fad0 2024-05-20T20:07:00.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.793+0000 7f239c5b6b80 1 -- 172.21.15.28:0/1287623436 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 msgr2=0x56024e335500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.793+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e335500 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f237c000c00 tx=0x7f237c011790 comp rx=0 tx=0).stop 2024-05-20T20:07:00.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 -- 172.21.15.28:0/1287623436 shutdown_connections 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e325940 0x56024e3231e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e31fad0 0x56024e335500 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/1287623436 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e31f4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 -- 172.21.15.28:0/1287623436 >> 172.21.15.28:0/1287623436 conn(0x56024e24dbe0 msgr2=0x56024e319c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 -- 172.21.15.28:0/1287623436 shutdown_connections 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 -- 172.21.15.28:0/1287623436 wait complete. 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 -- start start 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.794+0000 7f239c5b6b80 1 Processor -- start 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e331a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e331f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f239c5b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325940 0x56024e340240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1b1370 con 0x56024e325020 2024-05-20T20:07:00.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1b0ca0 con 0x56024e31fad0 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f239c5b6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f238c1adea0 con 0x56024e325940 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f239c5b6b80 10 client.0 _ll_register_callbacks cb 0x56024e217c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f238aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e331a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f238aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e331a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48014/0 (socket says 172.21.15.28:48014) 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f238aa00700 1 -- 172.21.15.28:0/627318292 learned_addr learned my addr 172.21.15.28:0/627318292 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f2383e00700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e331f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.795+0000 7f238b400700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325940 0x56024e340240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2383e00700 1 -- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325940 msgr2=0x56024e340240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2383e00700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325940 0x56024e340240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2383e00700 1 -- 172.21.15.28:0/627318292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 msgr2=0x56024e331a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2383e00700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e331a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2383e00700 1 -- 172.21.15.28:0/627318292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56024e340ab0 con 0x56024e325020 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f238aa00700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e331a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2383e00700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e331f90 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f23740a59b0 tx=0x7f23740b21a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=626911fa3693cd74 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2378c00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2378c00700 1 -- 172.21.15.28:0/627318292 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23740aa030 con 0x56024e325020 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f2378c00700 1 -- 172.21.15.28:0/627318292 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f23740094b0 con 0x56024e325020 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.796+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f237c0054f0 con 0x56024e325020 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f2378c00700 1 -- 172.21.15.28:0/627318292 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f23740a9930 con 0x56024e325020 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 10 client.15468 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:00.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56024e3a3660 con 0x56024e325020 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f2378c00700 1 -- 172.21.15.28:0/627318292 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f237c0054f0 con 0x56024e325020 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f2378c00700 1 -- 172.21.15.28:0/627318292 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f23740cabe0 con 0x56024e325020 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 10 client.15468 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 10 client.15468 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 20 client.15468 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e3a3cc0 0x56024e3a60b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 4 client.15468 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.797+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56024e279570 con 0x56024e3a3cc0 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.798+0000 7f238aa00700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e3a3cc0 0x56024e3a60b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.798+0000 7f238aa00700 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e3a3cc0 0x56024e3a60b0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=60dea8ce70bfe127 server_cookie=8c1d4a3efd780e7a in_seq=0 out_seq=0 2024-05-20T20:07:00.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.798+0000 7f2378c00700 10 client.15468 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.798+0000 7f2378c00700 1 -- 172.21.15.28:0/627318292 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56024e279570 con 0x56024e3a3cc0 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.798+0000 7f2378c00700 10 client.15468 handle_command_reply: tid=0 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 client.15468 shutdown 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 msgr2=0x56024e331f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e331f90 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f23740a59b0 tx=0x7f23740b21a0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 shutdown_connections 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56024e325940 0x56024e340240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56024e31fad0 0x56024e331a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56024e3a3cc0 0x56024e3a60b0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 --2- 172.21.15.28:0/627318292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56024e325020 0x56024e331f90 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 >> 172.21.15.28:0/627318292 conn(0x56024e24dbe0 msgr2=0x56024e328900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:00.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f2378c00700 0 client.15468 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:00.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 shutdown_connections 2024-05-20T20:07:00.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 1 -- 172.21.15.28:0/627318292 wait complete. 2024-05-20T20:07:00.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.799+0000 7f239c5b6b80 20 client.15468 trim_cache size 0 max 16384 2024-05-20T20:07:00.803 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23700b2c20 msgr2=0x7f23700b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23700b2c20 0x7f23700b50e0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f2384009fd0 tx=0x7f238400bf60 comp rx=0 tx=0).stop 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f238c104f50 msgr2=0x7f238c1a1e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f238c104f50 0x7f238c1a1e80 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f237400e9c0 tx=0x7f237400ecd0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 shutdown_connections 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f238c10da60 0x7f238c19bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23700b2c20 0x7f23700b50e0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f238c104630 0x7f238c1a1940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 --2- 172.21.15.28:0/4271604830 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f238c104f50 0x7f238c1a1e80 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:00.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 >> 172.21.15.28:0/4271604830 conn(0x7f238c0fde60 msgr2=0x7f238c10fd00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:00.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 shutdown_connections 2024-05-20T20:07:00.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:00.800+0000 7f238be00700 1 -- 172.21.15.28:0/4271604830 wait complete. 2024-05-20T20:07:01.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:00 smithi028 ceph-mon[31178]: pgmap v44: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:07:01.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:00 smithi154 ceph-mon[36314]: pgmap v44: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:07:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:00 smithi033 ceph-mon[36179]: pgmap v44: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:07:01.746 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:07:02.232 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:03.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:02 smithi028 ceph-mon[31178]: pgmap v45: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:03.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:02 smithi154 ceph-mon[36314]: pgmap v45: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:02 smithi033 ceph-mon[36179]: pgmap v45: 97 pgs: 97 active+clean; 1.3 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.305+0000 7fad53200640 1 -- 172.21.15.28:0/674087308 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 msgr2=0x7fad4c108c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.305+0000 7fad53200640 1 --2- 172.21.15.28:0/674087308 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 0x7fad4c108c40 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fad40009860 tx=0x7fad4002a340 comp rx=0 tx=0).stop 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.305+0000 7fad53200640 1 -- 172.21.15.28:0/674087308 shutdown_connections 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.305+0000 7fad53200640 1 --2- 172.21.15.28:0/674087308 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fad4c103210 0x7fad4c10f720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.305+0000 7fad53200640 1 --2- 172.21.15.28:0/674087308 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 0x7fad4c102cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.305+0000 7fad53200640 1 --2- 172.21.15.28:0/674087308 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 0x7fad4c108c40 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.305+0000 7fad53200640 1 -- 172.21.15.28:0/674087308 >> 172.21.15.28:0/674087308 conn(0x7fad4c0fe3a0 msgr2=0x7fad4c1007c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.306+0000 7fad53200640 1 -- 172.21.15.28:0/674087308 shutdown_connections 2024-05-20T20:07:04.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.306+0000 7fad53200640 1 -- 172.21.15.28:0/674087308 wait complete. 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 Processor -- start 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 -- start start 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 0x7fad4c19b280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fad4c103210 0x7fad4c19b7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 0x7fad4c19fb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c112b20 con 0x7fad4c102850 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c1129a0 con 0x7fad4c108860 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad53200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c112ca0 con 0x7fad4c103210 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.307+0000 7fad51e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 0x7fad4c19b280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fad4c103210 0x7fad4c19b7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 0x7fad4c19b280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41522/0 (socket says 172.21.15.28:41522) 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51e00640 1 -- 172.21.15.28:0/32836456 learned_addr learned my addr 172.21.15.28:0/32836456 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad52800640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 0x7fad4c19fb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51400640 1 -- 172.21.15.28:0/32836456 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 msgr2=0x7fad4c19fb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51400640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 0x7fad4c19fb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51400640 1 -- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 msgr2=0x7fad4c19b280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51400640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 0x7fad4c19b280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51400640 1 -- 172.21.15.28:0/32836456 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fad4c1a0270 con 0x7fad4c103210 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51e00640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 0x7fad4c19b280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.308+0000 7fad51400640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fad4c103210 0x7fad4c19b7c0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fad3c005fd0 tx=0x7fad3c00ee50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.309+0000 7fad46a00640 1 -- 172.21.15.28:0/32836456 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad3c00cd00 con 0x7fad4c103210 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.309+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fad4c1a0560 con 0x7fad4c103210 2024-05-20T20:07:04.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.309+0000 7fad46a00640 1 -- 172.21.15.28:0/32836456 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fad3c0035f0 con 0x7fad4c103210 2024-05-20T20:07:04.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.309+0000 7fad46a00640 1 -- 172.21.15.28:0/32836456 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad3c010690 con 0x7fad4c103210 2024-05-20T20:07:04.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.309+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fad4c1a7df0 con 0x7fad4c103210 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.312+0000 7fad575e5740 1 Processor -- start 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad46a00640 1 -- 172.21.15.28:0/32836456 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7fad3c003a60 con 0x7fad4c103210 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad575e5740 1 -- start start 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad575e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d1641230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad575e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1641770 0x5595d164baf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad575e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164e420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad575e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5595d15b1b90 con 0x5595d1640e50 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad575e5740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5595d15a1730 con 0x5595d164c030 2024-05-20T20:07:04.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad575e5740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5595d15a5770 con 0x5595d1641770 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad51e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1641770 0x5595d164baf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad51400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164e420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad52800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d1641230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad51e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1641770 0x5595d164baf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60758/0 (socket says 172.21.15.28:60758) 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.313+0000 7fad51e00640 1 -- 172.21.15.28:0/2649588175 learned_addr learned my addr 172.21.15.28:0/2649588175 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad51400640 1 -- 172.21.15.28:0/2649588175 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1641770 msgr2=0x5595d164baf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad51400640 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1641770 0x5595d164baf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad51400640 1 -- 172.21.15.28:0/2649588175 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 msgr2=0x5595d1641230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad51400640 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d1641230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad51400640 1 -- 172.21.15.28:0/2649588175 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5595d153a8b0 con 0x5595d164c030 2024-05-20T20:07:04.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad51e00640 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1641770 0x5595d164baf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:04.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad51400640 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164e420 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fad3c0dff90 tx=0x7fad3c0e6e60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=98352bbc34c281a9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.314+0000 7fad3b000640 1 -- 172.21.15.28:0/2649588175 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad3c0f6070 con 0x5595d164c030 2024-05-20T20:07:04.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.315+0000 7fad3b000640 1 -- 172.21.15.28:0/2649588175 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fad3c0f1480 con 0x5595d164c030 2024-05-20T20:07:04.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.315+0000 7fad3b000640 1 -- 172.21.15.28:0/2649588175 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad3c0e2e60 con 0x5595d164c030 2024-05-20T20:07:04.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.315+0000 7fad46a00640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fad340b2b00 0x7fad340b4fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.315+0000 7fad46a00640 1 -- 172.21.15.28:0/32836456 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fad3c0e2a70 con 0x7fad4c103210 2024-05-20T20:07:04.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.315+0000 7fad52800640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fad340b2b00 0x7fad340b4fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.315+0000 7fad575e5740 1 -- 172.21.15.28:0/2649588175 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 msgr2=0x5595d164e420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.315+0000 7fad575e5740 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164e420 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fad3c0dff90 tx=0x7fad3c0e6e60 comp rx=0 tx=0).stop 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 -- 172.21.15.28:0/2649588175 shutdown_connections 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164e420 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1641770 0x5595d164baf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 --2- 172.21.15.28:0/2649588175 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d1641230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 -- 172.21.15.28:0/2649588175 >> 172.21.15.28:0/2649588175 conn(0x5595d163d360 msgr2=0x5595d163d740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 -- 172.21.15.28:0/2649588175 shutdown_connections 2024-05-20T20:07:04.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 -- 172.21.15.28:0/2649588175 wait complete. 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad52800640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fad340b2b00 0x7fad340b4fc0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fad48005fd0 tx=0x7fad48009600 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.316+0000 7fad575e5740 1 -- start start 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 1 Processor -- start 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1640e50 0x5595d164ba70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d1648d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d16494a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5595d1287790 con 0x5595d1641770 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5595d1549bc0 con 0x5595d1640e50 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5595d164ec60 con 0x5595d164c030 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad575e5740 10 client.0 _ll_register_callbacks cb 0x5595d1527400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.317+0000 7fad51400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d1648d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1640e50 0x5595d164ba70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad52800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d16494a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1640e50 0x5595d164ba70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48062/0 (socket says 172.21.15.28:48062) 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 -- 172.21.15.28:0/2360024102 learned_addr learned my addr 172.21.15.28:0/2360024102 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 -- 172.21.15.28:0/2360024102 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 msgr2=0x5595d16494a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d16494a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 -- 172.21.15.28:0/2360024102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 msgr2=0x5595d1648d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d1648d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 -- 172.21.15.28:0/2360024102 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5595d12b6210 con 0x5595d1640e50 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad51e00640 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1640e50 0x5595d164ba70 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fad400306e0 tx=0x7fad40030710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c46f9c76e55737bc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.318+0000 7fad2f400640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad2f400640 1 -- 172.21.15.28:0/2360024102 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad40041070 con 0x5595d1640e50 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fad3c0f1480 con 0x5595d1640e50 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad2f400640 1 -- 172.21.15.28:0/2360024102 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fad40004240 con 0x5595d1640e50 2024-05-20T20:07:04.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad2f400640 1 -- 172.21.15.28:0/2360024102 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fad4004d050 con 0x5595d1640e50 2024-05-20T20:07:04.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad575e5740 10 client.25625 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:04.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5595d1382620 con 0x5595d1640e50 2024-05-20T20:07:04.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad2f400640 1 -- 172.21.15.28:0/2360024102 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fad3c0f1480 con 0x5595d1640e50 2024-05-20T20:07:04.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.319+0000 7fad2f400640 1 -- 172.21.15.28:0/2360024102 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fad400037b0 con 0x5595d1640e50 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.320+0000 7fad575e5740 10 client.25625 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.320+0000 7fad575e5740 10 client.25625 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.320+0000 7fad575e5740 20 client.25625 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.320+0000 7fad575e5740 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d165ba80 0x5595d165de70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.320+0000 7fad575e5740 4 client.25625 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.320+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5595d15b3ae0 con 0x5595d165ba80 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.320+0000 7fad51400640 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d165ba80 0x5595d165de70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.321+0000 7fad51400640 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d165ba80 0x5595d165de70 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6d5f3ce459b23ffc server_cookie=322ac588e372eb5c in_seq=0 out_seq=0 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.321+0000 7fad2f400640 10 client.25625 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.321+0000 7fad2f400640 1 -- 172.21.15.28:0/2360024102 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5595d15b3ae0 con 0x5595d165ba80 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.321+0000 7fad2f400640 10 client.25625 handle_command_reply: tid=0 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 client.25625 shutdown 2024-05-20T20:07:04.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1640e50 msgr2=0x5595d164ba70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1640e50 0x5595d164ba70 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fad400306e0 tx=0x7fad40030710 comp rx=0 tx=0).stop 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 shutdown_connections 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d165ba80 0x5595d165de70 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d16494a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d1648d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad2f400640 0 client.25625 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 --2- 172.21.15.28:0/2360024102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1640e50 0x5595d164ba70 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 >> 172.21.15.28:0/2360024102 conn(0x5595d163d360 msgr2=0x5595d16430e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 shutdown_connections 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 1 -- 172.21.15.28:0/2360024102 wait complete. 2024-05-20T20:07:04.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.322+0000 7fad575e5740 20 client.25625 trim_cache size 0 max 16384 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.339+0000 7fad575e5740 1 Processor -- start 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad575e5740 1 -- start start 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad575e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d16430e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad575e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1641770 0x5595d163c340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad575e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d164b5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad575e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c1b0770 con 0x5595d1640e50 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad575e5740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c1b22b0 con 0x5595d1641770 2024-05-20T20:07:04.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad575e5740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c1b0e40 con 0x5595d164c030 2024-05-20T20:07:04.341 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad51e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d16430e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad51400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1641770 0x5595d163c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad51e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d16430e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41546/0 (socket says 172.21.15.28:41546) 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad51e00640 1 -- 172.21.15.28:0/1531022685 learned_addr learned my addr 172.21.15.28:0/1531022685 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.340+0000 7fad52800640 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d164b5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad51400640 1 -- 172.21.15.28:0/1531022685 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 msgr2=0x5595d164b5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad51400640 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d164b5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad51400640 1 -- 172.21.15.28:0/1531022685 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 msgr2=0x5595d16430e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad51400640 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d16430e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad51400640 1 -- 172.21.15.28:0/1531022685 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5595d1649810 con 0x5595d1641770 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad51400640 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1641770 0x5595d163c340 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fad3c00c6a0 tx=0x7fad3c0e64a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=95a0fb2e21138a52 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad3a600640 1 -- 172.21.15.28:0/1531022685 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad3c0e2e60 con 0x5595d1641770 2024-05-20T20:07:04.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad3a600640 1 -- 172.21.15.28:0/1531022685 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fad3c0e62f0 con 0x5595d1641770 2024-05-20T20:07:04.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.341+0000 7fad3a600640 1 -- 172.21.15.28:0/1531022685 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad3c0a9630 con 0x5595d1641770 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 -- 172.21.15.28:0/1531022685 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1641770 msgr2=0x5595d163c340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1641770 0x5595d163c340 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fad3c00c6a0 tx=0x7fad3c0e64a0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 -- 172.21.15.28:0/1531022685 shutdown_connections 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d1641770 0x5595d163c340 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d164c030 0x5595d164b5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 --2- 172.21.15.28:0/1531022685 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1640e50 0x5595d16430e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 -- 172.21.15.28:0/1531022685 >> 172.21.15.28:0/1531022685 conn(0x5595d163d360 msgr2=0x5595d163bc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 -- 172.21.15.28:0/1531022685 shutdown_connections 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.342+0000 7fad575e5740 1 -- 172.21.15.28:0/1531022685 wait complete. 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.343+0000 7fad575e5740 1 -- start start 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.343+0000 7fad575e5740 1 Processor -- start 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad575e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1640e50 0x5595d1654640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad575e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d164fa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad575e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164ffa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad575e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c1b1be0 con 0x5595d1641770 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad575e5740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c1b1510 con 0x5595d164c030 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad575e5740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fad4c1ae750 con 0x5595d1640e50 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad575e5740 10 client.0 _ll_register_callbacks cb 0x7fad400096b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164ffa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164ffa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48072/0 (socket says 172.21.15.28:48072) 2024-05-20T20:07:04.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad51400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d164fa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 -- 172.21.15.28:0/3074605046 learned_addr learned my addr 172.21.15.28:0/3074605046 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad51e00640 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1640e50 0x5595d1654640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 -- 172.21.15.28:0/3074605046 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1640e50 msgr2=0x5595d1654640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1640e50 0x5595d1654640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 -- 172.21.15.28:0/3074605046 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 msgr2=0x5595d164fa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d164fa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad52800640 1 -- 172.21.15.28:0/3074605046 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5595d16507d0 con 0x5595d164c030 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.344+0000 7fad51400640 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d164fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad52800640 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164ffa0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fad48011f00 tx=0x7fad48004a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fd8e66312978b508 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad2f400640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad2f400640 1 -- 172.21.15.28:0/3074605046 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad4802f060 con 0x5595d164c030 2024-05-20T20:07:04.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fad3c0e62f0 con 0x5595d164c030 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad2f400640 1 -- 172.21.15.28:0/3074605046 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fad48003d80 con 0x5595d164c030 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad2f400640 1 -- 172.21.15.28:0/3074605046 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fad48020960 con 0x5595d164c030 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad575e5740 10 client.25631 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.345+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5595d16c6750 con 0x5595d164c030 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad2f400640 1 -- 172.21.15.28:0/3074605046 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fad3c0e62f0 con 0x5595d164c030 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad2f400640 1 -- 172.21.15.28:0/3074605046 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fad48029030 con 0x5595d164c030 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad575e5740 10 client.25631 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad575e5740 10 client.25631 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad575e5740 20 client.25631 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad575e5740 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d16c6f00 0x5595d16c9320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad575e5740 4 client.25631 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5595d15ad2c0 con 0x5595d16c6f00 2024-05-20T20:07:04.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.346+0000 7fad51e00640 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d16c6f00 0x5595d16c9320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.347+0000 7fad51e00640 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d16c6f00 0x5595d16c9320 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e52305a8a8089961 server_cookie=de883aff899582c1 in_seq=0 out_seq=0 2024-05-20T20:07:04.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.347+0000 7fad2f400640 10 client.25631 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:04.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad2f400640 1 -- 172.21.15.28:0/3074605046 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5595d15ad2c0 con 0x5595d16c6f00 2024-05-20T20:07:04.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad2f400640 10 client.25631 handle_command_reply: tid=0 2024-05-20T20:07:04.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 client.25631 shutdown 2024-05-20T20:07:04.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 msgr2=0x5595d164ffa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164ffa0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fad48011f00 tx=0x7fad48004a40 comp rx=0 tx=0).stop 2024-05-20T20:07:04.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 shutdown_connections 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5595d16c6f00 0x5595d16c9320 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595d164c030 0x5595d164ffa0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595d1641770 0x5595d164fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad2f400640 0 client.25631 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 --2- 172.21.15.28:0/3074605046 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595d1640e50 0x5595d1654640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 >> 172.21.15.28:0/3074605046 conn(0x5595d163d360 msgr2=0x5595d1642f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 shutdown_connections 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 1 -- 172.21.15.28:0/3074605046 wait complete. 2024-05-20T20:07:04.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.355+0000 7fad575e5740 20 client.25631 trim_cache size 0 max 16384 2024-05-20T20:07:04.357 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:07:04.357 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:07:04.357 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:07:04.357 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fad340b2b00 msgr2=0x7fad340b4fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fad340b2b00 0x7fad340b4fc0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fad48005fd0 tx=0x7fad48009600 comp rx=0 tx=0).stop 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fad4c103210 msgr2=0x7fad4c19b7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fad4c103210 0x7fad4c19b7c0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fad3c005fd0 tx=0x7fad3c00ee50 comp rx=0 tx=0).stop 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 shutdown_connections 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fad4c108860 0x7fad4c19fb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fad340b2b00 0x7fad340b4fc0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fad4c103210 0x7fad4c19b7c0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 --2- 172.21.15.28:0/32836456 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fad4c102850 0x7fad4c19b280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.356+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 >> 172.21.15.28:0/32836456 conn(0x7fad4c0fe3a0 msgr2=0x7fad4c10e060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.357+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 shutdown_connections 2024-05-20T20:07:04.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:04.357+0000 7fad53200640 1 -- 172.21.15.28:0/32836456 wait complete. 2024-05-20T20:07:04.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.388+0000 7f2f3de00700 1 -- 172.21.15.28:0/3733109558 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 msgr2=0x7f2f381081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.388+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3733109558 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 0x7f2f381081c0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f2f240099c0 tx=0x7f2f24009cd0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.389+0000 7f2f3de00700 1 -- 172.21.15.28:0/3733109558 shutdown_connections 2024-05-20T20:07:04.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.389+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3733109558 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f381088f0 0x7f2f38110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.389+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3733109558 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 0x7f2f38103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.389+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3733109558 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 0x7f2f381081c0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.389+0000 7f2f3de00700 1 -- 172.21.15.28:0/3733109558 >> 172.21.15.28:0/3733109558 conn(0x7f2f3807a6a0 msgr2=0x7f2f3807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.389+0000 7f2f3de00700 1 -- 172.21.15.28:0/3733109558 shutdown_connections 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.389+0000 7f2f3de00700 1 -- 172.21.15.28:0/3733109558 wait complete. 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.390+0000 7f2f3de00700 1 Processor -- start 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.390+0000 7f2f3de00700 1 -- start start 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f3de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 0x7f2f3819bdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f3de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 0x7f2f3819c2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f3de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f381088f0 0x7f2f381a0680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f3de00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2f38113720 con 0x7f2f381037f0 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f3de00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381135a0 con 0x7f2f381041a0 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f3de00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381138a0 con 0x7f2f381088f0 2024-05-20T20:07:04.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 0x7f2f3819bdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 0x7f2f3819bdb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41592/0 (socket says 172.21.15.28:41592) 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 -- 172.21.15.28:0/3878129684 learned_addr learned my addr 172.21.15.28:0/3878129684 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36000700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 0x7f2f3819c2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f37400700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f381088f0 0x7f2f381a0680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 -- 172.21.15.28:0/3878129684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f381088f0 msgr2=0x7f2f381a0680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f381088f0 0x7f2f381a0680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 -- 172.21.15.28:0/3878129684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 msgr2=0x7f2f3819c2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 0x7f2f3819c2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.391+0000 7f2f36a00700 1 -- 172.21.15.28:0/3878129684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2f381a0dc0 con 0x7f2f381037f0 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.392+0000 7f2f36a00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 0x7f2f3819bdb0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f2f2800e9a0 tx=0x7f2f2800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.392+0000 7f2f3ca00700 1 -- 172.21.15.28:0/3878129684 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f2800ca10 con 0x7f2f381037f0 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.392+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2f381a10b0 con 0x7f2f381037f0 2024-05-20T20:07:04.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.392+0000 7f2f3ca00700 1 -- 172.21.15.28:0/3878129684 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f2800cbb0 con 0x7f2f381037f0 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.392+0000 7f2f3ca00700 1 -- 172.21.15.28:0/3878129684 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f28018680 con 0x7f2f381037f0 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.392+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2f381a8930 con 0x7f2f381037f0 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.398+0000 7f2f3ca00700 1 -- 172.21.15.28:0/3878129684 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f2f28010430 con 0x7f2f381037f0 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.398+0000 7f2f487eab80 1 Processor -- start 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.399+0000 7f2f487eab80 1 -- start start 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.399+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032de69f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.399+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032de0b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.399+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de1140 0x561032de15c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.399+0000 7f2f487eab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561032d3ee40 con 0x561032de6610 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.399+0000 7f2f487eab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561032d2e9e0 con 0x561032de6f30 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.399+0000 7f2f487eab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561032d32a20 con 0x561032de1140 2024-05-20T20:07:04.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032de0b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f3ca00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f1c0b2ba0 0x7f2f1c0b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032de0b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48090/0 (socket says 172.21.15.28:48090) 2024-05-20T20:07:04.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36a00700 1 -- 172.21.15.28:0/403969861 learned_addr learned my addr 172.21.15.28:0/403969861 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36000700 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de1140 0x561032de15c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f37400700 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032de69f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f3ca00700 1 -- 172.21.15.28:0/3878129684 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2f280e2a30 con 0x7f2f381037f0 2024-05-20T20:07:04.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f37400700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f1c0b2ba0 0x7f2f1c0b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36000700 1 -- 172.21.15.28:0/403969861 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 msgr2=0x561032de0b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36000700 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032de0b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36000700 1 -- 172.21.15.28:0/403969861 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 msgr2=0x561032de69f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36000700 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032de69f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.400+0000 7f2f36000700 1 -- 172.21.15.28:0/403969861 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561032cf7420 con 0x561032de1140 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.401+0000 7f2f37400700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f1c0b2ba0 0x7f2f1c0b5060 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f2f30009fd0 tx=0x7f2f30009100 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.401+0000 7f2f36000700 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de1140 0x561032de15c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f2f24009990 tx=0x7f2f24005bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e57815114781876d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.401+0000 7f2f2cc00700 1 -- 172.21.15.28:0/403969861 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f24028070 con 0x561032de1140 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.401+0000 7f2f2cc00700 1 -- 172.21.15.28:0/403969861 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f240037c0 con 0x561032de1140 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.401+0000 7f2f2cc00700 1 -- 172.21.15.28:0/403969861 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f2400b9f0 con 0x561032de1140 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 -- 172.21.15.28:0/403969861 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de1140 msgr2=0x561032de15c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de1140 0x561032de15c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f2f24009990 tx=0x7f2f24005bc0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 -- 172.21.15.28:0/403969861 shutdown_connections 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de1140 0x561032de15c0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032de69f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 --2- 172.21.15.28:0/403969861 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032de0b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 -- 172.21.15.28:0/403969861 >> 172.21.15.28:0/403969861 conn(0x561032d0ebe0 msgr2=0x561032ddb690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.402+0000 7f2f487eab80 1 -- 172.21.15.28:0/403969861 shutdown_connections 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.403+0000 7f2f487eab80 1 -- 172.21.15.28:0/403969861 wait complete. 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.403+0000 7f2f487eab80 1 -- start start 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.403+0000 7f2f487eab80 1 Processor -- start 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032decce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6610 0x561032dea080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 0x561032dea5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f487eab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561032c34fa0 con 0x561032de1140 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f487eab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561032ccf760 con 0x561032de6610 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f487eab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561032ded980 con 0x561032de6f30 2024-05-20T20:07:04.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f487eab80 10 client.0 _ll_register_callbacks cb 0x561032cd8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032decce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032decce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41624/0 (socket says 172.21.15.28:41624) 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f36a00700 1 -- 172.21.15.28:0/2480598310 learned_addr learned my addr 172.21.15.28:0/2480598310 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.404+0000 7f2f36000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6610 0x561032dea080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f36a00700 1 -- 172.21.15.28:0/2480598310 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 msgr2=0x561032dea5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f36a00700 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 0x561032dea5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f36a00700 1 -- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6610 msgr2=0x561032dea080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f36a00700 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6610 0x561032dea080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f36a00700 1 -- 172.21.15.28:0/2480598310 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561032c7e8d0 con 0x561032de1140 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f36000700 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6610 0x561032dea080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f36a00700 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032decce0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f2f280a5b60 tx=0x7f2f280b2790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ee489284043ca9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f21600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:04.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f21600700 1 -- 172.21.15.28:0/2480598310 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f280a5270 con 0x561032de1140 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f21600700 1 -- 172.21.15.28:0/2480598310 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2f280a8f50 con 0x561032de1140 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.405+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2f240037c0 con 0x561032de1140 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f21600700 1 -- 172.21.15.28:0/2480598310 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2f280a95f0 con 0x561032de1140 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 10 client.15498 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561032cff060 con 0x561032de1140 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f21600700 1 -- 172.21.15.28:0/2480598310 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f240037c0 con 0x561032de1140 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f21600700 1 -- 172.21.15.28:0/2480598310 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f2f280a5450 con 0x561032de1140 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 10 client.15498 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 10 client.15498 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 20 client.15498 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032dfa5a0 0x561032dfc990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 4 client.15498 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:04.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.406+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561032d40d90 con 0x561032dfa5a0 2024-05-20T20:07:04.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.407+0000 7f2f36000700 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032dfa5a0 0x561032dfc990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.408+0000 7f2f36000700 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032dfa5a0 0x561032dfc990 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ae8b31042e53f762 server_cookie=a05d6f52aafc1708 in_seq=0 out_seq=0 2024-05-20T20:07:04.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.408+0000 7f2f21600700 10 client.15498 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.408+0000 7f2f21600700 1 -- 172.21.15.28:0/2480598310 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561032d40d90 con 0x561032dfa5a0 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.408+0000 7f2f21600700 10 client.15498 handle_command_reply: tid=0 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.408+0000 7f2f487eab80 1 client.15498 shutdown 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 msgr2=0x561032decce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032decce0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f2f280a5b60 tx=0x7f2f280b2790 comp rx=0 tx=0).stop 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 shutdown_connections 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032dfa5a0 0x561032dfc990 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 0x561032dea5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f21600700 0 client.15498 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032decce0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2480598310 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6610 0x561032dea080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 >> 172.21.15.28:0/2480598310 conn(0x561032d0ebe0 msgr2=0x561032ddad70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 shutdown_connections 2024-05-20T20:07:04.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 1 -- 172.21.15.28:0/2480598310 wait complete. 2024-05-20T20:07:04.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.409+0000 7f2f487eab80 20 client.15498 trim_cache size 0 max 16384 2024-05-20T20:07:04.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.426+0000 7f2f487eab80 1 Processor -- start 2024-05-20T20:07:04.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.426+0000 7f2f487eab80 1 -- start start 2024-05-20T20:07:04.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de1140 0x561032de0b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032decce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 0x561032de8fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f487eab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381aff90 con 0x561032de6610 2024-05-20T20:07:04.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f487eab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381b1ad0 con 0x561032de1140 2024-05-20T20:07:04.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f487eab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381b0660 con 0x561032de6f30 2024-05-20T20:07:04.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f36000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032decce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de1140 0x561032de0b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f36000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032decce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41634/0 (socket says 172.21.15.28:41634) 2024-05-20T20:07:04.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f36000700 1 -- 172.21.15.28:0/2618131182 learned_addr learned my addr 172.21.15.28:0/2618131182 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.427+0000 7f2f37400700 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 0x561032de8fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f36a00700 1 -- 172.21.15.28:0/2618131182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 msgr2=0x561032de8fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f36a00700 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 0x561032de8fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f36a00700 1 -- 172.21.15.28:0/2618131182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 msgr2=0x561032decce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f36a00700 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032decce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f36a00700 1 -- 172.21.15.28:0/2618131182 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561032dfcfc0 con 0x561032de1140 2024-05-20T20:07:04.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f36a00700 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de1140 0x561032de0b20 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f2f28000c00 tx=0x7f2f280e00e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6d5c7b23efc0bcd4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f23e00700 1 -- 172.21.15.28:0/2618131182 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f280a5270 con 0x561032de1140 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f23e00700 1 -- 172.21.15.28:0/2618131182 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f280aa1e0 con 0x561032de1140 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.428+0000 7f2f23e00700 1 -- 172.21.15.28:0/2618131182 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f280a8970 con 0x561032de1140 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 -- 172.21.15.28:0/2618131182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de1140 msgr2=0x561032de0b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de1140 0x561032de0b20 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f2f28000c00 tx=0x7f2f280e00e0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 -- 172.21.15.28:0/2618131182 shutdown_connections 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6f30 0x561032de8fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de1140 0x561032de0b20 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 --2- 172.21.15.28:0/2618131182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de6610 0x561032decce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 -- 172.21.15.28:0/2618131182 >> 172.21.15.28:0/2618131182 conn(0x561032d0ebe0 msgr2=0x561032de7310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 -- 172.21.15.28:0/2618131182 shutdown_connections 2024-05-20T20:07:04.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.429+0000 7f2f487eab80 1 -- 172.21.15.28:0/2618131182 wait complete. 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 -- start start 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 Processor -- start 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032df2ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6610 0x561032df3410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032e01720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381b1400 con 0x561032de1140 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381b0d30 con 0x561032de6f30 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.430+0000 7f2f487eab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2f381adf30 con 0x561032de6610 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f487eab80 10 client.0 _ll_register_callbacks cb 0x561032cd8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:04.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032e01720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f36000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6610 0x561032df3410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f36a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032df2ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032e01720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:48128/0 (socket says 172.21.15.28:48128) 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 -- 172.21.15.28:0/3892877907 learned_addr learned my addr 172.21.15.28:0/3892877907 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f36000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6610 0x561032df3410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60838/0 (socket says 172.21.15.28:60838) 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 -- 172.21.15.28:0/3892877907 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6610 msgr2=0x561032df3410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6610 0x561032df3410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 -- 172.21.15.28:0/3892877907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 msgr2=0x561032df2ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032df2ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.431+0000 7f2f37400700 1 -- 172.21.15.28:0/3892877907 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561032e01f30 con 0x561032de6f30 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f37400700 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032e01720 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f2f30019730 tx=0x7f2f30009900 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ec4f57c3e8dc6f30 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f21600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f21600700 1 -- 172.21.15.28:0/3892877907 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f30029030 con 0x561032de6f30 2024-05-20T20:07:04.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2f280aa1e0 con 0x561032de6f30 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f21600700 1 -- 172.21.15.28:0/3892877907 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2f30022040 con 0x561032de6f30 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f21600700 1 -- 172.21.15.28:0/3892877907 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2f30033050 con 0x561032de6f30 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f487eab80 10 client.25649 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.432+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561032e6d5c0 con 0x561032de6f30 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f21600700 1 -- 172.21.15.28:0/3892877907 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f280aa1e0 con 0x561032de6f30 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f21600700 1 -- 172.21.15.28:0/3892877907 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f2f30005020 con 0x561032de6f30 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f487eab80 10 client.25649 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f487eab80 10 client.25649 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f487eab80 20 client.25649 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f487eab80 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032e6dd00 0x561032e70120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f487eab80 4 client.25649 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:04.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x561032d3a570 con 0x561032e6dd00 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.433+0000 7f2f36a00700 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032e6dd00 0x561032e70120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.434+0000 7f2f36a00700 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032e6dd00 0x561032e70120 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e1f8153fc4995cc7 server_cookie=f6032fef3e773837 in_seq=0 out_seq=0 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.434+0000 7f2f21600700 10 client.25649 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.434+0000 7f2f21600700 1 -- 172.21.15.28:0/3892877907 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x561032d3a570 con 0x561032e6dd00 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.434+0000 7f2f21600700 10 client.25649 handle_command_reply: tid=0 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.434+0000 7f2f487eab80 1 client.25649 shutdown 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 msgr2=0x561032e01720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032e01720 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f2f30019730 tx=0x7f2f30009900 comp rx=0 tx=0).stop 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 shutdown_connections 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561032e6dd00 0x561032e70120 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561032de6f30 0x561032e01720 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561032de1140 0x561032df2ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 --2- 172.21.15.28:0/3892877907 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561032de6610 0x561032df3410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 >> 172.21.15.28:0/3892877907 conn(0x561032d0ebe0 msgr2=0x561032dea570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f21600700 0 client.25649 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:04.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 shutdown_connections 2024-05-20T20:07:04.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.435+0000 7f2f487eab80 1 -- 172.21.15.28:0/3892877907 wait complete. 2024-05-20T20:07:04.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.436+0000 7f2f487eab80 20 client.25649 trim_cache size 0 max 16384 2024-05-20T20:07:04.440 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.436+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f1c0b2ba0 msgr2=0x7f2f1c0b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.436+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f1c0b2ba0 0x7f2f1c0b5060 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f2f30009fd0 tx=0x7f2f30009100 comp rx=0 tx=0).stop 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.436+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 msgr2=0x7f2f3819bdb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.436+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 0x7f2f3819bdb0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f2f2800e9a0 tx=0x7f2f2800ecb0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 shutdown_connections 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f381088f0 0x7f2f381a0680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f381037f0 0x7f2f3819bdb0 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f1c0b2ba0 0x7f2f1c0b5060 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 --2- 172.21.15.28:0/3878129684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f381041a0 0x7f2f3819c2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:04.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 >> 172.21.15.28:0/3878129684 conn(0x7f2f3807a6a0 msgr2=0x7f2f381018f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:04.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 shutdown_connections 2024-05-20T20:07:04.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:04.437+0000 7f2f3de00700 1 -- 172.21.15.28:0/3878129684 wait complete. 2024-05-20T20:07:04.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:04 smithi028 ceph-mon[31178]: pgmap v46: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 24 KiB/s wr, 28 op/s 2024-05-20T20:07:05.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:04 smithi154 ceph-mon[36314]: pgmap v46: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 24 KiB/s wr, 28 op/s 2024-05-20T20:07:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:04 smithi033 ceph-mon[36179]: pgmap v46: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 24 KiB/s wr, 28 op/s 2024-05-20T20:07:05.419 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:07:05.941 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:07.022 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:06 smithi028 ceph-mon[31178]: pgmap v47: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 24 KiB/s wr, 23 op/s 2024-05-20T20:07:07.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:06 smithi154 ceph-mon[36314]: pgmap v47: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 24 KiB/s wr, 23 op/s 2024-05-20T20:07:07.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:06 smithi033 ceph-mon[36179]: pgmap v47: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 24 KiB/s wr, 23 op/s 2024-05-20T20:07:08.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 -- 172.21.15.28:0/2647442578 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0109540 msgr2=0x7f43e0109920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 --2- 172.21.15.28:0/2647442578 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0109540 0x7f43e0109920 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f43d40099c0 tx=0x7f43d4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 -- 172.21.15.28:0/2647442578 shutdown_connections 2024-05-20T20:07:08.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 --2- 172.21.15.28:0/2647442578 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43e0103e50 0x7f43e0110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 --2- 172.21.15.28:0/2647442578 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0109540 0x7f43e0109920 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 --2- 172.21.15.28:0/2647442578 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 0x7f43e0103910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 -- 172.21.15.28:0/2647442578 >> 172.21.15.28:0/2647442578 conn(0x7f43e00fefc0 msgr2=0x7f43e01013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:08.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 -- 172.21.15.28:0/2647442578 shutdown_connections 2024-05-20T20:07:08.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.165+0000 7f43e6800700 1 -- 172.21.15.28:0/2647442578 wait complete. 2024-05-20T20:07:08.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.166+0000 7f43e6800700 1 Processor -- start 2024-05-20T20:07:08.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.166+0000 7f43e6800700 1 -- start start 2024-05-20T20:07:08.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e6800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 0x7f43e01a0980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e6800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0103e50 0x7f43e01a0ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e6800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43e0109540 0x7f43e019bec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e6800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f43e0076070 con 0x7f43e0103e50 2024-05-20T20:07:08.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e6800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f43e0075ef0 con 0x7f43e0109540 2024-05-20T20:07:08.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e6800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f43e00761f0 con 0x7f43e0103490 2024-05-20T20:07:08.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 0x7f43e01a0980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0103e50 0x7f43e01a0ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 0x7f43e01a0980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52438/0 (socket says 172.21.15.28:52438) 2024-05-20T20:07:08.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e5400700 1 -- 172.21.15.28:0/1880601728 learned_addr learned my addr 172.21.15.28:0/1880601728 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:08.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.167+0000 7f43e5e00700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43e0109540 0x7f43e019bec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e5400700 1 -- 172.21.15.28:0/1880601728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43e0109540 msgr2=0x7f43e019bec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e5400700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43e0109540 0x7f43e019bec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e5400700 1 -- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0103e50 msgr2=0x7f43e01a0ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e5400700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0103e50 0x7f43e01a0ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e5400700 1 -- 172.21.15.28:0/1880601728 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f43e019c660 con 0x7f43e0103490 2024-05-20T20:07:08.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e5400700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 0x7f43e01a0980 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f43d4006010 tx=0x7f43d40045e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:08.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43de000700 1 -- 172.21.15.28:0/1880601728 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43d4026070 con 0x7f43e0103490 2024-05-20T20:07:08.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43de000700 1 -- 172.21.15.28:0/1880601728 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43d400f460 con 0x7f43e0103490 2024-05-20T20:07:08.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f43e019c8f0 con 0x7f43e0103490 2024-05-20T20:07:08.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43de000700 1 -- 172.21.15.28:0/1880601728 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43d4009e60 con 0x7f43e0103490 2024-05-20T20:07:08.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.168+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f43e01a88b0 con 0x7f43e0103490 2024-05-20T20:07:08.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.172+0000 7f43de000700 1 -- 172.21.15.28:0/1880601728 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 26) ==== 149936+0+0 (secure 0 0 0) 0x7f43d4029070 con 0x7f43e0103490 2024-05-20T20:07:08.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 Processor -- start 2024-05-20T20:07:08.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 -- start start 2024-05-20T20:07:08.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823a8ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823a2d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563282300e40 con 0x5632823a87c0 2024-05-20T20:07:08.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5632822f09e0 con 0x5632823a90e0 2024-05-20T20:07:08.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.173+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5632822f4a20 con 0x5632823a3390 2024-05-20T20:07:08.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e5e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823a8ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823a2d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823a37f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52442/0 (socket says 172.21.15.28:52442) 2024-05-20T20:07:08.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 -- 172.21.15.28:0/320134663 learned_addr learned my addr 172.21.15.28:0/320134663 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:08.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43de000700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43c80b2ba0 0x7f43c80b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 -- 172.21.15.28:0/320134663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 msgr2=0x5632823a2d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823a2d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 -- 172.21.15.28:0/320134663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 msgr2=0x5632823a8ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823a8ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e4a00700 1 -- 172.21.15.28:0/320134663 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5632822b9420 con 0x5632823a3390 2024-05-20T20:07:08.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.174+0000 7f43e5400700 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823a2d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:08.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.175+0000 7f43de000700 1 -- 172.21.15.28:0/1880601728 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f43d40e3090 con 0x7f43e0103490 2024-05-20T20:07:08.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.175+0000 7f43e5e00700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43c80b2ba0 0x7f43c80b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.175+0000 7f43e4a00700 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823a37f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f43cc009a80 tx=0x7f43cc009d90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=72ed5dbfaf97d5fc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:08.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.175+0000 7f43d2a00700 1 -- 172.21.15.28:0/320134663 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43cc017070 con 0x5632823a3390 2024-05-20T20:07:08.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.175+0000 7f43d2a00700 1 -- 172.21.15.28:0/320134663 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43cc00ed80 con 0x5632823a3390 2024-05-20T20:07:08.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.175+0000 7f43d2a00700 1 -- 172.21.15.28:0/320134663 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43cc013da0 con 0x5632823a3390 2024-05-20T20:07:08.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 -- 172.21.15.28:0/320134663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 msgr2=0x5632823a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823a37f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f43cc009a80 tx=0x7f43cc009d90 comp rx=0 tx=0).stop 2024-05-20T20:07:08.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43e5e00700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43c80b2ba0 0x7f43c80b5060 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f43d8009fd0 tx=0x7f43d8009040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:08.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 -- 172.21.15.28:0/320134663 shutdown_connections 2024-05-20T20:07:08.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823a37f0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823a8ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/320134663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823a2d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 -- 172.21.15.28:0/320134663 >> 172.21.15.28:0/320134663 conn(0x5632822d0be0 msgr2=0x56328239d810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:08.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 -- 172.21.15.28:0/320134663 shutdown_connections 2024-05-20T20:07:08.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.176+0000 7f43f27dfb80 1 -- 172.21.15.28:0/320134663 wait complete. 2024-05-20T20:07:08.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.177+0000 7f43f27dfb80 1 -- start start 2024-05-20T20:07:08.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.177+0000 7f43f27dfb80 1 Processor -- start 2024-05-20T20:07:08.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823aede0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823ac180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823ac6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5632821f6fa0 con 0x5632823a87c0 2024-05-20T20:07:08.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563282291760 con 0x5632823a90e0 2024-05-20T20:07:08.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5632823afc00 con 0x5632823a3390 2024-05-20T20:07:08.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43f27dfb80 10 client.0 _ll_register_callbacks cb 0x56328229ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:08.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823ac180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823ac180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36678/0 (socket says 172.21.15.28:36678) 2024-05-20T20:07:08.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e5e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823ac6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823aede0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e4a00700 1 -- 172.21.15.28:0/1295875114 learned_addr learned my addr 172.21.15.28:0/1295875114 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:08.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e5400700 1 -- 172.21.15.28:0/1295875114 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 msgr2=0x5632823ac6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e5400700 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823ac6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e5400700 1 -- 172.21.15.28:0/1295875114 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 msgr2=0x5632823ac180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e5400700 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823ac180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e5400700 1 -- 172.21.15.28:0/1295875114 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5632822408d0 con 0x5632823a3390 2024-05-20T20:07:08.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.178+0000 7f43e4a00700 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823ac180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:08.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43e5400700 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823aede0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f43d40e05b0 tx=0x7f43d4009380 comp rx=0 tx=0).ready entity=mon.1 client_cookie=31390a60eacd58d4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:08.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43c7400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:07:08.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43c7400700 1 -- 172.21.15.28:0/1295875114 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43d40a9fa0 con 0x5632823a3390 2024-05-20T20:07:08.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43c7400700 1 -- 172.21.15.28:0/1295875114 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f43d40aa430 con 0x5632823a3390 2024-05-20T20:07:08.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f43cc00ed80 con 0x5632823a3390 2024-05-20T20:07:08.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43c7400700 1 -- 172.21.15.28:0/1295875114 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f43d40a9940 con 0x5632823a3390 2024-05-20T20:07:08.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43f27dfb80 10 client.25627 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:08.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.179+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5632822c1060 con 0x5632823a3390 2024-05-20T20:07:08.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43c7400700 1 -- 172.21.15.28:0/1295875114 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43cc00ed80 con 0x5632823a3390 2024-05-20T20:07:08.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43c7400700 1 -- 172.21.15.28:0/1295875114 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f43d40ae0f0 con 0x5632823a3390 2024-05-20T20:07:08.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43f27dfb80 10 client.25627 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:08.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43f27dfb80 10 client.25627 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:08.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43f27dfb80 20 client.25627 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:08.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632823bc710 0x5632823beb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43f27dfb80 4 client.25627 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:08.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563282302d90 con 0x5632823bc710 2024-05-20T20:07:08.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.180+0000 7f43e4a00700 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632823bc710 0x5632823beb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.181+0000 7f43e4a00700 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632823bc710 0x5632823beb60 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2efe87a154fb8cb0 server_cookie=43f506c08749c173 in_seq=0 out_seq=0 2024-05-20T20:07:08.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.181+0000 7f43c7400700 10 client.25627 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:08.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43c7400700 1 -- 172.21.15.28:0/1295875114 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x563282302d90 con 0x5632823bc710 2024-05-20T20:07:08.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43c7400700 10 client.25627 handle_command_reply: tid=0 2024-05-20T20:07:08.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 client.25627 shutdown 2024-05-20T20:07:08.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 msgr2=0x5632823aede0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823aede0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f43d40e05b0 tx=0x7f43d4009380 comp rx=0 tx=0).stop 2024-05-20T20:07:08.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 shutdown_connections 2024-05-20T20:07:08.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632823bc710 0x5632823beb60 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823ac6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823aede0 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/1295875114 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823ac180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43c7400700 0 client.25627 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:08.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 >> 172.21.15.28:0/1295875114 conn(0x5632822d0be0 msgr2=0x56328239cfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:08.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 shutdown_connections 2024-05-20T20:07:08.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 1 -- 172.21.15.28:0/1295875114 wait complete. 2024-05-20T20:07:08.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.182+0000 7f43f27dfb80 20 client.25627 trim_cache size 0 max 16384 2024-05-20T20:07:08.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 Processor -- start 2024-05-20T20:07:08.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 -- start start 2024-05-20T20:07:08.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a3390 0x5632823a2d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a87c0 0x5632823ab0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a90e0 0x5632823a9bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f43e01afca0 con 0x5632823a3390 2024-05-20T20:07:08.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f43e01b17e0 con 0x5632823a87c0 2024-05-20T20:07:08.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.199+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f43e01b0370 con 0x5632823a90e0 2024-05-20T20:07:08.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a3390 0x5632823a2d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a3390 0x5632823a2d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36686/0 (socket says 172.21.15.28:36686) 2024-05-20T20:07:08.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 -- 172.21.15.28:0/2706836156 learned_addr learned my addr 172.21.15.28:0/2706836156 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:08.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5e00700 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a90e0 0x5632823a9bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e4a00700 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a87c0 0x5632823ab0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 -- 172.21.15.28:0/2706836156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a90e0 msgr2=0x5632823a9bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a90e0 0x5632823a9bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 -- 172.21.15.28:0/2706836156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a87c0 msgr2=0x5632823ab0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a87c0 0x5632823ab0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 -- 172.21.15.28:0/2706836156 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5632823aa6f0 con 0x5632823a3390 2024-05-20T20:07:08.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e4a00700 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a87c0 0x5632823ab0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:08.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.200+0000 7f43e5400700 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a3390 0x5632823a2d70 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f43d40059d0 tx=0x7f43d40ae650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=887aad9fba512624 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:08.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43d2000700 1 -- 172.21.15.28:0/2706836156 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43d40a9b20 con 0x5632823a3390 2024-05-20T20:07:08.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43d2000700 1 -- 172.21.15.28:0/2706836156 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43d40a9ce0 con 0x5632823a3390 2024-05-20T20:07:08.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43d2000700 1 -- 172.21.15.28:0/2706836156 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43d40aa690 con 0x5632823a3390 2024-05-20T20:07:08.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 -- 172.21.15.28:0/2706836156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a3390 msgr2=0x5632823a2d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a3390 0x5632823a2d70 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f43d40059d0 tx=0x7f43d40ae650 comp rx=0 tx=0).stop 2024-05-20T20:07:08.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 -- 172.21.15.28:0/2706836156 shutdown_connections 2024-05-20T20:07:08.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a90e0 0x5632823a9bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a3390 0x5632823a2d70 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/2706836156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a87c0 0x5632823ab0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 -- 172.21.15.28:0/2706836156 >> 172.21.15.28:0/2706836156 conn(0x5632822d0be0 msgr2=0x5632823a94c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:08.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.201+0000 7f43f27dfb80 1 -- 172.21.15.28:0/2706836156 shutdown_connections 2024-05-20T20:07:08.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.202+0000 7f43f27dfb80 1 -- 172.21.15.28:0/2706836156 wait complete. 2024-05-20T20:07:08.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.202+0000 7f43f27dfb80 1 -- start start 2024-05-20T20:07:08.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.202+0000 7f43f27dfb80 1 Processor -- start 2024-05-20T20:07:08.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823b5440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823c3b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43f27dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823c4080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f43e01b1110 con 0x5632823a87c0 2024-05-20T20:07:08.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f43e01b0a40 con 0x5632823a90e0 2024-05-20T20:07:08.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43f27dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f43e01adc60 con 0x5632823a3390 2024-05-20T20:07:08.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43f27dfb80 10 client.0 _ll_register_callbacks cb 0x56328229ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:08.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823c3b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e5e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823c4080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823b5440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823c3b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36690/0 (socket says 172.21.15.28:36690) 2024-05-20T20:07:08.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 -- 172.21.15.28:0/965219581 learned_addr learned my addr 172.21.15.28:0/965219581 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:08.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 -- 172.21.15.28:0/965219581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 msgr2=0x5632823b5440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823b5440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 -- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 msgr2=0x5632823c4080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:08.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823c4080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e4a00700 1 -- 172.21.15.28:0/965219581 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5632823c4700 con 0x5632823a87c0 2024-05-20T20:07:08.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.203+0000 7f43e5e00700 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823c4080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:08.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43e4a00700 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823c3b40 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f43cc008010 tx=0x7f43cc009530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5e92e3e38114b1de server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:08.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43c7400700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:08.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43c7400700 1 -- 172.21.15.28:0/965219581 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43cc02f060 con 0x5632823a87c0 2024-05-20T20:07:08.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43c7400700 1 -- 172.21.15.28:0/965219581 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f43cc013a50 con 0x5632823a87c0 2024-05-20T20:07:08.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f43d40a9ce0 con 0x5632823a87c0 2024-05-20T20:07:08.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43c7400700 1 -- 172.21.15.28:0/965219581 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f43cc017040 con 0x5632823a87c0 2024-05-20T20:07:08.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43f27dfb80 10 client.15522 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:08.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.204+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563282427170 con 0x5632823a87c0 2024-05-20T20:07:08.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43c7400700 1 -- 172.21.15.28:0/965219581 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43d40a9ce0 con 0x5632823a87c0 2024-05-20T20:07:08.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43c7400700 1 -- 172.21.15.28:0/965219581 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f43cc029550 con 0x5632823a87c0 2024-05-20T20:07:08.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43f27dfb80 10 client.15522 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:08.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43f27dfb80 10 client.15522 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:08.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43f27dfb80 20 client.15522 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:08.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632824275d0 0x5632824299f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:08.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43f27dfb80 4 client.15522 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:08.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5632822fc570 con 0x5632824275d0 2024-05-20T20:07:08.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.205+0000 7f43e5400700 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632824275d0 0x5632824299f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:08.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43e5400700 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632824275d0 0x5632824299f0 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5b689a577338c827 server_cookie=73a0320165e6f7c8 in_seq=0 out_seq=0 2024-05-20T20:07:08.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43c7400700 10 client.15522 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:08.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43c7400700 1 -- 172.21.15.28:0/965219581 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5632822fc570 con 0x5632824275d0 2024-05-20T20:07:08.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43c7400700 10 client.15522 handle_command_reply: tid=0 2024-05-20T20:07:08.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 client.15522 shutdown 2024-05-20T20:07:08.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 msgr2=0x5632823c3b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823c3b40 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f43cc008010 tx=0x7f43cc009530 comp rx=0 tx=0).stop 2024-05-20T20:07:08.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 shutdown_connections 2024-05-20T20:07:08.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5632824275d0 0x5632824299f0 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5632823a90e0 0x5632823c4080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5632823a3390 0x5632823b5440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 --2- 172.21.15.28:0/965219581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5632823a87c0 0x5632823c3b40 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 >> 172.21.15.28:0/965219581 conn(0x5632822d0be0 msgr2=0x5632823ad7a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:08.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.206+0000 7f43c7400700 0 client.15522 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:08.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.207+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 shutdown_connections 2024-05-20T20:07:08.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.207+0000 7f43f27dfb80 1 -- 172.21.15.28:0/965219581 wait complete. 2024-05-20T20:07:08.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.207+0000 7f43f27dfb80 20 client.15522 trim_cache size 0 max 16384 2024-05-20T20:07:08.219 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:08.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.207+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43c80b2ba0 msgr2=0x7f43c80b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.207+0000 7f43e6800700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43c80b2ba0 0x7f43c80b5060 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f43d8009fd0 tx=0x7f43d8009040 comp rx=0 tx=0).stop 2024-05-20T20:07:08.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.207+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 msgr2=0x7f43e01a0980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:08.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.207+0000 7f43e6800700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 0x7f43e01a0980 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f43d4006010 tx=0x7f43d40045e0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 shutdown_connections 2024-05-20T20:07:08.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43c80b2ba0 0x7f43c80b5060 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f43e0109540 0x7f43e019bec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43e0103490 0x7f43e01a0980 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 --2- 172.21.15.28:0/1880601728 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f43e0103e50 0x7f43e01a0ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:08.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 >> 172.21.15.28:0/1880601728 conn(0x7f43e00fefc0 msgr2=0x7f43e00ff500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:08.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 shutdown_connections 2024-05-20T20:07:08.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:08.208+0000 7f43e6800700 1 -- 172.21.15.28:0/1880601728 wait complete. 2024-05-20T20:07:08.961 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:08 smithi028 ceph-mon[31178]: pgmap v48: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 24 KiB/s wr, 23 op/s 2024-05-20T20:07:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:08 smithi154 ceph-mon[36314]: pgmap v48: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 24 KiB/s wr, 23 op/s 2024-05-20T20:07:09.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:08 smithi033 ceph-mon[36179]: pgmap v48: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 24 KiB/s wr, 23 op/s 2024-05-20T20:07:10.025 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:07:10.026 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:07:10.504 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:10.998 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:10 smithi028 ceph-mon[31178]: pgmap v49: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 26 KiB/s wr, 31 op/s 2024-05-20T20:07:10.998 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:10 smithi028 ceph-mon[31178]: mgrmap e27: x(active, since 92s), standbys: z, y 2024-05-20T20:07:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:10 smithi154 ceph-mon[36314]: pgmap v49: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 26 KiB/s wr, 31 op/s 2024-05-20T20:07:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:10 smithi154 ceph-mon[36314]: mgrmap e27: x(active, since 92s), standbys: z, y 2024-05-20T20:07:11.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:10 smithi033 ceph-mon[36179]: pgmap v49: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 26 KiB/s wr, 31 op/s 2024-05-20T20:07:11.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:10 smithi033 ceph-mon[36179]: mgrmap e27: x(active, since 92s), standbys: z, y 2024-05-20T20:07:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.722+0000 7f317ee00700 1 -- 172.21.15.28:0/139915821 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f31781046f0 msgr2=0x7f3178104ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.722+0000 7f317ee00700 1 --2- 172.21.15.28:0/139915821 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f31781046f0 0x7f3178104ad0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f316c00b0a0 tx=0x7f316c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.722+0000 7f317ee00700 1 -- 172.21.15.28:0/139915821 shutdown_connections 2024-05-20T20:07:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.722+0000 7f317ee00700 1 --2- 172.21.15.28:0/139915821 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f317810dad0 0x7f317810ff20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.722+0000 7f317ee00700 1 --2- 172.21.15.28:0/139915821 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f31781046f0 0x7f3178104ad0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.722+0000 7f317ee00700 1 --2- 172.21.15.28:0/139915821 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3178105010 0x7f317810d500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.722+0000 7f317ee00700 1 -- 172.21.15.28:0/139915821 >> 172.21.15.28:0/139915821 conn(0x7f31780fde20 msgr2=0x7f3178100240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.723+0000 7f317ee00700 1 -- 172.21.15.28:0/139915821 shutdown_connections 2024-05-20T20:07:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.723+0000 7f317ee00700 1 -- 172.21.15.28:0/139915821 wait complete. 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 Processor -- start 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 -- start start 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f31781046f0 0x7f3178102350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3178105010 0x7f3178102890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f317810dad0 0x7f3178102dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f31781136d0 con 0x7f317810dad0 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3178113550 con 0x7f3178105010 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.724+0000 7f317ee00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3178113850 con 0x7f31781046f0 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3178105010 0x7f3178102890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317e400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f317810dad0 0x7f3178102dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3178105010 0x7f3178102890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49304/0 (socket says 172.21.15.28:49304) 2024-05-20T20:07:12.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 -- 172.21.15.28:0/920010106 learned_addr learned my addr 172.21.15.28:0/920010106 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:12.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317da00700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f31781046f0 0x7f3178102350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 -- 172.21.15.28:0/920010106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f31781046f0 msgr2=0x7f3178102350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f31781046f0 0x7f3178102350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 -- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f317810dad0 msgr2=0x7f3178102dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f317810dad0 0x7f3178102dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.725+0000 7f317d000700 1 -- 172.21.15.28:0/920010106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f31781a8120 con 0x7f3178105010 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.726+0000 7f317d000700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3178105010 0x7f3178102890 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f316400e9f0 tx=0x7f316400ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.726+0000 7f3176000700 1 -- 172.21.15.28:0/920010106 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3164004a30 con 0x7f3178105010 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.726+0000 7f3176000700 1 -- 172.21.15.28:0/920010106 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f316400cc70 con 0x7f3178105010 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.726+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f31781a8410 con 0x7f3178105010 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.726+0000 7f3176000700 1 -- 172.21.15.28:0/920010106 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3164005530 con 0x7f3178105010 2024-05-20T20:07:12.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.726+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f31781a8950 con 0x7f3178105010 2024-05-20T20:07:12.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.730+0000 7f3176000700 1 -- 172.21.15.28:0/920010106 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3164004bd0 con 0x7f3178105010 2024-05-20T20:07:12.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.731+0000 7f318ad63b80 1 Processor -- start 2024-05-20T20:07:12.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.731+0000 7f318ad63b80 1 -- start start 2024-05-20T20:07:12.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.731+0000 7f3176000700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f31600b2a70 0x7f31600b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f104b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f109f0 0x558aa6f0a600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6f0b0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f318ad63b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558aa6e5fec0 con 0x558aa6f0ac20 2024-05-20T20:07:12.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f318ad63b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558aa6e4fa60 con 0x558aa6f109f0 2024-05-20T20:07:12.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f318ad63b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558aa6e53aa0 con 0x558aa6f100d0 2024-05-20T20:07:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f3176000700 1 -- 172.21.15.28:0/920010106 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f31640e27f0 con 0x7f3178105010 2024-05-20T20:07:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f317e400700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f31600b2a70 0x7f31600b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f317e400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6f0b0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f317da00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f104b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f317e400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6f0b0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36720/0 (socket says 172.21.15.28:36720) 2024-05-20T20:07:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f317e400700 1 -- 172.21.15.28:0/2265541543 learned_addr learned my addr 172.21.15.28:0/2265541543 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.732+0000 7f317d000700 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f109f0 0x558aa6f0a600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317e400700 1 -- 172.21.15.28:0/2265541543 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 msgr2=0x558aa6f104b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317e400700 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f104b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317e400700 1 -- 172.21.15.28:0/2265541543 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f109f0 msgr2=0x558aa6f0a600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317e400700 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f109f0 0x558aa6f0a600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317e400700 1 -- 172.21.15.28:0/2265541543 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558aa6e184a0 con 0x558aa6f0ac20 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317d000700 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f109f0 0x558aa6f0a600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317e400700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f31600b2a70 0x7f31600b4f30 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f31700139d0 tx=0x7f3170009bf0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:12.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f317e400700 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6f0b0a0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f3170009940 tx=0x7f317001d000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=220f1cce141c63c8 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:12.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f316aa00700 1 -- 172.21.15.28:0/2265541543 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3170011e40 con 0x558aa6f0ac20 2024-05-20T20:07:12.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.733+0000 7f316aa00700 1 -- 172.21.15.28:0/2265541543 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3170013020 con 0x558aa6f0ac20 2024-05-20T20:07:12.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f316aa00700 1 -- 172.21.15.28:0/2265541543 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3170012dd0 con 0x558aa6f0ac20 2024-05-20T20:07:12.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f318ad63b80 1 -- 172.21.15.28:0/2265541543 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 msgr2=0x558aa6f0b0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6f0b0a0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f3170009940 tx=0x7f317001d000 comp rx=0 tx=0).stop 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f318ad63b80 1 -- 172.21.15.28:0/2265541543 shutdown_connections 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6f0b0a0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f104b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2265541543 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f109f0 0x558aa6f0a600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.734+0000 7f318ad63b80 1 -- 172.21.15.28:0/2265541543 >> 172.21.15.28:0/2265541543 conn(0x558aa6e2fc60 msgr2=0x558aa6efc890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.735+0000 7f318ad63b80 1 -- 172.21.15.28:0/2265541543 shutdown_connections 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.735+0000 7f318ad63b80 1 -- 172.21.15.28:0/2265541543 wait complete. 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.735+0000 7f318ad63b80 1 -- start start 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.735+0000 7f318ad63b80 1 Processor -- start 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f166f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f13a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f14450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f318ad63b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558aa6d56020 con 0x558aa6f100d0 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f318ad63b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558aa6df07e0 con 0x558aa6f0ac20 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f318ad63b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558aa6f17430 con 0x558aa6f13fd0 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f318ad63b80 10 client.0 _ll_register_callbacks cb 0x558aa6df9cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f317da00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f166f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f317da00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f166f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49318/0 (socket says 172.21.15.28:49318) 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f317da00700 1 -- 172.21.15.28:0/548704556 learned_addr learned my addr 172.21.15.28:0/548704556 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f317d000700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f13a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.736+0000 7f317e400700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f14450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f317e400700 1 -- 172.21.15.28:0/548704556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 msgr2=0x558aa6f166f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f317e400700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f166f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f317e400700 1 -- 172.21.15.28:0/548704556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 msgr2=0x558aa6f13a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f317e400700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f13a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f317e400700 1 -- 172.21.15.28:0/548704556 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558aa6d9f950 con 0x558aa6f13fd0 2024-05-20T20:07:12.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f317d000700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f13a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:12.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f317e400700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f14450 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f31700097a0 tx=0x7f3170013400 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d2a5e8eaca28fd7e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:12.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.737+0000 7f315f400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:07:12.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.738+0000 7f315f400700 1 -- 172.21.15.28:0/548704556 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3170012450 con 0x558aa6f13fd0 2024-05-20T20:07:12.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.738+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3170013020 con 0x558aa6f13fd0 2024-05-20T20:07:12.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.738+0000 7f315f400700 1 -- 172.21.15.28:0/548704556 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f31700125f0 con 0x558aa6f13fd0 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.738+0000 7f315f400700 1 -- 172.21.15.28:0/548704556 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f31700127f0 con 0x558aa6f13fd0 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.738+0000 7f318ad63b80 10 client.25645 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.738+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558aa6e200e0 con 0x558aa6f13fd0 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.738+0000 7f315f400700 1 -- 172.21.15.28:0/548704556 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3170013020 con 0x558aa6f13fd0 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f315f400700 1 -- 172.21.15.28:0/548704556 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3170020070 con 0x558aa6f13fd0 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f318ad63b80 10 client.25645 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f318ad63b80 10 client.25645 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f318ad63b80 20 client.25645 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f318ad63b80 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f242d0 0x558aa6f266c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f318ad63b80 4 client.25645 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558aa6e61e10 con 0x558aa6f242d0 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.739+0000 7f317da00700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f242d0 0x558aa6f266c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.740+0000 7f317da00700 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f242d0 0x558aa6f266c0 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3b03131d21ac5edc server_cookie=3c9f2610f67c7565 in_seq=0 out_seq=0 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.740+0000 7f315f400700 10 client.25645 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f315f400700 1 -- 172.21.15.28:0/548704556 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558aa6e61e10 con 0x558aa6f242d0 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f315f400700 10 client.25645 handle_command_reply: tid=0 2024-05-20T20:07:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 client.25645 shutdown 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 msgr2=0x558aa6f14450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f14450 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f31700097a0 tx=0x7f3170013400 comp rx=0 tx=0).stop 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 shutdown_connections 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f242d0 0x558aa6f266c0 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f315f400700 0 client.25645 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f14450 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f166f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 --2- 172.21.15.28:0/548704556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f13a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.741+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 >> 172.21.15.28:0/548704556 conn(0x558aa6e2fc60 msgr2=0x558aa6efc040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.742+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 shutdown_connections 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.742+0000 7f318ad63b80 1 -- 172.21.15.28:0/548704556 wait complete. 2024-05-20T20:07:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.742+0000 7f318ad63b80 20 client.25645 trim_cache size 0 max 16384 2024-05-20T20:07:12.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.767+0000 7f318ad63b80 1 Processor -- start 2024-05-20T20:07:12.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.767+0000 7f318ad63b80 1 -- start start 2024-05-20T20:07:12.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6efc7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f12980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f13fd0 0x558aa6f2bd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f318ad63b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f31781abb40 con 0x558aa6f0ac20 2024-05-20T20:07:12.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f318ad63b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f31781ad680 con 0x558aa6f13fd0 2024-05-20T20:07:12.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f318ad63b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f31781ac210 con 0x558aa6f100d0 2024-05-20T20:07:12.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f317d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f12980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f317d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f12980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52512/0 (socket says 172.21.15.28:52512) 2024-05-20T20:07:12.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f317da00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6efc7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f317d000700 1 -- 172.21.15.28:0/2985123446 learned_addr learned my addr 172.21.15.28:0/2985123446 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.768+0000 7f317e400700 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f13fd0 0x558aa6f2bd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f317d000700 1 -- 172.21.15.28:0/2985123446 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f13fd0 msgr2=0x558aa6f2bd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f317d000700 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f13fd0 0x558aa6f2bd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f317d000700 1 -- 172.21.15.28:0/2985123446 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 msgr2=0x558aa6efc7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f317d000700 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6efc7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f317d000700 1 -- 172.21.15.28:0/2985123446 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558aa6f0a850 con 0x558aa6f100d0 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f317d000700 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f12980 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f31640a9a10 tx=0x7f31640a9850 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1e3ad1e0d66a843 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f316a000700 1 -- 172.21.15.28:0/2985123446 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f31640a9590 con 0x558aa6f100d0 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f316a000700 1 -- 172.21.15.28:0/2985123446 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f316400ce20 con 0x558aa6f100d0 2024-05-20T20:07:12.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.769+0000 7f316a000700 1 -- 172.21.15.28:0/2985123446 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f31640b26b0 con 0x558aa6f100d0 2024-05-20T20:07:12.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 -- 172.21.15.28:0/2985123446 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 msgr2=0x558aa6f12980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f12980 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f31640a9a10 tx=0x7f31640a9850 comp rx=0 tx=0).stop 2024-05-20T20:07:12.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 -- 172.21.15.28:0/2985123446 shutdown_connections 2024-05-20T20:07:12.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f13fd0 0x558aa6f2bd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f0ac20 0x558aa6efc7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 --2- 172.21.15.28:0/2985123446 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f100d0 0x558aa6f12980 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 -- 172.21.15.28:0/2985123446 >> 172.21.15.28:0/2985123446 conn(0x558aa6e2fc60 msgr2=0x558aa6f12270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:12.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 -- 172.21.15.28:0/2985123446 shutdown_connections 2024-05-20T20:07:12.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.770+0000 7f318ad63b80 1 -- 172.21.15.28:0/2985123446 wait complete. 2024-05-20T20:07:12.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.771+0000 7f318ad63b80 1 -- start start 2024-05-20T20:07:12.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.771+0000 7f318ad63b80 1 Processor -- start 2024-05-20T20:07:12.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f1ceb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f1a2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f318ad63b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f1a900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f318ad63b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f31781acfb0 con 0x558aa6f100d0 2024-05-20T20:07:12.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f318ad63b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f31781ac8e0 con 0x558aa6f0ac20 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f318ad63b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f317819da50 con 0x558aa6f13fd0 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f318ad63b80 10 client.0 _ll_register_callbacks cb 0x558aa6df9cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f317e400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f1a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f317d000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f1a2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f317e400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f1a900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52524/0 (socket says 172.21.15.28:52524) 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f317e400700 1 -- 172.21.15.28:0/1275992337 learned_addr learned my addr 172.21.15.28:0/1275992337 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.772+0000 7f317da00700 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f1ceb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f317e400700 1 -- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 msgr2=0x558aa6f1ceb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f317e400700 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f1ceb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f317e400700 1 -- 172.21.15.28:0/1275992337 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 msgr2=0x558aa6f1a2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f317e400700 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f1a2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f317e400700 1 -- 172.21.15.28:0/1275992337 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558aa6f8fc70 con 0x558aa6f13fd0 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f317e400700 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f1a900 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f3170017010 tx=0x7f3170013560 comp rx=0 tx=0).ready entity=mon.1 client_cookie=822efa62abb7190d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f315f400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f315f400700 1 -- 172.21.15.28:0/1275992337 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3170012450 con 0x558aa6f13fd0 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f315f400700 1 -- 172.21.15.28:0/1275992337 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f31700125f0 con 0x558aa6f13fd0 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.773+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f316400ce20 con 0x558aa6f13fd0 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f315f400700 1 -- 172.21.15.28:0/1275992337 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f317004c050 con 0x558aa6f13fd0 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 10 client.25651 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:12.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558aa6f90df0 con 0x558aa6f13fd0 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f315f400700 1 -- 172.21.15.28:0/1275992337 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f316400ce20 con 0x558aa6f13fd0 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f315f400700 1 -- 172.21.15.28:0/1275992337 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f317003c470 con 0x558aa6f13fd0 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 10 client.25651 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 10 client.25651 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 20 client.25651 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f914c0 0x558aa6f938e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 4 client.25651 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:07:12.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.774+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x558aa6e5b5f0 con 0x558aa6f914c0 2024-05-20T20:07:12.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.775+0000 7f317da00700 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f914c0 0x558aa6f938e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:12.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.775+0000 7f317da00700 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f914c0 0x558aa6f938e0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e51be0477dd8b24 server_cookie=b527b72f303b093e in_seq=0 out_seq=0 2024-05-20T20:07:12.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.775+0000 7f315f400700 10 client.25651 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:12.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.776+0000 7f315f400700 1 -- 172.21.15.28:0/1275992337 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x558aa6e5b5f0 con 0x558aa6f914c0 2024-05-20T20:07:12.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.776+0000 7f315f400700 10 client.25651 handle_command_reply: tid=0 2024-05-20T20:07:12.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.776+0000 7f318ad63b80 1 client.25651 shutdown 2024-05-20T20:07:12.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 msgr2=0x558aa6f1a900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f1a900 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f3170017010 tx=0x7f3170013560 comp rx=0 tx=0).stop 2024-05-20T20:07:12.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 shutdown_connections 2024-05-20T20:07:12.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558aa6f914c0 0x558aa6f938e0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558aa6f13fd0 0x558aa6f1a900 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558aa6f0ac20 0x558aa6f1ceb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 --2- 172.21.15.28:0/1275992337 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558aa6f100d0 0x558aa6f1a2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 >> 172.21.15.28:0/1275992337 conn(0x558aa6e2fc60 msgr2=0x558aa6f12b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:12.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f315f400700 0 client.25651 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:12.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 shutdown_connections 2024-05-20T20:07:12.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 1 -- 172.21.15.28:0/1275992337 wait complete. 2024-05-20T20:07:12.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.777+0000 7f318ad63b80 20 client.25651 trim_cache size 0 max 16384 2024-05-20T20:07:12.781 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:07:12.782 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:07:12.782 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "036e2ce8-cba6-4536-880a-5571e7c410cb", 2024-05-20T20:07:12.782 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:07:12.782 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:07:12.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f31600b2a70 msgr2=0x7f31600b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f31600b2a70 0x7f31600b4f30 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f31700139d0 tx=0x7f3170009bf0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3178105010 msgr2=0x7f3178102890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:12.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3178105010 0x7f3178102890 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f316400e9f0 tx=0x7f316400ed00 comp rx=0 tx=0).stop 2024-05-20T20:07:12.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 shutdown_connections 2024-05-20T20:07:12.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f317810dad0 0x7f3178102dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f31781046f0 0x7f3178102350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f31600b2a70 0x7f31600b4f30 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 --2- 172.21.15.28:0/920010106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3178105010 0x7f3178102890 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:12.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 >> 172.21.15.28:0/920010106 conn(0x7f31780fde20 msgr2=0x7f31780febd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:12.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 shutdown_connections 2024-05-20T20:07:12.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:12.778+0000 7f317ee00700 1 -- 172.21.15.28:0/920010106 wait complete. 2024-05-20T20:07:12.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:12 smithi028 ceph-mon[31178]: pgmap v50: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 26 KiB/s wr, 26 op/s 2024-05-20T20:07:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:12 smithi154 ceph-mon[36314]: pgmap v50: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 26 KiB/s wr, 26 op/s 2024-05-20T20:07:13.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:12 smithi033 ceph-mon[36179]: pgmap v50: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 26 KiB/s wr, 26 op/s 2024-05-20T20:07:13.850 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:07:14.026 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:13 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:07:14.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:13 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:14.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:13 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:07:14.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:13 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:14.027 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:13 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:07:14.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:13 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:07:14.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:13 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:14.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:13 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:07:14.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:13 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:14.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:13 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:07:14.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:13 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:07:14.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:13 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:14.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:13 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:07:14.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:13 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:14.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:13 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:07:14.290 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:15.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:14 smithi028 ceph-mon[31178]: pgmap v51: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 26 KiB/s wr, 32 op/s 2024-05-20T20:07:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:14 smithi154 ceph-mon[36314]: pgmap v51: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 26 KiB/s wr, 32 op/s 2024-05-20T20:07:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:14 smithi033 ceph-mon[36179]: pgmap v51: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 26 KiB/s wr, 32 op/s 2024-05-20T20:07:15.128 INFO:tasks.quiescer.fs.[cephfs]:Total duration (52.7) is longer than `split_if_longer` (35.0), will split into two consecutive quiesces 2024-05-20T20:07:15.128 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 26.35 2024-05-20T20:07:15.128 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:07:16.046 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:16.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.823+0000 7f07ab200700 1 -- 172.21.15.28:0/517498975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 msgr2=0x7f07ac112120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.823+0000 7f07ab200700 1 --2- 172.21.15.28:0/517498975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 0x7f07ac112120 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f07a400b0a0 tx=0x7f07a400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.823+0000 7f07ab200700 1 -- 172.21.15.28:0/517498975 shutdown_connections 2024-05-20T20:07:16.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.823+0000 7f07ab200700 1 --2- 172.21.15.28:0/517498975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 0x7f07ac112120 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.823+0000 7f07ab200700 1 --2- 172.21.15.28:0/517498975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07ac075910 0x7f07ac075cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.823+0000 7f07ab200700 1 --2- 172.21.15.28:0/517498975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac076230 0x7f07ac10f7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.823+0000 7f07ab200700 1 -- 172.21.15.28:0/517498975 >> 172.21.15.28:0/517498975 conn(0x7f07ac0fefc0 msgr2=0x7f07ac1013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.824+0000 7f07ab200700 1 -- 172.21.15.28:0/517498975 shutdown_connections 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.824+0000 7f07ab200700 1 -- 172.21.15.28:0/517498975 wait complete. 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 Processor -- start 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 -- start start 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac075910 0x7f07ac19bd60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07ac076230 0x7f07ac19c2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 0x7f07ac1a05e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac115890 con 0x7f07ac10fd10 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac115710 con 0x7f07ac075910 2024-05-20T20:07:16.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.825+0000 7f07ab200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac115a10 con 0x7f07ac076230 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac075910 0x7f07ac19bd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac075910 0x7f07ac19bd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49354/0 (socket says 172.21.15.28:49354) 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07aa800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 0x7f07ac1a05e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 -- 172.21.15.28:0/2543525223 learned_addr learned my addr 172.21.15.28:0/2543525223 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 -- 172.21.15.28:0/2543525223 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07ac076230 msgr2=0x7f07ac19c2a0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07ac076230 0x7f07ac19c2a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 -- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 msgr2=0x7f07ac1a05e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 0x7f07ac1a05e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 -- 172.21.15.28:0/2543525223 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f07ac1a0d20 con 0x7f07ac075910 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.826+0000 7f07a9e00700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac075910 0x7f07ac19bd60 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f079c00eb30 tx=0x7f079c00ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.827+0000 7f07a2a00700 1 -- 172.21.15.28:0/2543525223 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f079c00cc90 con 0x7f07ac075910 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.827+0000 7f07a2a00700 1 -- 172.21.15.28:0/2543525223 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f079c0044d0 con 0x7f07ac075910 2024-05-20T20:07:16.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.827+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f07ac1a1010 con 0x7f07ac075910 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.827+0000 7f07a2a00700 1 -- 172.21.15.28:0/2543525223 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f079c010430 con 0x7f07ac075910 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.827+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f07ac1a8890 con 0x7f07ac075910 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.832+0000 7f07bb530b80 1 Processor -- start 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.832+0000 7f07bb530b80 1 -- start start 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963737c0 0x559e96373ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9636dd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e9636e390 0x559e9636e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07a2a00700 1 -- 172.21.15.28:0/2543525223 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f079c003ea0 con 0x7f07ac075910 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07bb530b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559e962cbe40 con 0x559e963740e0 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07bb530b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x559e962bb9e0 con 0x559e9636e390 2024-05-20T20:07:16.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07bb530b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559e962bfa20 con 0x559e963737c0 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07a9e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9636dd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07aa800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963737c0 0x559e96373ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.833+0000 7f07a9e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9636dd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36748/0 (socket says 172.21.15.28:36748) 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07a9e00700 1 -- 172.21.15.28:0/2144378334 learned_addr learned my addr 172.21.15.28:0/2144378334 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07a9400700 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e9636e390 0x559e9636e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07aa800700 1 -- 172.21.15.28:0/2144378334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e9636e390 msgr2=0x559e9636e7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07aa800700 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e9636e390 0x559e9636e7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07aa800700 1 -- 172.21.15.28:0/2144378334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 msgr2=0x559e9636dd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07aa800700 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9636dd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07aa800700 1 -- 172.21.15.28:0/2144378334 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559e96284420 con 0x559e963737c0 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.834+0000 7f07aa800700 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963737c0 0x559e96373ba0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f07a400b070 tx=0x7f07a40092e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=79f7412047ce3a3f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.835+0000 7f0797400700 1 -- 172.21.15.28:0/2144378334 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f07a400bcb0 con 0x559e963737c0 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.835+0000 7f0797400700 1 -- 172.21.15.28:0/2144378334 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f07a40074e0 con 0x559e963737c0 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.835+0000 7f0797400700 1 -- 172.21.15.28:0/2144378334 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f07a4005280 con 0x559e963737c0 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.835+0000 7f07a2a00700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07900b29f0 0x7f07900b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07a2a00700 1 -- 172.21.15.28:0/2543525223 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f079c0e2910 con 0x7f07ac075910 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07a9e00700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07900b29f0 0x7f07900b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07bb530b80 1 -- 172.21.15.28:0/2144378334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963737c0 msgr2=0x559e96373ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07bb530b80 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963737c0 0x559e96373ba0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f07a400b070 tx=0x7f07a40092e0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07bb530b80 1 -- 172.21.15.28:0/2144378334 shutdown_connections 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07bb530b80 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e9636e390 0x559e9636e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07bb530b80 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963737c0 0x559e96373ba0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07bb530b80 1 --2- 172.21.15.28:0/2144378334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9636dd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.836+0000 7f07bb530b80 1 -- 172.21.15.28:0/2144378334 >> 172.21.15.28:0/2144378334 conn(0x559e9629bbe0 msgr2=0x559e96368810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.837+0000 7f07bb530b80 1 -- 172.21.15.28:0/2144378334 shutdown_connections 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.837+0000 7f07bb530b80 1 -- 172.21.15.28:0/2144378334 wait complete. 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.837+0000 7f07a9e00700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07900b29f0 0x7f07900b4eb0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f079c0e2bb0 tx=0x7f079c0e8530 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 -- start start 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 Processor -- start 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e9636e390 0x559e96379e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96377200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963740e0 0x559e96377740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559e961c1fa0 con 0x559e9636e390 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x559e9625c760 con 0x559e963737c0 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.838+0000 7f07bb530b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559e9637aba0 con 0x559e963740e0 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07bb530b80 10 client.0 _ll_register_callbacks cb 0x559e96265c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07aa800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e9636e390 0x559e96379e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07a9400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96377200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07aa800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e9636e390 0x559e96379e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36764/0 (socket says 172.21.15.28:36764) 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07aa800700 1 -- 172.21.15.28:0/1510330863 learned_addr learned my addr 172.21.15.28:0/1510330863 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07aa800700 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963740e0 0x559e96377740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07a9400700 1 -- 172.21.15.28:0/1510330863 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963740e0 msgr2=0x559e96377740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.839+0000 7f07a9400700 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963740e0 0x559e96377740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f07a9400700 1 -- 172.21.15.28:0/1510330863 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e9636e390 msgr2=0x559e96379e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f07a9400700 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e9636e390 0x559e96379e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f07a9400700 1 -- 172.21.15.28:0/1510330863 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559e9620b8d0 con 0x559e963737c0 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f07aa800700 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e9636e390 0x559e96379e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f07a9400700 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96377200 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f07980116c0 tx=0x7f07980119d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f6cfed14be701660 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f078be00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f078be00700 1 -- 172.21.15.28:0/1510330863 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f079801e060 con 0x559e963737c0 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f078be00700 1 -- 172.21.15.28:0/1510330863 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0798004460 con 0x559e963737c0 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.840+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f07a40074e0 con 0x559e963737c0 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.841+0000 7f078be00700 1 -- 172.21.15.28:0/1510330863 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0798003680 con 0x559e963737c0 2024-05-20T20:07:16.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.841+0000 7f07bb530b80 10 client.25697 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.841+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559e9628c060 con 0x559e963737c0 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.841+0000 7f078be00700 1 -- 172.21.15.28:0/1510330863 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f07a40074e0 con 0x559e963737c0 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.842+0000 7f078be00700 1 -- 172.21.15.28:0/1510330863 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f07980199d0 con 0x559e963737c0 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.842+0000 7f07bb530b80 10 client.25697 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.842+0000 7f07bb530b80 10 client.25697 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.842+0000 7f07bb530b80 20 client.25697 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.842+0000 7f07bb530b80 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e96387770 0x559e96389b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.842+0000 7f07bb530b80 4 client.25697 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:16.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.842+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559e962cdd90 con 0x559e96387770 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.843+0000 7f07aa800700 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e96387770 0x559e96389b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.843+0000 7f07aa800700 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e96387770 0x559e96389b60 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cdb83957bc9bbd4c server_cookie=817fcee1d5d1820f in_seq=0 out_seq=0 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.843+0000 7f078be00700 10 client.25697 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.844+0000 7f078be00700 1 -- 172.21.15.28:0/1510330863 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x559e962cdd90 con 0x559e96387770 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.844+0000 7f078be00700 10 client.25697 handle_command_reply: tid=0 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.844+0000 7f07bb530b80 1 client.25697 shutdown 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 msgr2=0x559e96377200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96377200 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f07980116c0 tx=0x7f07980119d0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 shutdown_connections 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e96387770 0x559e96389b60 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e963740e0 0x559e96377740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e9636e390 0x559e96379e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 --2- 172.21.15.28:0/1510330863 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96377200 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 >> 172.21.15.28:0/1510330863 conn(0x559e9629bbe0 msgr2=0x559e96367fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f078be00700 0 client.25697 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 shutdown_connections 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 1 -- 172.21.15.28:0/1510330863 wait complete. 2024-05-20T20:07:16.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.845+0000 7f07bb530b80 20 client.25697 trim_cache size 0 max 16384 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 Processor -- start 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 -- start start 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9636dd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96368740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e96374bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac1aff70 con 0x559e963740e0 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac1b1ab0 con 0x559e963737c0 2024-05-20T20:07:16.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.871+0000 7f07bb530b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac1b0640 con 0x559e9636e390 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96368740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96368740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49380/0 (socket says 172.21.15.28:49380) 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 -- 172.21.15.28:0/3297135587 learned_addr learned my addr 172.21.15.28:0/3297135587 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07aa800700 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9636dd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07aa800700 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e96374bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 -- 172.21.15.28:0/3297135587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 msgr2=0x559e9636dd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9636dd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 -- 172.21.15.28:0/3297135587 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 msgr2=0x559e96374bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e96374bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 -- 172.21.15.28:0/3297135587 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559e96375790 con 0x559e963737c0 2024-05-20T20:07:16.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.872+0000 7f07a9400700 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96368740 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f0798009dd0 tx=0x7f0798001e50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cd0381268e1d6b69 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:16.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.873+0000 7f0796a00700 1 -- 172.21.15.28:0/3297135587 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0798030070 con 0x559e963737c0 2024-05-20T20:07:16.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.873+0000 7f0796a00700 1 -- 172.21.15.28:0/3297135587 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0798003820 con 0x559e963737c0 2024-05-20T20:07:16.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.873+0000 7f0796a00700 1 -- 172.21.15.28:0/3297135587 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f079800e9e0 con 0x559e963737c0 2024-05-20T20:07:16.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 -- 172.21.15.28:0/3297135587 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 msgr2=0x559e96368740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96368740 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f0798009dd0 tx=0x7f0798001e50 comp rx=0 tx=0).stop 2024-05-20T20:07:16.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 -- 172.21.15.28:0/3297135587 shutdown_connections 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e96374bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9636dd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 --2- 172.21.15.28:0/3297135587 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e96368740 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 -- 172.21.15.28:0/3297135587 >> 172.21.15.28:0/3297135587 conn(0x559e9629bbe0 msgr2=0x559e963744c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 -- 172.21.15.28:0/3297135587 shutdown_connections 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.874+0000 7f07bb530b80 1 -- 172.21.15.28:0/3297135587 wait complete. 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.875+0000 7f07bb530b80 1 -- start start 2024-05-20T20:07:16.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.875+0000 7f07bb530b80 1 Processor -- start 2024-05-20T20:07:16.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9638f130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e9638f670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07bb530b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9638fbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07bb530b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac1b13e0 con 0x559e963740e0 2024-05-20T20:07:16.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07bb530b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac1b0d10 con 0x559e963737c0 2024-05-20T20:07:16.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07bb530b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac1adf10 con 0x559e9636e390 2024-05-20T20:07:16.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07bb530b80 10 client.0 _ll_register_callbacks cb 0x559e96265c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:16.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e9638f670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e9638f670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49384/0 (socket says 172.21.15.28:49384) 2024-05-20T20:07:16.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 -- 172.21.15.28:0/4752548 learned_addr learned my addr 172.21.15.28:0/4752548 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07aa800700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9638f130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07aa800700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9638fbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 -- 172.21.15.28:0/4752548 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 msgr2=0x559e9638f130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9638f130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 -- 172.21.15.28:0/4752548 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 msgr2=0x559e9638fbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9638fbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07a9400700 1 -- 172.21.15.28:0/4752548 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559e963f14e0 con 0x559e963737c0 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.876+0000 7f07aa800700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9638fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:16.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.877+0000 7f07a9400700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e9638f670 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f0798019ec0 tx=0x7f0798009610 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5ca4b3cf0fa08e1c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:16.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.877+0000 7f078be00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:16.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.877+0000 7f078be00700 1 -- 172.21.15.28:0/4752548 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0798030070 con 0x559e963737c0 2024-05-20T20:07:16.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.877+0000 7f078be00700 1 -- 172.21.15.28:0/4752548 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f07980039d0 con 0x559e963737c0 2024-05-20T20:07:16.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.877+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0798003820 con 0x559e963737c0 2024-05-20T20:07:16.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f078be00700 1 -- 172.21.15.28:0/4752548 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0798003bd0 con 0x559e963737c0 2024-05-20T20:07:16.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f07bb530b80 10 client.25703 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:16.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559e963f27d0 con 0x559e963737c0 2024-05-20T20:07:16.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f078be00700 1 -- 172.21.15.28:0/4752548 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0798003820 con 0x559e963737c0 2024-05-20T20:07:16.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f078be00700 1 -- 172.21.15.28:0/4752548 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0798019550 con 0x559e963737c0 2024-05-20T20:07:16.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f07bb530b80 10 client.25703 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:16.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f07bb530b80 10 client.25703 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:16.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f07bb530b80 20 client.25703 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:16.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.878+0000 7f07bb530b80 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e963f2d10 0x559e963f5100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:16.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.879+0000 7f07bb530b80 4 client.25703 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.879+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x559e962c7570 con 0x559e963f2d10 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.879+0000 7f07aa800700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e963f2d10 0x559e963f5100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.879+0000 7f07aa800700 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e963f2d10 0x559e963f5100 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=45338c75e1c8080b server_cookie=cbe4eadc145dd93a in_seq=0 out_seq=0 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f078be00700 10 client.25703 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f078be00700 1 -- 172.21.15.28:0/4752548 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x559e962c7570 con 0x559e963f2d10 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f078be00700 10 client.25703 handle_command_reply: tid=0 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 client.25703 shutdown 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 msgr2=0x559e9638f670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e9638f670 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f0798019ec0 tx=0x7f0798009610 comp rx=0 tx=0).stop 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 shutdown_connections 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e963f2d10 0x559e963f5100 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e963740e0 0x559e9638fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e9636e390 0x559e9638f130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 --2- 172.21.15.28:0/4752548 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e963737c0 0x559e9638f670 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f078be00700 0 client.25703 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 >> 172.21.15.28:0/4752548 conn(0x559e9629bbe0 msgr2=0x559e96374d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.880+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 shutdown_connections 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.881+0000 7f07bb530b80 1 -- 172.21.15.28:0/4752548 wait complete. 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.881+0000 7f07bb530b80 20 client.25703 trim_cache size 0 max 16384 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:07:16.884 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:07:16.885 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:07:16.885 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:07:16.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07900b29f0 msgr2=0x7f07900b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07900b29f0 0x7f07900b4eb0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f079c0e2bb0 tx=0x7f079c0e8530 comp rx=0 tx=0).stop 2024-05-20T20:07:16.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac075910 msgr2=0x7f07ac19bd60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac075910 0x7f07ac19bd60 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f079c00eb30 tx=0x7f079c00ee40 comp rx=0 tx=0).stop 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 shutdown_connections 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f07ac10fd10 0x7f07ac1a05e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f07900b29f0 0x7f07900b4eb0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f07ac075910 0x7f07ac19bd60 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 --2- 172.21.15.28:0/2543525223 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07ac076230 0x7f07ac19c2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 >> 172.21.15.28:0/2543525223 conn(0x7f07ac0fefc0 msgr2=0x7f07ac0ffcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 shutdown_connections 2024-05-20T20:07:16.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:16.882+0000 7f07ab200700 1 -- 172.21.15.28:0/2543525223 wait complete. 2024-05-20T20:07:17.000 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:16 smithi028 ceph-mon[31178]: pgmap v52: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:07:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:16 smithi154 ceph-mon[36314]: pgmap v52: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:07:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:16 smithi033 ceph-mon[36179]: pgmap v52: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:07:17.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 -- 172.21.15.28:0/1426821507 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c103a10 msgr2=0x7f4e9c103df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 --2- 172.21.15.28:0/1426821507 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c103a10 0x7f4e9c103df0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4e8c009900 tx=0x7f4e8c02a480 comp rx=0 tx=0).stop 2024-05-20T20:07:17.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 -- 172.21.15.28:0/1426821507 shutdown_connections 2024-05-20T20:07:17.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 --2- 172.21.15.28:0/1426821507 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e9c10ce30 0x7f4e9c10f2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 --2- 172.21.15.28:0/1426821507 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c104330 0x7f4e9c10c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 --2- 172.21.15.28:0/1426821507 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c103a10 0x7f4e9c103df0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 -- 172.21.15.28:0/1426821507 >> 172.21.15.28:0/1426821507 conn(0x7f4e9c0fd270 msgr2=0x7f4e9c0ff690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:17.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 -- 172.21.15.28:0/1426821507 shutdown_connections 2024-05-20T20:07:17.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.517+0000 7f4ea1600640 1 -- 172.21.15.28:0/1426821507 wait complete. 2024-05-20T20:07:17.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 Processor -- start 2024-05-20T20:07:17.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 -- start start 2024-05-20T20:07:17.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c103a10 0x7f4e9c199090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e9c104330 0x7f4e9c1995d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c10ce30 0x7f4e9c19d950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c112b30 con 0x7f4e9c103a10 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c1129b0 con 0x7f4e9c10ce30 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.518+0000 7f4ea1600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c112cb0 con 0x7f4e9c104330 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c103a10 0x7f4e9c199090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c103a10 0x7f4e9c199090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40236/0 (socket says 172.21.15.28:40236) 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 -- 172.21.15.28:0/2576721206 learned_addr learned my addr 172.21.15.28:0/2576721206 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9b400640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e9c104330 0x7f4e9c1995d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4ea0c00640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c10ce30 0x7f4e9c19d950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 -- 172.21.15.28:0/2576721206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e9c104330 msgr2=0x7f4e9c1995d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e9c104330 0x7f4e9c1995d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 -- 172.21.15.28:0/2576721206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c10ce30 msgr2=0x7f4e9c19d950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c10ce30 0x7f4e9c19d950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.519+0000 7f4e9be00640 1 -- 172.21.15.28:0/2576721206 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4e9c19e080 con 0x7f4e9c103a10 2024-05-20T20:07:17.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.520+0000 7f4e9be00640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c103a10 0x7f4e9c199090 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f4e8c005ab0 tx=0x7f4e8c003c80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:17.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.520+0000 7f4e98c00640 1 -- 172.21.15.28:0/2576721206 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e8c03f070 con 0x7f4e9c103a10 2024-05-20T20:07:17.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.520+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4e9c19e310 con 0x7f4e9c103a10 2024-05-20T20:07:17.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.520+0000 7f4e98c00640 1 -- 172.21.15.28:0/2576721206 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e8c031d60 con 0x7f4e9c103a10 2024-05-20T20:07:17.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.520+0000 7f4e98c00640 1 -- 172.21.15.28:0/2576721206 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e8c035030 con 0x7f4e9c103a10 2024-05-20T20:07:17.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.520+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4e9c1a5c00 con 0x7f4e9c103a10 2024-05-20T20:07:17.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 Processor -- start 2024-05-20T20:07:17.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4e98c00640 1 -- 172.21.15.28:0/2576721206 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4e8c042070 con 0x7f4e9c103a10 2024-05-20T20:07:17.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 -- start start 2024-05-20T20:07:17.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a6944150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a6952f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6955830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5630a68b4b90 con 0x5630a6953440 2024-05-20T20:07:17.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5630a68a4730 con 0x5630a6943d70 2024-05-20T20:07:17.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.524+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5630a68a8770 con 0x5630a6944690 2024-05-20T20:07:17.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9be00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a6952f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9be00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a6952f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:36142/0 (socket says 172.21.15.28:36142) 2024-05-20T20:07:17.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9be00640 1 -- 172.21.15.28:0/2577065664 learned_addr learned my addr 172.21.15.28:0/2577065664 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:17.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9b400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6955830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4ea0c00640 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a6944150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9b400640 1 -- 172.21.15.28:0/2577065664 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 msgr2=0x5630a6952f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9b400640 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a6952f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9b400640 1 -- 172.21.15.28:0/2577065664 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 msgr2=0x5630a6944150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9b400640 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a6944150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9b400640 1 -- 172.21.15.28:0/2577065664 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5630a683d8b0 con 0x5630a6953440 2024-05-20T20:07:17.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.525+0000 7f4e9be00640 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a6952f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:17.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.526+0000 7f4e9b400640 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6955830 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f4e88009be0 tx=0x7f4e88011810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52ada52fc1baab48 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:17.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.526+0000 7f4e91200640 1 -- 172.21.15.28:0/2577065664 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e88004050 con 0x5630a6953440 2024-05-20T20:07:17.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.526+0000 7f4e98c00640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4e800b2950 0x7f4e800b4e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.526+0000 7f4e91200640 1 -- 172.21.15.28:0/2577065664 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e880054d0 con 0x5630a6953440 2024-05-20T20:07:17.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.526+0000 7f4e91200640 1 -- 172.21.15.28:0/2577065664 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e88004c80 con 0x5630a6953440 2024-05-20T20:07:17.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.526+0000 7f4e98c00640 1 -- 172.21.15.28:0/2576721206 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f4e8c102e00 con 0x7f4e9c103a10 2024-05-20T20:07:17.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.526+0000 7f4ea0c00640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4e800b2950 0x7f4e800b4e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 -- 172.21.15.28:0/2577065664 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 msgr2=0x5630a6955830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6955830 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f4e88009be0 tx=0x7f4e88011810 comp rx=0 tx=0).stop 2024-05-20T20:07:17.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea0c00640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4e800b2950 0x7f4e800b4e10 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f4e94005fd0 tx=0x7f4e94009600 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:17.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 -- 172.21.15.28:0/2577065664 shutdown_connections 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6955830 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a6952f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/2577065664 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a6944150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 -- 172.21.15.28:0/2577065664 >> 172.21.15.28:0/2577065664 conn(0x5630a6940360 msgr2=0x5630a6940740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 -- 172.21.15.28:0/2577065664 shutdown_connections 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 -- 172.21.15.28:0/2577065664 wait complete. 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.527+0000 7f4ea59ec740 1 -- start start 2024-05-20T20:07:17.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 1 Processor -- start 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6943d70 0x5630a6952d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6944690 0x5630a6950040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6953440 0x5630a6950580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5630a658a790 con 0x5630a6943d70 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5630a684cbc0 con 0x5630a6944690 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5630a6956070 con 0x5630a6953440 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea59ec740 10 client.0 _ll_register_callbacks cb 0x5630a682a400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4e9be00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6943d70 0x5630a6952d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.528+0000 7f4ea0c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6953440 0x5630a6950580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6943d70 0x5630a6952d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40244/0 (socket says 172.21.15.28:40244) 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 -- 172.21.15.28:0/65277939 learned_addr learned my addr 172.21.15.28:0/65277939 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9b400640 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6944690 0x5630a6950040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 -- 172.21.15.28:0/65277939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6953440 msgr2=0x5630a6950580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6953440 0x5630a6950580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 -- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6944690 msgr2=0x5630a6950040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6944690 0x5630a6950040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 -- 172.21.15.28:0/65277939 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5630a65b9210 con 0x5630a6943d70 2024-05-20T20:07:17.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e9be00640 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6943d70 0x5630a6952d50 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f4e8c0c61c0 tx=0x7f4e8c02a480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e7026da904d4ecf2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:17.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e85600640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:17.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e85600640 1 -- 172.21.15.28:0/65277939 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e8c049070 con 0x5630a6943d70 2024-05-20T20:07:17.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.529+0000 7f4e85600640 1 -- 172.21.15.28:0/65277939 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f4e8c100780 con 0x5630a6943d70 2024-05-20T20:07:17.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4e880054d0 con 0x5630a6943d70 2024-05-20T20:07:17.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4e85600640 1 -- 172.21.15.28:0/65277939 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4e8c0c6000 con 0x5630a6943d70 2024-05-20T20:07:17.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 10 client.15570 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:17.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5630a6685620 con 0x5630a6943d70 2024-05-20T20:07:17.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4e85600640 1 -- 172.21.15.28:0/65277939 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e880054d0 con 0x5630a6943d70 2024-05-20T20:07:17.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4e85600640 1 -- 172.21.15.28:0/65277939 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4e8c0d2a60 con 0x5630a6943d70 2024-05-20T20:07:17.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 10 client.15570 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:17.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 10 client.15570 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:17.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 20 client.15570 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:17.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a6962d60 0x5630a6965170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 4 client.15570 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:17.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.530+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5630a68b6ae0 con 0x5630a6962d60 2024-05-20T20:07:17.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.531+0000 7f4e9b400640 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a6962d60 0x5630a6965170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.531+0000 7f4e9b400640 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a6962d60 0x5630a6965170 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=df9c0ca0fd957f65 server_cookie=43c1897d9f4e42e0 in_seq=0 out_seq=0 2024-05-20T20:07:17.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.531+0000 7f4e85600640 10 client.15570 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:17.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4e85600640 1 -- 172.21.15.28:0/65277939 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5630a68b6ae0 con 0x5630a6962d60 2024-05-20T20:07:17.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4e85600640 10 client.15570 handle_command_reply: tid=0 2024-05-20T20:07:17.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 client.15570 shutdown 2024-05-20T20:07:17.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6943d70 msgr2=0x5630a6952d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6943d70 0x5630a6952d50 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f4e8c0c61c0 tx=0x7f4e8c02a480 comp rx=0 tx=0).stop 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 shutdown_connections 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a6962d60 0x5630a6965170 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6944690 0x5630a6950040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6953440 0x5630a6950580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/65277939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6943d70 0x5630a6952d50 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 >> 172.21.15.28:0/65277939 conn(0x5630a6940360 msgr2=0x5630a69461d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.532+0000 7f4e85600640 0 client.15570 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.533+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 shutdown_connections 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.533+0000 7f4ea59ec740 1 -- 172.21.15.28:0/65277939 wait complete. 2024-05-20T20:07:17.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.533+0000 7f4ea59ec740 20 client.15570 trim_cache size 0 max 16384 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.556+0000 7f4ea59ec740 1 Processor -- start 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea59ec740 1 -- start start 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a695f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a694b5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6952880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c1ac350 con 0x5630a6953440 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c1ade90 con 0x5630a6943d70 2024-05-20T20:07:17.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c1aca20 con 0x5630a6944690 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.557+0000 7f4ea0c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6952880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9b400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a694b5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4ea0c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6952880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40260/0 (socket says 172.21.15.28:40260) 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4ea0c00640 1 -- 172.21.15.28:0/3477016724 learned_addr learned my addr 172.21.15.28:0/3477016724 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9be00640 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a695f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9b400640 1 -- 172.21.15.28:0/3477016724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 msgr2=0x5630a695f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9b400640 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a695f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9b400640 1 -- 172.21.15.28:0/3477016724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 msgr2=0x5630a6952880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9b400640 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6952880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9b400640 1 -- 172.21.15.28:0/3477016724 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5630a694e590 con 0x5630a6944690 2024-05-20T20:07:17.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.558+0000 7f4e9b400640 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a694b5b0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f4e8801b040 tx=0x7f4e880095d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=798959f4eabe24bb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:17.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.559+0000 7f4e87e00640 1 -- 172.21.15.28:0/3477016724 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e88009790 con 0x5630a6944690 2024-05-20T20:07:17.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.559+0000 7f4e87e00640 1 -- 172.21.15.28:0/3477016724 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e880198e0 con 0x5630a6944690 2024-05-20T20:07:17.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.559+0000 7f4e87e00640 1 -- 172.21.15.28:0/3477016724 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e8802c760 con 0x5630a6944690 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.559+0000 7f4ea59ec740 1 -- 172.21.15.28:0/3477016724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 msgr2=0x5630a694b5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.559+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a694b5b0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f4e8801b040 tx=0x7f4e880095d0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 -- 172.21.15.28:0/3477016724 shutdown_connections 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6953440 0x5630a6952880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6944690 0x5630a694b5b0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/3477016724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6943d70 0x5630a695f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 -- 172.21.15.28:0/3477016724 >> 172.21.15.28:0/3477016724 conn(0x5630a6940360 msgr2=0x5630a694aea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 -- 172.21.15.28:0/3477016724 shutdown_connections 2024-05-20T20:07:17.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 -- 172.21.15.28:0/3477016724 wait complete. 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 -- start start 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.560+0000 7f4ea59ec740 1 Processor -- start 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6943d70 0x5630a6956670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6944690 0x5630a6956bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea59ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6953440 0x5630a69cb940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c1ad7c0 con 0x5630a6944690 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c1ad0f0 con 0x5630a6953440 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea59ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4e9c1aa330 con 0x5630a6943d70 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea59ec740 10 client.0 _ll_register_callbacks cb 0x7f4e8c1065a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4e9b400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6944690 0x5630a6956bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4e9b400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6944690 0x5630a6956bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40276/0 (socket says 172.21.15.28:40276) 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4e9b400640 1 -- 172.21.15.28:0/4232038178 learned_addr learned my addr 172.21.15.28:0/4232038178 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:17.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4e9be00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6943d70 0x5630a6956670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.561+0000 7f4ea0c00640 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6953440 0x5630a69cb940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.562+0000 7f4e9b400640 1 -- 172.21.15.28:0/4232038178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6943d70 msgr2=0x5630a6956670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.562+0000 7f4e9b400640 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6943d70 0x5630a6956670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.562+0000 7f4e9b400640 1 -- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6953440 msgr2=0x5630a69cb940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.562+0000 7f4e9b400640 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6953440 0x5630a69cb940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.562+0000 7f4e9b400640 1 -- 172.21.15.28:0/4232038178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5630a69cbe80 con 0x5630a6944690 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.562+0000 7f4e9b400640 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6944690 0x5630a6956bb0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f4e88009250 tx=0x7f4e88004710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2549a30d75e6947 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:17.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.562+0000 7f4e85600640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:17.566 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.563+0000 7f4e85600640 1 -- 172.21.15.28:0/4232038178 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e8803d060 con 0x5630a6944690 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.563+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4e880198e0 con 0x5630a6944690 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.563+0000 7f4e85600640 1 -- 172.21.15.28:0/4232038178 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f4e88019430 con 0x5630a6944690 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.563+0000 7f4e85600640 1 -- 172.21.15.28:0/4232038178 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4e88032030 con 0x5630a6944690 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.563+0000 7f4ea59ec740 10 client.15576 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.563+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5630a69cd140 con 0x5630a6944690 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.563+0000 7f4e85600640 1 -- 172.21.15.28:0/4232038178 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e880198e0 con 0x5630a6944690 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4e85600640 1 -- 172.21.15.28:0/4232038178 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4e8802c9f0 con 0x5630a6944690 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4ea59ec740 10 client.15576 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4ea59ec740 10 client.15576 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4ea59ec740 20 client.15576 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a69cd650 0x5630a69cfa70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4ea59ec740 4 client.15576 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:07:17.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5630a6950910 con 0x5630a69cd650 2024-05-20T20:07:17.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.564+0000 7f4e9be00640 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a69cd650 0x5630a69cfa70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:17.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.565+0000 7f4e9be00640 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a69cd650 0x5630a69cfa70 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9a64bc7bad9459c2 server_cookie=44c61862bfbec11d in_seq=0 out_seq=0 2024-05-20T20:07:17.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.565+0000 7f4e85600640 10 client.15576 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:17.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4e85600640 1 -- 172.21.15.28:0/4232038178 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5630a6950910 con 0x5630a69cd650 2024-05-20T20:07:17.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4e85600640 10 client.15576 handle_command_reply: tid=0 2024-05-20T20:07:17.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 client.15576 shutdown 2024-05-20T20:07:17.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6944690 msgr2=0x5630a6956bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6944690 0x5630a6956bb0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f4e88009250 tx=0x7f4e88004710 comp rx=0 tx=0).stop 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 shutdown_connections 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5630a69cd650 0x5630a69cfa70 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5630a6953440 0x5630a69cb940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5630a6944690 0x5630a6956bb0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4e85600640 0 client.15576 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 --2- 172.21.15.28:0/4232038178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5630a6943d70 0x5630a6956670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.571+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 >> 172.21.15.28:0/4232038178 conn(0x5630a6940360 msgr2=0x5630a693ec30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.572+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 shutdown_connections 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.572+0000 7f4ea59ec740 1 -- 172.21.15.28:0/4232038178 wait complete. 2024-05-20T20:07:17.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.572+0000 7f4ea59ec740 20 client.15576 trim_cache size 0 max 16384 2024-05-20T20:07:17.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:07:17.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:07:17.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4e800b2950 msgr2=0x7f4e800b4e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4e800b2950 0x7f4e800b4e10 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f4e94005fd0 tx=0x7f4e94009600 comp rx=0 tx=0).stop 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c103a10 msgr2=0x7f4e9c199090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c103a10 0x7f4e9c199090 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f4e8c005ab0 tx=0x7f4e8c003c80 comp rx=0 tx=0).stop 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 shutdown_connections 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4e9c10ce30 0x7f4e9c19d950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4e800b2950 0x7f4e800b4e10 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e9c104330 0x7f4e9c1995d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 --2- 172.21.15.28:0/2576721206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4e9c103a10 0x7f4e9c199090 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:17.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 >> 172.21.15.28:0/2576721206 conn(0x7f4e9c0fd270 msgr2=0x7f4e9c0feb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:17.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 shutdown_connections 2024-05-20T20:07:17.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:17.573+0000 7f4ea1600640 1 -- 172.21.15.28:0/2576721206 wait complete. 2024-05-20T20:07:18.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 -- 172.21.15.28:0/3985772702 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe0102f80 msgr2=0x7fdfe0106fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:18.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3985772702 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe0102f80 0x7fdfe0106fd0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fdfd0009990 tx=0x7fdfd0009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:18.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 -- 172.21.15.28:0/3985772702 shutdown_connections 2024-05-20T20:07:18.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3985772702 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdfe01076b0 0x7fdfe010f280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:18.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3985772702 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdfe01025d0 0x7fdfe01029b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:18.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3985772702 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe0102f80 0x7fdfe0106fd0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:18.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 -- 172.21.15.28:0/3985772702 >> 172.21.15.28:0/3985772702 conn(0x7fdfe00fde60 msgr2=0x7fdfe0100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:18.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 -- 172.21.15.28:0/3985772702 shutdown_connections 2024-05-20T20:07:18.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.875+0000 7fdfe6200700 1 -- 172.21.15.28:0/3985772702 wait complete. 2024-05-20T20:07:18.878 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:18 smithi028 ceph-mon[31178]: pgmap v53: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:07:18.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.877+0000 7fdfe6200700 1 Processor -- start 2024-05-20T20:07:18.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.878+0000 7fdfe6200700 1 -- start start 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.878+0000 7fdfe6200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe01025d0 0x7fdfe019bd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.878+0000 7fdfe6200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdfe0102f80 0x7fdfe019c260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.878+0000 7fdfe6200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdfe01076b0 0x7fdfe01a05f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.878+0000 7fdfe6200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdfe0112520 con 0x7fdfe0102f80 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.878+0000 7fdfe6200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdfe01123a0 con 0x7fdfe01025d0 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.878+0000 7fdfe6200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdfe01126a0 con 0x7fdfe01076b0 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfdfe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdfe0102f80 0x7fdfe019c260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe01025d0 0x7fdfe019bd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe01025d0 0x7fdfe019bd20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46282/0 (socket says 172.21.15.28:46282) 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 -- 172.21.15.28:0/3010000918 learned_addr learned my addr 172.21.15.28:0/3010000918 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfdfe00700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdfe0102f80 0x7fdfe019c260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40286/0 (socket says 172.21.15.28:40286) 2024-05-20T20:07:18.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe5800700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdfe01076b0 0x7fdfe01a05f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:18.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 -- 172.21.15.28:0/3010000918 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdfe01076b0 msgr2=0x7fdfe01a05f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:18.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdfe01076b0 0x7fdfe01a05f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:18.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 -- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdfe0102f80 msgr2=0x7fdfe019c260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdfe0102f80 0x7fdfe019c260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.879+0000 7fdfe4e00700 1 -- 172.21.15.28:0/3010000918 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdfe01a0d30 con 0x7fdfe01025d0 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.880+0000 7fdfe4e00700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe01025d0 0x7fdfe019bd20 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fdfd400e9a0 tx=0x7fdfd400ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.880+0000 7fdfdd600700 1 -- 172.21.15.28:0/3010000918 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdfd400ca10 con 0x7fdfe01025d0 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.880+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdfe01a1020 con 0x7fdfe01025d0 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.880+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdfe01a88a0 con 0x7fdfe01025d0 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.881+0000 7fdfdd600700 1 -- 172.21.15.28:0/3010000918 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdfd400cbb0 con 0x7fdfe01025d0 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.881+0000 7fdfdd600700 1 -- 172.21.15.28:0/3010000918 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdfd4018680 con 0x7fdfe01025d0 2024-05-20T20:07:18.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.882+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdfc4005330 con 0x7fdfe01025d0 2024-05-20T20:07:18.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.884+0000 7fdfdd600700 1 -- 172.21.15.28:0/3010000918 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fdfd4010430 con 0x7fdfe01025d0 2024-05-20T20:07:18.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.885+0000 7fdfdd600700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdfc80b2a70 0x7fdfc80b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:18.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.885+0000 7fdfdd600700 1 -- 172.21.15.28:0/3010000918 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fdfd40e26b0 con 0x7fdfe01025d0 2024-05-20T20:07:18.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.886+0000 7fdfdfe00700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdfc80b2a70 0x7fdfc80b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:18.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.886+0000 7fdfdfe00700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdfc80b2a70 0x7fdfc80b4f30 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fdfd000b2e0 tx=0x7fdfd0005960 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:18.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:18.890+0000 7fdfdd600700 1 -- 172.21.15.28:0/3010000918 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fdfd40aa040 con 0x7fdfe01025d0 2024-05-20T20:07:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:18 smithi154 ceph-mon[36314]: pgmap v53: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:07:19.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:18 smithi033 ceph-mon[36179]: pgmap v53: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:07:19.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.238+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fdfc40058f0 con 0x7fdfe01025d0 2024-05-20T20:07:19.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.239+0000 7fdfdd600700 1 -- 172.21.15.28:0/3010000918 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fdfd40b20c0 con 0x7fdfe01025d0 2024-05-20T20:07:19.241 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:07:19.241 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.243+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdfc80b2a70 msgr2=0x7fdfc80b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.243+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdfc80b2a70 0x7fdfc80b4f30 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fdfd000b2e0 tx=0x7fdfd0005960 comp rx=0 tx=0).stop 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.243+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe01025d0 msgr2=0x7fdfe019bd20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.243+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe01025d0 0x7fdfe019bd20 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fdfd400e9a0 tx=0x7fdfd400ecb0 comp rx=0 tx=0).stop 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 shutdown_connections 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdfe01076b0 0x7fdfe01a05f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdfc80b2a70 0x7fdfc80b4f30 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdfe01025d0 0x7fdfe019bd20 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 --2- 172.21.15.28:0/3010000918 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdfe0102f80 0x7fdfe019c260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 >> 172.21.15.28:0/3010000918 conn(0x7fdfe00fde60 msgr2=0x7fdfe00ff4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:19.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 shutdown_connections 2024-05-20T20:07:19.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:19.244+0000 7fdfe6200700 1 -- 172.21.15.28:0/3010000918 wait complete. 2024-05-20T20:07:19.246 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:07:19.300 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:07:19.301 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:07:20.033 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:19 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3010000918' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:20.095 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 146.35 --await-for 45.0' 2024-05-20T20:07:20.096 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 146.35 --await-for 45.0 2024-05-20T20:07:20.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:19 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3010000918' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:20.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:19 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3010000918' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:20.197 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:21.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:20 smithi028 ceph-mon[31178]: pgmap v54: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 8.0 KiB/s rd, 2.7 KiB/s wr, 17 op/s 2024-05-20T20:07:21.112 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:20 smithi154 ceph-mon[36314]: pgmap v54: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 8.0 KiB/s rd, 2.7 KiB/s wr, 17 op/s 2024-05-20T20:07:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:20 smithi033 ceph-mon[36179]: pgmap v54: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 8.0 KiB/s rd, 2.7 KiB/s wr, 17 op/s 2024-05-20T20:07:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.594+0000 7f9bd4a00700 1 -- 172.21.15.28:0/2344298969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103490 msgr2=0x7f9bd0103910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.594+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/2344298969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103490 0x7f9bd0103910 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f9bbc0099c0 tx=0x7f9bbc009cd0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.595+0000 7f9bd4a00700 1 -- 172.21.15.28:0/2344298969 shutdown_connections 2024-05-20T20:07:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.595+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/2344298969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9bd0103e50 0x7f9bd0110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.595+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/2344298969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0109540 0x7f9bd0109920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.595+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/2344298969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103490 0x7f9bd0103910 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.595+0000 7f9bd4a00700 1 -- 172.21.15.28:0/2344298969 >> 172.21.15.28:0/2344298969 conn(0x7f9bd00fefc0 msgr2=0x7f9bd01013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:22.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.595+0000 7f9bd4a00700 1 -- 172.21.15.28:0/2344298969 shutdown_connections 2024-05-20T20:07:22.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.595+0000 7f9bd4a00700 1 -- 172.21.15.28:0/2344298969 wait complete. 2024-05-20T20:07:22.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 Processor -- start 2024-05-20T20:07:22.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 -- start start 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0103490 0x7f9bd01a1a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103e50 0x7f9bd01a1f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9bd0109540 0x7f9bd019bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9bd01136a0 con 0x7f9bd0103e50 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9bd0113520 con 0x7f9bd0103490 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.597+0000 7f9bd4a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9bd0113820 con 0x7f9bd0109540 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0103490 0x7f9bd01a1a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0103490 0x7f9bd01a1a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46306/0 (socket says 172.21.15.28:46306) 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 -- 172.21.15.28:0/1033091394 learned_addr learned my addr 172.21.15.28:0/1033091394 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bcde00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103e50 0x7f9bd01a1f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:22.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bcf200700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9bd0109540 0x7f9bd019bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:22.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 -- 172.21.15.28:0/1033091394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9bd0109540 msgr2=0x7f9bd019bb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:22.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9bd0109540 0x7f9bd019bb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 -- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103e50 msgr2=0x7f9bd01a1f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:22.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103e50 0x7f9bd01a1f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.598+0000 7f9bce800700 1 -- 172.21.15.28:0/1033091394 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9bd019c3f0 con 0x7f9bd0103490 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.599+0000 7f9bce800700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0103490 0x7f9bd01a1a40 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f9bc000e9a0 tx=0x7f9bc000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.599+0000 7f9bc7400700 1 -- 172.21.15.28:0/1033091394 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9bc000cbd0 con 0x7f9bd0103490 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.599+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9bd019c6e0 con 0x7f9bd0103490 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.599+0000 7f9bc7400700 1 -- 172.21.15.28:0/1033091394 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9bc000cd70 con 0x7f9bd0103490 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.599+0000 7f9bc7400700 1 -- 172.21.15.28:0/1033091394 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9bc0018680 con 0x7f9bd0103490 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.599+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9bd01a89c0 con 0x7f9bd0103490 2024-05-20T20:07:22.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.600+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9bd0104bb0 con 0x7f9bd0103490 2024-05-20T20:07:22.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.604+0000 7f9bc7400700 1 -- 172.21.15.28:0/1033091394 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f9bc0010430 con 0x7f9bd0103490 2024-05-20T20:07:22.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.606+0000 7f9bc7400700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9bb40b29f0 0x7f9bb40b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:22.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.607+0000 7f9bc7400700 1 -- 172.21.15.28:0/1033091394 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f9bc00e28f0 con 0x7f9bd0103490 2024-05-20T20:07:22.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.607+0000 7f9bcde00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9bb40b29f0 0x7f9bb40b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:22.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.607+0000 7f9bcde00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9bb40b29f0 0x7f9bb40b4eb0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9bd019d2f0 tx=0x7f9bbc005fb0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:22.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.611+0000 7f9bc7400700 1 -- 172.21.15.28:0/1033091394 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f9bc00aa280 con 0x7f9bd0103490 2024-05-20T20:07:22.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.946+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f9bd019cfd0 con 0x7f9bd0103490 2024-05-20T20:07:22.948 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:22 smithi028 ceph-mon[31178]: pgmap v55: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 341 B/s wr, 10 op/s 2024-05-20T20:07:22.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.947+0000 7f9bc7400700 1 -- 172.21.15.28:0/1033091394 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f9bc00b2300 con 0x7f9bd0103490 2024-05-20T20:07:22.949 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:07:22.949 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:07:22.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9bb40b29f0 msgr2=0x7f9bb40b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:22.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9bb40b29f0 0x7f9bb40b4eb0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9bd019d2f0 tx=0x7f9bbc005fb0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0103490 msgr2=0x7f9bd01a1a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:22.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0103490 0x7f9bd01a1a40 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f9bc000e9a0 tx=0x7f9bc000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 shutdown_connections 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9bd0109540 0x7f9bd019bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9bb40b29f0 0x7f9bb40b4eb0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9bd0103490 0x7f9bd01a1a40 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 --2- 172.21.15.28:0/1033091394 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9bd0103e50 0x7f9bd01a1f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.951+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 >> 172.21.15.28:0/1033091394 conn(0x7f9bd00fefc0 msgr2=0x7f9bd0100420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.952+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 shutdown_connections 2024-05-20T20:07:22.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:22.952+0000 7f9bd4a00700 1 -- 172.21.15.28:0/1033091394 wait complete. 2024-05-20T20:07:22.954 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:07:23.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:22 smithi154 ceph-mon[36314]: pgmap v55: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 341 B/s wr, 10 op/s 2024-05-20T20:07:23.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:22 smithi033 ceph-mon[36179]: pgmap v55: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 341 B/s wr, 10 op/s 2024-05-20T20:07:24.020 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:23 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1033091394' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:24.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:23 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1033091394' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:24.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:23 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1033091394' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:24.314 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:07:24.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:24 smithi028 ceph-mon[31178]: pgmap v56: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 341 B/s wr, 14 op/s 2024-05-20T20:07:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.868+0000 7ffbd9200700 1 -- 172.21.15.28:0/3829574925 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 msgr2=0x7ffbd41093c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.868+0000 7ffbd9200700 1 --2- 172.21.15.28:0/3829574925 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 0x7ffbd41093c0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7ffbc00099c0 tx=0x7ffbc0009cd0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.868+0000 7ffbd9200700 1 -- 172.21.15.28:0/3829574925 shutdown_connections 2024-05-20T20:07:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.868+0000 7ffbd9200700 1 --2- 172.21.15.28:0/3829574925 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbd4109af0 0x7ffbd4110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.868+0000 7ffbd9200700 1 --2- 172.21.15.28:0/3829574925 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 0x7ffbd4104940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.868+0000 7ffbd9200700 1 --2- 172.21.15.28:0/3829574925 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 0x7ffbd41093c0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.868+0000 7ffbd9200700 1 -- 172.21.15.28:0/3829574925 >> 172.21.15.28:0/3829574925 conn(0x7ffbd40fdd90 msgr2=0x7ffbd41001b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.869+0000 7ffbd9200700 1 -- 172.21.15.28:0/3829574925 shutdown_connections 2024-05-20T20:07:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.869+0000 7ffbd9200700 1 -- 172.21.15.28:0/3829574925 wait complete. 2024-05-20T20:07:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.870+0000 7ffbd9200700 1 Processor -- start 2024-05-20T20:07:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.870+0000 7ffbd9200700 1 -- start start 2024-05-20T20:07:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.870+0000 7ffbd9200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 0x7ffbd4199b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd9200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 0x7ffbd419a0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd9200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbd4109af0 0x7ffbd419e480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd9200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd4113670 con 0x7ffbd4104f10 2024-05-20T20:07:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd9200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41134f0 con 0x7ffbd4104560 2024-05-20T20:07:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd9200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41137f0 con 0x7ffbd4109af0 2024-05-20T20:07:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 0x7ffbd419a0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 0x7ffbd4199b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 0x7ffbd4199b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46328/0 (socket says 172.21.15.28:46328) 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 -- 172.21.15.28:0/1590352036 learned_addr learned my addr 172.21.15.28:0/1590352036 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbd4109af0 0x7ffbd419e480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 -- 172.21.15.28:0/1590352036 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbd4109af0 msgr2=0x7ffbd419e480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbd4109af0 0x7ffbd419e480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 -- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 msgr2=0x7ffbd419a0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 0x7ffbd419a0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.871+0000 7ffbd3400700 1 -- 172.21.15.28:0/1590352036 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffbd419eb80 con 0x7ffbd4104560 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.872+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 0x7ffbd419a0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.872+0000 7ffbd3400700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 0x7ffbd4199b80 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7ffbc400ea50 tx=0x7ffbc400ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:24.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.872+0000 7ffbcbe00700 1 -- 172.21.15.28:0/1590352036 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbc400cbd0 con 0x7ffbd4104560 2024-05-20T20:07:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.872+0000 7ffbcbe00700 1 -- 172.21.15.28:0/1590352036 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffbc400cd70 con 0x7ffbd4104560 2024-05-20T20:07:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.872+0000 7ffbcbe00700 1 -- 172.21.15.28:0/1590352036 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbc4018750 con 0x7ffbd4104560 2024-05-20T20:07:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.872+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffbd419ee70 con 0x7ffbd4104560 2024-05-20T20:07:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.872+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffbd41020f0 con 0x7ffbd4104560 2024-05-20T20:07:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.877+0000 7ffbcbe00700 1 -- 172.21.15.28:0/1590352036 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ffbc401f020 con 0x7ffbd4104560 2024-05-20T20:07:24.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.878+0000 7ffbe51fbb80 1 Processor -- start 2024-05-20T20:07:24.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbe51fbb80 1 -- start start 2024-05-20T20:07:24.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f3aa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f34b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f35600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c656e92fe0 con 0x55c656f3af70 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c656e82b80 con 0x55c656f3a650 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c656e86bc0 con 0x55c656f35180 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.879+0000 7ffbcbe00700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffbb80b2b20 0x7ffbb80b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f34b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f3aa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f3aa30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46330/0 (socket says 172.21.15.28:46330) 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 -- 172.21.15.28:0/3469266397 learned_addr learned my addr 172.21.15.28:0/3469266397 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffbb80b2b20 0x7ffbb80b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbcbe00700 1 -- 172.21.15.28:0/1590352036 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ffbc40e2750 con 0x7ffbd4104560 2024-05-20T20:07:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f35600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 -- 172.21.15.28:0/3469266397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 msgr2=0x55c656f35600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f35600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 -- 172.21.15.28:0/3469266397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 msgr2=0x55c656f34b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f34b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.880+0000 7ffbd3e00700 1 -- 172.21.15.28:0/3469266397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c656e4b5c0 con 0x55c656f3a650 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.881+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f3aa30 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7ffbcc008f90 tx=0x7ffbcc01a6c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=da5f1657c7eeedec server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.881+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffbb80b2b20 0x7ffbb80b4fe0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7ffbcc010000 tx=0x7ffbcc01e000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.881+0000 7ffbc8c00700 1 -- 172.21.15.28:0/3469266397 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbcc008640 con 0x55c656f3a650 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.881+0000 7ffbc8c00700 1 -- 172.21.15.28:0/3469266397 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffbcc01f040 con 0x55c656f3a650 2024-05-20T20:07:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.881+0000 7ffbc8c00700 1 -- 172.21.15.28:0/3469266397 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbcc024a80 con 0x55c656f3a650 2024-05-20T20:07:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3469266397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 msgr2=0x55c656f3aa30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f3aa30 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7ffbcc008f90 tx=0x7ffbcc01a6c0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3469266397 shutdown_connections 2024-05-20T20:07:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f35600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f3aa30 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3469266397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f34b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3469266397 >> 172.21.15.28:0/3469266397 conn(0x55c656e62d80 msgr2=0x55c656f2f6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:24.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.882+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3469266397 shutdown_connections 2024-05-20T20:07:24.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.883+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3469266397 wait complete. 2024-05-20T20:07:24.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 -- start start 2024-05-20T20:07:24.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 Processor -- start 2024-05-20T20:07:24.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f35180 0x55c656f40d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f3a650 0x55c656f3e110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f3e650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c656d89140 con 0x55c656f3af70 2024-05-20T20:07:24.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c656e23900 con 0x55c656f35180 2024-05-20T20:07:24.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.884+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c656f419c0 con 0x55c656f3a650 2024-05-20T20:07:24.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbe51fbb80 10 client.0 _ll_register_callbacks cb 0x55c656e2ce10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:24.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f35180 0x55c656f40d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f3e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f35180 0x55c656f40d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46340/0 (socket says 172.21.15.28:46340) 2024-05-20T20:07:24.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd3400700 1 -- 172.21.15.28:0/3571621290 learned_addr learned my addr 172.21.15.28:0/3571621290 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:24.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f3a650 0x55c656f3e110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd3400700 1 -- 172.21.15.28:0/3571621290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f3a650 msgr2=0x55c656f3e110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd3400700 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f3a650 0x55c656f3e110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.885+0000 7ffbd3400700 1 -- 172.21.15.28:0/3571621290 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 msgr2=0x55c656f3e650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbd3400700 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f3e650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbd3400700 1 -- 172.21.15.28:0/3571621290 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c656dd2a70 con 0x55c656f35180 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f3e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbd3400700 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f35180 0x55c656f40d70 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7ffbc40a5ac0 tx=0x7ffbc400aec0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ef046625578e7ea6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbbd600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbbd600700 1 -- 172.21.15.28:0/3571621290 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbc40b2040 con 0x55c656f35180 2024-05-20T20:07:24.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbbd600700 1 -- 172.21.15.28:0/3571621290 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ffbc40a9eb0 con 0x55c656f35180 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.886+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffbcc01f040 con 0x55c656f35180 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbbd600700 1 -- 172.21.15.28:0/3571621290 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffbc40a51d0 con 0x55c656f35180 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 10 client.25745 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c656e53200 con 0x55c656f35180 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbbd600700 1 -- 172.21.15.28:0/3571621290 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffbcc01f040 con 0x55c656f35180 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbbd600700 1 -- 172.21.15.28:0/3571621290 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffbc40b2410 con 0x55c656f35180 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 10 client.25745 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 10 client.25745 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 20 client.25745 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f4e620 0x55c656f50a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 4 client.25745 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:24.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.887+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c656e94f30 con 0x55c656f4e620 2024-05-20T20:07:24.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.888+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f4e620 0x55c656f50a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.889+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f4e620 0x55c656f50a10 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=65fb31b1050ba104 server_cookie=781cf58e8a08aa24 in_seq=0 out_seq=0 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.889+0000 7ffbbd600700 10 client.25745 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.889+0000 7ffbbd600700 1 -- 172.21.15.28:0/3571621290 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55c656e94f30 con 0x55c656f4e620 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.889+0000 7ffbbd600700 10 client.25745 handle_command_reply: tid=0 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.889+0000 7ffbe51fbb80 1 client.25745 shutdown 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f35180 msgr2=0x55c656f40d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f35180 0x55c656f40d70 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7ffbc40a5ac0 tx=0x7ffbc400aec0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 shutdown_connections 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f4e620 0x55c656f50a10 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f3af70 0x55c656f3e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f35180 0x55c656f40d70 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3571621290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f3a650 0x55c656f3e110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 >> 172.21.15.28:0/3571621290 conn(0x55c656e62d80 msgr2=0x55c656f33b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:24.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbbd600700 0 client.25745 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:24.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 shutdown_connections 2024-05-20T20:07:24.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3571621290 wait complete. 2024-05-20T20:07:24.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.890+0000 7ffbe51fbb80 20 client.25745 trim_cache size 0 max 16384 2024-05-20T20:07:24.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.917+0000 7ffbe51fbb80 1 Processor -- start 2024-05-20T20:07:24.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.917+0000 7ffbe51fbb80 1 -- start start 2024-05-20T20:07:24.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656e62d80 0x55c656f34b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f35180 0x55c656f29690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f562a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41ab9e0 con 0x55c656f35180 2024-05-20T20:07:24.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41ad520 con 0x55c656f3a650 2024-05-20T20:07:24.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41ac0b0 con 0x55c656e62d80 2024-05-20T20:07:24.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbd2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f35180 0x55c656f29690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656e62d80 0x55c656f34b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbd2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f35180 0x55c656f29690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40320/0 (socket says 172.21.15.28:40320) 2024-05-20T20:07:24.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.918+0000 7ffbd2a00700 1 -- 172.21.15.28:0/3364395893 learned_addr learned my addr 172.21.15.28:0/3364395893 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:24.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.919+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f562a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.919+0000 7ffbd2a00700 1 -- 172.21.15.28:0/3364395893 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656e62d80 msgr2=0x55c656f34b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.919+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656e62d80 0x55c656f34b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.919+0000 7ffbd2a00700 1 -- 172.21.15.28:0/3364395893 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 msgr2=0x55c656f562a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.919+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f562a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.919+0000 7ffbd2a00700 1 -- 172.21.15.28:0/3364395893 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c656f2bc70 con 0x55c656f35180 2024-05-20T20:07:24.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.919+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f35180 0x55c656f29690 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7ffbd406b500 tx=0x7ffbc0005040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b48703a0c5e535e8 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:24.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.920+0000 7ffbbfe00700 1 -- 172.21.15.28:0/3364395893 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbc0005330 con 0x55c656f35180 2024-05-20T20:07:24.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.920+0000 7ffbbfe00700 1 -- 172.21.15.28:0/3364395893 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffbc0003c70 con 0x55c656f35180 2024-05-20T20:07:24.922 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:24.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.920+0000 7ffbbfe00700 1 -- 172.21.15.28:0/3364395893 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbc000baf0 con 0x55c656f35180 2024-05-20T20:07:24.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3364395893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f35180 msgr2=0x55c656f29690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f35180 0x55c656f29690 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7ffbd406b500 tx=0x7ffbc0005040 comp rx=0 tx=0).stop 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3364395893 shutdown_connections 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656f562a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656e62d80 0x55c656f34b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/3364395893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656f35180 0x55c656f29690 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3364395893 >> 172.21.15.28:0/3364395893 conn(0x55c656f2fa00 msgr2=0x55c656f28f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3364395893 shutdown_connections 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/3364395893 wait complete. 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.921+0000 7ffbe51fbb80 1 -- start start 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 1 Processor -- start 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656e62d80 0x55c656f43fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f44520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656fb9180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41ace50 con 0x55c656e62d80 2024-05-20T20:07:24.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41ac780 con 0x55c656f3a650 2024-05-20T20:07:24.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffbd41a99a0 con 0x55c656f35180 2024-05-20T20:07:24.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.922+0000 7ffbe51fbb80 10 client.0 _ll_register_callbacks cb 0x55c656e2ce10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:24.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656e62d80 0x55c656f43fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f44520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656fb9180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656e62d80 0x55c656f43fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40336/0 (socket says 172.21.15.28:40336) 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 -- 172.21.15.28:0/1405845165 learned_addr learned my addr 172.21.15.28:0/1405845165 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 -- 172.21.15.28:0/1405845165 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 msgr2=0x55c656f44520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f44520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 -- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 msgr2=0x55c656fb9180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656fb9180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3400700 1 -- 172.21.15.28:0/1405845165 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c656fb96c0 con 0x55c656e62d80 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.923+0000 7ffbd3e00700 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656fb9180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbd3400700 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656e62d80 0x55c656f43fe0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7ffbc400afd0 tx=0x7ffbc40adeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=25a21256d03c3a67 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbbd600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbbd600700 1 -- 172.21.15.28:0/1405845165 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffbc40a9550 con 0x55c656e62d80 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbbd600700 1 -- 172.21.15.28:0/1405845165 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ffbc4010430 con 0x55c656e62d80 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffbc0003c70 con 0x55c656e62d80 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbbd600700 1 -- 172.21.15.28:0/1405845165 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffbc40aa140 con 0x55c656e62d80 2024-05-20T20:07:24.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbe51fbb80 10 client.15606 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.924+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c656fba900 con 0x55c656e62d80 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbbd600700 1 -- 172.21.15.28:0/1405845165 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffbc0003c70 con 0x55c656e62d80 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbbd600700 1 -- 172.21.15.28:0/1405845165 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffbc40b2af0 con 0x55c656e62d80 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbe51fbb80 10 client.15606 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbe51fbb80 10 client.15606 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbe51fbb80 20 client.15606 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f3af70 0x55c656fbcfd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbe51fbb80 4 client.15606 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 146.35, "await_for": 45.0}] 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.925+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 146.35, "await_for": 45.0}) -- 0x55c656e8e710 con 0x55c656f3af70 2024-05-20T20:07:24.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.926+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f3af70 0x55c656fbcfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:24.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.926+0000 7ffbd2a00700 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f3af70 0x55c656fbcfd0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=da1b850c210d4b54 server_cookie=af2a7af55f3941d in_seq=0 out_seq=0 2024-05-20T20:07:24.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.926+0000 7ffbbd600700 10 client.15606 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.935+0000 7ffbbd600700 1 -- 172.21.15.28:0/1405845165 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+589 (crc 0 0 0) 0x55c656e8e710 con 0x55c656f3af70 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.935+0000 7ffbbd600700 10 client.15606 handle_command_reply: tid=0 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.935+0000 7ffbe51fbb80 1 client.15606 shutdown 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.935+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656e62d80 msgr2=0x55c656f43fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.935+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656e62d80 0x55c656f43fe0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7ffbc400afd0 tx=0x7ffbc40adeb0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 shutdown_connections 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c656f3af70 0x55c656fbcfd0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c656f3a650 0x55c656fb9180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c656e62d80 0x55c656f43fe0 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 --2- 172.21.15.28:0/1405845165 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c656f35180 0x55c656f44520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 >> 172.21.15.28:0/1405845165 conn(0x55c656f2fa00 msgr2=0x55c656f3ca00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:24.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbbd600700 0 client.15606 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:24.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 shutdown_connections 2024-05-20T20:07:24.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 1 -- 172.21.15.28:0/1405845165 wait complete. 2024-05-20T20:07:24.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.936+0000 7ffbe51fbb80 20 client.15606 trim_cache size 0 max 16384 2024-05-20T20:07:24.938 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:07:24.938 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 2, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "f564f325": { 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 2, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 146.3, 2024-05-20T20:07:24.939 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:24.940 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffbb80b2b20 msgr2=0x7ffbb80b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffbb80b2b20 0x7ffbb80b4fe0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7ffbcc010000 tx=0x7ffbcc01e000 comp rx=0 tx=0).stop 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 msgr2=0x7ffbd4199b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 0x7ffbd4199b80 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7ffbc400ea50 tx=0x7ffbc400ed60 comp rx=0 tx=0).stop 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 shutdown_connections 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbd4109af0 0x7ffbd419e480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffbb80b2b20 0x7ffbb80b4fe0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbd4104560 0x7ffbd4199b80 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 --2- 172.21.15.28:0/1590352036 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbd4104f10 0x7ffbd419a0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.937+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 >> 172.21.15.28:0/1590352036 conn(0x7ffbd40fdd90 msgr2=0x7ffbd40ff920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:24.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.938+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 shutdown_connections 2024-05-20T20:07:24.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:24.938+0000 7ffbd9200700 1 -- 172.21.15.28:0/1590352036 wait complete. 2024-05-20T20:07:25.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:24 smithi154 ceph-mon[36314]: pgmap v56: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 341 B/s wr, 14 op/s 2024-05-20T20:07:25.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:24 smithi033 ceph-mon[36179]: pgmap v56: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 341 B/s wr, 14 op/s 2024-05-20T20:07:25.852 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set 'f564f325', quiesce took 10.7 seconds. Will release after: 15.650000000000002 2024-05-20T20:07:27.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:26 smithi028 ceph-mon[31178]: pgmap v57: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:27.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:26 smithi154 ceph-mon[36314]: pgmap v57: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:26 smithi033 ceph-mon[36179]: pgmap v57: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:27.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.983+0000 7f0b0cc00700 1 -- 172.21.15.28:0/1508410047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 msgr2=0x7f0b08105ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:27.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.983+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/1508410047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b08105ad0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f0af00098f0 tx=0x7f0af0009c00 comp rx=0 tx=0).stop 2024-05-20T20:07:27.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.983+0000 7f0b0cc00700 1 -- 172.21.15.28:0/1508410047 shutdown_connections 2024-05-20T20:07:27.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.983+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/1508410047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b08106010 0x7f0b08112520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:27.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.983+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/1508410047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b0810b650 0x7f0b0810ba30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:27.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.983+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/1508410047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b08105ad0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:27.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.983+0000 7f0b0cc00700 1 -- 172.21.15.28:0/1508410047 >> 172.21.15.28:0/1508410047 conn(0x7f0b08101120 msgr2=0x7f0b08103540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:27.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.985+0000 7f0b0cc00700 1 -- 172.21.15.28:0/1508410047 shutdown_connections 2024-05-20T20:07:27.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.985+0000 7f0b0cc00700 1 -- 172.21.15.28:0/1508410047 wait complete. 2024-05-20T20:07:27.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.986+0000 7f0b0cc00700 1 Processor -- start 2024-05-20T20:07:27.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.986+0000 7f0b0cc00700 1 -- start start 2024-05-20T20:07:27.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b0cc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b0819bdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:27.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b0cc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b08106010 0x7f0b0819c2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:27.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b0cc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b0810b650 0x7f0b081a0630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:27.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b0cc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0b081158a0 con 0x7f0b0810b650 2024-05-20T20:07:27.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b0cc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0b08115720 con 0x7f0b08105650 2024-05-20T20:07:27.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b0cc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0b08115a20 con 0x7f0b08106010 2024-05-20T20:07:27.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b07400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b0819bdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:27.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b07400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b0819bdb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42360/0 (socket says 172.21.15.28:42360) 2024-05-20T20:07:27.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b07400700 1 -- 172.21.15.28:0/2741097437 learned_addr learned my addr 172.21.15.28:0/2741097437 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:27.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.987+0000 7f0b07e00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b0810b650 0x7f0b081a0630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0affe00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b08106010 0x7f0b0819c2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0b07400700 1 -- 172.21.15.28:0/2741097437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b08106010 msgr2=0x7f0b0819c2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0b07400700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b08106010 0x7f0b0819c2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0b07400700 1 -- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b0810b650 msgr2=0x7f0b081a0630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0b07400700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b0810b650 0x7f0b081a0630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0b07400700 1 -- 172.21.15.28:0/2741097437 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b081a0d70 con 0x7f0b08105650 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0affe00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b08106010 0x7f0b0819c2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0b07e00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b0810b650 0x7f0b081a0630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.988+0000 7f0b07400700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b0819bdb0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f0af800e9c0 tx=0x7f0af800ed80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:27.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.989+0000 7f0b04c00700 1 -- 172.21.15.28:0/2741097437 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0af800cbd0 con 0x7f0b08105650 2024-05-20T20:07:27.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.989+0000 7f0b04c00700 1 -- 172.21.15.28:0/2741097437 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0af800cd70 con 0x7f0b08105650 2024-05-20T20:07:27.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.989+0000 7f0b04c00700 1 -- 172.21.15.28:0/2741097437 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0af80056a0 con 0x7f0b08105650 2024-05-20T20:07:27.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.989+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0b081a1060 con 0x7f0b08105650 2024-05-20T20:07:27.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.989+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0b08077980 con 0x7f0b08105650 2024-05-20T20:07:27.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.994+0000 7f0b04c00700 1 -- 172.21.15.28:0/2741097437 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0af8004750 con 0x7f0b08105650 2024-05-20T20:07:27.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.996+0000 7f0b18c2ab80 1 Processor -- start 2024-05-20T20:07:27.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.996+0000 7f0b18c2ab80 1 -- start start 2024-05-20T20:07:27.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.996+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe02400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:27.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fdfc4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:27.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fdfcf30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:27.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a71fd5ae40 con 0x55a71fe02020 2024-05-20T20:07:27.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a71fd4a9e0 con 0x55a71fe02940 2024-05-20T20:07:27.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a71fd4ea20 con 0x55a71fdfcad0 2024-05-20T20:07:27.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b07e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe02400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:27.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b07e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe02400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38054/0 (socket says 172.21.15.28:38054) 2024-05-20T20:07:27.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b07e00700 1 -- 172.21.15.28:0/2371770271 learned_addr learned my addr 172.21.15.28:0/2371770271 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:27.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0affe00700 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fdfcf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:27.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.997+0000 7f0b07400700 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fdfc4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b07e00700 1 -- 172.21.15.28:0/2371770271 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 msgr2=0x55a71fdfcf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b07e00700 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fdfcf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b07e00700 1 -- 172.21.15.28:0/2371770271 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 msgr2=0x55a71fdfc4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b07e00700 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fdfc4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b07e00700 1 -- 172.21.15.28:0/2371770271 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a71fd13420 con 0x55a71fe02020 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b07400700 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fdfc4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0affe00700 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fdfcf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:28.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b04c00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0aec0b2a70 0x7f0aec0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b04c00700 1 -- 172.21.15.28:0/2741097437 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0af80e27f0 con 0x7f0b08105650 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.998+0000 7f0b07e00700 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe02400 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f0b00009440 tx=0x7f0b00009920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=79b7810f96180213 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.999+0000 7f0afcc00700 1 -- 172.21.15.28:0/2371770271 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b00015640 con 0x55a71fe02020 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.999+0000 7f0afcc00700 1 -- 172.21.15.28:0/2371770271 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0b00015c90 con 0x55a71fe02020 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.999+0000 7f0afcc00700 1 -- 172.21.15.28:0/2371770271 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b00014e00 con 0x55a71fe02020 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:27.999+0000 7f0b07400700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0aec0b2a70 0x7f0aec0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/2371770271 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 msgr2=0x55a71fe02400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe02400 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f0b00009440 tx=0x7f0b00009920 comp rx=0 tx=0).stop 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b07400700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0aec0b2a70 0x7f0aec0b4f30 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f0af8012020 tx=0x7f0af8005910 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/2371770271 shutdown_connections 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fdfcf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe02400 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/2371770271 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fdfc4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/2371770271 >> 172.21.15.28:0/2371770271 conn(0x55a71fd2abe0 msgr2=0x55a71fdf7150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:28.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.000+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/2371770271 shutdown_connections 2024-05-20T20:07:28.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.001+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/2371770271 wait complete. 2024-05-20T20:07:28.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.001+0000 7f0b18c2ab80 1 -- start start 2024-05-20T20:07:28.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 1 Processor -- start 2024-05-20T20:07:28.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fe08730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe06010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a71fc50fa0 con 0x55a71fe02020 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a71fceb760 con 0x55a71fe02940 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a71fe09460 con 0x55a71fdfcad0 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.002+0000 7f0b18c2ab80 10 client.0 _ll_register_callbacks cb 0x55a71fcf4c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fe08730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fe08730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:51958/0 (socket says 172.21.15.28:51958) 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 -- 172.21.15.28:0/3226205535 learned_addr learned my addr 172.21.15.28:0/3226205535 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe06010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0affe00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 -- 172.21.15.28:0/3226205535 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 msgr2=0x55a71fe08730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fe08730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 -- 172.21.15.28:0/3226205535 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 msgr2=0x55a71fe05ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0b07e00700 1 -- 172.21.15.28:0/3226205535 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a71fc9a8d0 con 0x55a71fe02940 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.003+0000 7f0affe00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.004+0000 7f0b07e00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe06010 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0b00009410 tx=0x7f0b0000b040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=741c51af0dfde8ee server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.004+0000 7f0af5600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.004+0000 7f0af5600700 1 -- 172.21.15.28:0/3226205535 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b00014e40 con 0x55a71fe02940 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.004+0000 7f0af5600700 1 -- 172.21.15.28:0/3226205535 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0b00015440 con 0x55a71fe02940 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.004+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0b00015c90 con 0x55a71fe02940 2024-05-20T20:07:28.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0af5600700 1 -- 172.21.15.28:0/3226205535 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0b00033050 con 0x55a71fe02940 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 10 client.25760 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a71fd1b060 con 0x55a71fe02940 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0af5600700 1 -- 172.21.15.28:0/3226205535 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0b00015c90 con 0x55a71fe02940 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0af5600700 1 -- 172.21.15.28:0/3226205535 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0b000054f0 con 0x55a71fe02940 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 10 client.25760 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 10 client.25760 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 20 client.25760 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe15eb0 0x55a71fe182a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 4 client.25760 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:28.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.005+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a71fd5cd90 con 0x55a71fe15eb0 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.006+0000 7f0affe00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe15eb0 0x55a71fe182a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.006+0000 7f0affe00700 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe15eb0 0x55a71fe182a0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=1fa62a7e2dc7662f server_cookie=87cf289543a850eb in_seq=0 out_seq=0 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.006+0000 7f0af5600700 10 client.25760 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.007+0000 7f0af5600700 1 -- 172.21.15.28:0/3226205535 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a71fd5cd90 con 0x55a71fe15eb0 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.007+0000 7f0af5600700 10 client.25760 handle_command_reply: tid=0 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.007+0000 7f0b18c2ab80 1 client.25760 shutdown 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 msgr2=0x55a71fe06010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe06010 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0b00009410 tx=0x7f0b0000b040 comp rx=0 tx=0).stop 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 shutdown_connections 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe06010 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe15eb0 0x55a71fe182a0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fdfcad0 0x55a71fe08730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3226205535 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 >> 172.21.15.28:0/3226205535 conn(0x55a71fd2abe0 msgr2=0x55a71fdfb4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:28.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0af5600700 0 client.25760 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:07:28.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.008+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 shutdown_connections 2024-05-20T20:07:28.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.009+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3226205535 wait complete. 2024-05-20T20:07:28.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.009+0000 7f0b18c2ab80 20 client.25760 trim_cache size 0 max 16384 2024-05-20T20:07:28.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.025+0000 7f0b18c2ab80 1 Processor -- start 2024-05-20T20:07:28.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.025+0000 7f0b18c2ab80 1 -- start start 2024-05-20T20:07:28.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fdfcad0 0x55a71fdfc4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe02940 0x55a71fe1d840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0b081b43f0 con 0x55a71fe02020 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0b081b5f30 con 0x55a71fdfcad0 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0b081b4ac0 con 0x55a71fe02940 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0affe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0affe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38068/0 (socket says 172.21.15.28:38068) 2024-05-20T20:07:28.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0affe00700 1 -- 172.21.15.28:0/349180933 learned_addr learned my addr 172.21.15.28:0/349180933 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:28.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b07e00700 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fdfcad0 0x55a71fdfc4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.026+0000 7f0b07e00700 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe02940 0x55a71fe1d840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0affe00700 1 -- 172.21.15.28:0/349180933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe02940 msgr2=0x55a71fe1d840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:28.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0affe00700 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe02940 0x55a71fe1d840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0affe00700 1 -- 172.21.15.28:0/349180933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fdfcad0 msgr2=0x55a71fdfc4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0affe00700 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fdfcad0 0x55a71fdfc4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0affe00700 1 -- 172.21.15.28:0/349180933 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a71fe18610 con 0x55a71fe02020 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0b07e00700 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fdfcad0 0x55a71fdfc4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0affe00700 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05fc0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f0af00098f0 tx=0x7f0af0005e00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=23e964654fd75c54 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.027+0000 7f0af7e00700 1 -- 172.21.15.28:0/349180933 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0af0005040 con 0x55a71fe02020 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.028+0000 7f0af7e00700 1 -- 172.21.15.28:0/349180933 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0af0005690 con 0x55a71fe02020 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.028+0000 7f0af7e00700 1 -- 172.21.15.28:0/349180933 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0af000b7c0 con 0x55a71fe02020 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.028+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/349180933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 msgr2=0x55a71fe05fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:28.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.028+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05fc0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f0af00098f0 tx=0x7f0af0005e00 comp rx=0 tx=0).stop 2024-05-20T20:07:28.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/349180933 shutdown_connections 2024-05-20T20:07:28.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe02940 0x55a71fe1d840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fdfcad0 0x55a71fdfc4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/349180933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fe02020 0x55a71fe05fc0 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/349180933 >> 172.21.15.28:0/349180933 conn(0x55a71fd2abe0 msgr2=0x55a71fe058b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:28.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/349180933 shutdown_connections 2024-05-20T20:07:28.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/349180933 wait complete. 2024-05-20T20:07:28.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.029+0000 7f0b18c2ab80 1 -- start start 2024-05-20T20:07:28.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.030+0000 7f0b18c2ab80 1 Processor -- start 2024-05-20T20:07:28.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.030+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fdfcad0 0x55a71fe0eda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.030+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe1d0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.030+0000 7f0b18c2ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe1d5f0 0x55a71fe1da70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.030+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0b081b5860 con 0x55a71fdfcad0 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fdfcad0 0x55a71fe0eda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0b081b5190 con 0x55a71fe02940 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b18c2ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0b081b23c0 con 0x55a71fe1d5f0 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fdfcad0 0x55a71fe0eda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38072/0 (socket says 172.21.15.28:38072) 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 -- 172.21.15.28:0/3876965628 learned_addr learned my addr 172.21.15.28:0/3876965628 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b18c2ab80 10 client.0 _ll_register_callbacks cb 0x55a71fcf4c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:28.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0affe00700 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe1d0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 -- 172.21.15.28:0/3876965628 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe1d5f0 msgr2=0x55a71fe1da70 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:07:28.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe1d5f0 0x55a71fe1da70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 -- 172.21.15.28:0/3876965628 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 msgr2=0x55a71fe1d0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:28.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe1d0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.031+0000 7f0b07e00700 1 -- 172.21.15.28:0/3876965628 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a71fe7f810 con 0x55a71fdfcad0 2024-05-20T20:07:28.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.032+0000 7f0b07e00700 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fdfcad0 0x55a71fe0eda0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f0b00002230 tx=0x7f0b00004f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e68c44f5ebd98fbb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:28.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.032+0000 7f0af5600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:28.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.032+0000 7f0af5600700 1 -- 172.21.15.28:0/3876965628 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b0003b070 con 0x55a71fdfcad0 2024-05-20T20:07:28.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.032+0000 7f0af5600700 1 -- 172.21.15.28:0/3876965628 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0b00003c00 con 0x55a71fdfcad0 2024-05-20T20:07:28.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.032+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0af0005690 con 0x55a71fdfcad0 2024-05-20T20:07:28.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0af5600700 1 -- 172.21.15.28:0/3876965628 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0b00044050 con 0x55a71fdfcad0 2024-05-20T20:07:28.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0b18c2ab80 10 client.15624 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:28.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a71fe80af0 con 0x55a71fdfcad0 2024-05-20T20:07:28.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0af5600700 1 -- 172.21.15.28:0/3876965628 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0af0005690 con 0x55a71fdfcad0 2024-05-20T20:07:28.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0af5600700 1 -- 172.21.15.28:0/3876965628 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0b00005160 con 0x55a71fdfcad0 2024-05-20T20:07:28.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0b18c2ab80 10 client.15624 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:28.036 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0b18c2ab80 10 client.15624 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:07:28.036 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0b18c2ab80 20 client.15624 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:28.036 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.033+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe81230 0x55a71fe83650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:28.036 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.034+0000 7f0b18c2ab80 4 client.15624 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:28.036 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.034+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55a71fd56570 con 0x55a71fe81230 2024-05-20T20:07:28.036 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.034+0000 7f0affe00700 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe81230 0x55a71fe83650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.035+0000 7f0affe00700 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe81230 0x55a71fe83650 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=81b844ffca9500f5 server_cookie=69819f4553fa0d69 in_seq=0 out_seq=0 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.035+0000 7f0af5600700 10 client.15624 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.035+0000 7f0af5600700 1 -- 172.21.15.28:0/3876965628 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55a71fd56570 con 0x55a71fe81230 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.035+0000 7f0af5600700 10 client.15624 handle_command_reply: tid=0 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.035+0000 7f0b18c2ab80 1 client.15624 shutdown 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fdfcad0 msgr2=0x55a71fe0eda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fdfcad0 0x55a71fe0eda0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f0b00002230 tx=0x7f0b00004f10 comp rx=0 tx=0).stop 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 shutdown_connections 2024-05-20T20:07:28.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55a71fe81230 0x55a71fe83650 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a71fe1d5f0 0x55a71fe1da70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a71fdfcad0 0x55a71fe0eda0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 --2- 172.21.15.28:0/3876965628 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a71fe02940 0x55a71fe1d0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 >> 172.21.15.28:0/3876965628 conn(0x55a71fd2abe0 msgr2=0x55a71fe043c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:28.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0af5600700 0 client.15624 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:07:28.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 shutdown_connections 2024-05-20T20:07:28.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.036+0000 7f0b18c2ab80 1 -- 172.21.15.28:0/3876965628 wait complete. 2024-05-20T20:07:28.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.037+0000 7f0b18c2ab80 20 client.15624 trim_cache size 0 max 16384 2024-05-20T20:07:28.039 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0aec0b2a70 msgr2=0x7f0aec0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0aec0b2a70 0x7f0aec0b4f30 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f0af8012020 tx=0x7f0af8005910 comp rx=0 tx=0).stop 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 msgr2=0x7f0b0819bdb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b0819bdb0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f0af800e9c0 tx=0x7f0af800ed80 comp rx=0 tx=0).stop 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 shutdown_connections 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b0810b650 0x7f0b081a0630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0aec0b2a70 0x7f0aec0b4f30 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b08105650 0x7f0b0819bdb0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 --2- 172.21.15.28:0/2741097437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b08106010 0x7f0b0819c2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 >> 172.21.15.28:0/2741097437 conn(0x7f0b08101120 msgr2=0x7f0b081018b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:28.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.038+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 shutdown_connections 2024-05-20T20:07:28.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:28.039+0000 7f0b0cc00700 1 -- 172.21.15.28:0/2741097437 wait complete. 2024-05-20T20:07:28.893 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:07:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:28 smithi154 ceph-mon[36314]: pgmap v58: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:29.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:28 smithi033 ceph-mon[36179]: pgmap v58: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:29.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:28 smithi028 ceph-mon[31178]: pgmap v58: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:29.327 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.761+0000 7ff7ad600640 1 -- 172.21.15.28:0/1987163662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8105850 msgr2=0x7ff7a8109890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.761+0000 7ff7ad600640 1 --2- 172.21.15.28:0/1987163662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8105850 0x7ff7a8109890 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7ff794009900 tx=0x7ff79402a420 comp rx=0 tx=0).stop 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.761+0000 7ff7ad600640 1 -- 172.21.15.28:0/1987163662 shutdown_connections 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.761+0000 7ff7ad600640 1 --2- 172.21.15.28:0/1987163662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7a8109f70 0x7ff7a8111b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.761+0000 7ff7ad600640 1 --2- 172.21.15.28:0/1987163662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8105850 0x7ff7a8109890 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.761+0000 7ff7ad600640 1 --2- 172.21.15.28:0/1987163662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff7a8104ea0 0x7ff7a8105280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.761+0000 7ff7ad600640 1 -- 172.21.15.28:0/1987163662 >> 172.21.15.28:0/1987163662 conn(0x7ff7a8100740 msgr2=0x7ff7a8102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.762+0000 7ff7ad600640 1 -- 172.21.15.28:0/1987163662 shutdown_connections 2024-05-20T20:07:30.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.762+0000 7ff7ad600640 1 -- 172.21.15.28:0/1987163662 wait complete. 2024-05-20T20:07:30.764 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.763+0000 7ff7ad600640 1 Processor -- start 2024-05-20T20:07:30.764 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.763+0000 7ff7ad600640 1 -- start start 2024-05-20T20:07:30.765 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7ad600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8104ea0 0x7ff7a819b590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.765 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7ad600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7a8109f70 0x7ff7a819bad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.765 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7ad600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff7a819fe00 0x7ff7a81a02b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.765 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7ad600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a8114fb0 con 0x7ff7a8104ea0 2024-05-20T20:07:30.765 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7ad600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a8114e30 con 0x7ff7a819fe00 2024-05-20T20:07:30.765 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7ad600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a8115130 con 0x7ff7a8109f70 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7a6a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8104ea0 0x7ff7a819b590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7a6000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7a8109f70 0x7ff7a819bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7a6000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7a8109f70 0x7ff7a819bad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52004/0 (socket says 172.21.15.28:52004) 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7a6000640 1 -- 172.21.15.28:0/3217104522 learned_addr learned my addr 172.21.15.28:0/3217104522 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7a6a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8104ea0 0x7ff7a819b590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38088/0 (socket says 172.21.15.28:38088) 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.764+0000 7ff7a7400640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff7a819fe00 0x7ff7a81a02b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.765+0000 7ff7a6a00640 1 -- 172.21.15.28:0/3217104522 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7a8109f70 msgr2=0x7ff7a819bad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.765+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7a8109f70 0x7ff7a819bad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.765+0000 7ff7a6a00640 1 -- 172.21.15.28:0/3217104522 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff7a819fe00 msgr2=0x7ff7a81a02b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.765+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff7a819fe00 0x7ff7a81a02b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.765+0000 7ff7a6a00640 1 -- 172.21.15.28:0/3217104522 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff7a81a08f0 con 0x7ff7a8104ea0 2024-05-20T20:07:30.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.765+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8104ea0 0x7ff7a819b590 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7ff79800b3f0 tx=0x7ff79800b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:30.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.765+0000 7ff79f400640 1 -- 172.21.15.28:0/3217104522 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff798015020 con 0x7ff7a8104ea0 2024-05-20T20:07:30.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.766+0000 7ff79f400640 1 -- 172.21.15.28:0/3217104522 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff798011ce0 con 0x7ff7a8104ea0 2024-05-20T20:07:30.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.766+0000 7ff79f400640 1 -- 172.21.15.28:0/3217104522 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff798010d80 con 0x7ff7a8104ea0 2024-05-20T20:07:30.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.766+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff7a81a0be0 con 0x7ff7a8104ea0 2024-05-20T20:07:30.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.766+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff7a81a83d0 con 0x7ff7a8104ea0 2024-05-20T20:07:30.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.770+0000 7ff7b03a2740 1 Processor -- start 2024-05-20T20:07:30.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.771+0000 7ff7b03a2740 1 -- start start 2024-05-20T20:07:30.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff79f400640 1 -- 172.21.15.28:0/3217104522 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ff798018020 con 0x7ff7a8104ea0 2024-05-20T20:07:30.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfe0880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe0dc0 0x55733dfeb140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeda70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55733df48b90 con 0x55733dfe04a0 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55733df38730 con 0x55733dfeb680 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55733df3c770 con 0x55733dfe0dc0 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7a6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeda70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7a6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeda70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42420/0 (socket says 172.21.15.28:42420) 2024-05-20T20:07:30.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.772+0000 7ff7a6000640 1 -- 172.21.15.28:0/1304430034 learned_addr learned my addr 172.21.15.28:0/1304430034 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:30.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.773+0000 7ff7a7400640 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfe0880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.773+0000 7ff7a6000640 1 -- 172.21.15.28:0/1304430034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe0dc0 msgr2=0x55733dfeb140 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:07:30.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.773+0000 7ff7a6000640 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe0dc0 0x55733dfeb140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.773+0000 7ff7a6000640 1 -- 172.21.15.28:0/1304430034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 msgr2=0x55733dfe0880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:30.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.773+0000 7ff7a6000640 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfe0880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.773+0000 7ff7a6000640 1 -- 172.21.15.28:0/1304430034 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55733ded18b0 con 0x55733dfeb680 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.773+0000 7ff7a7400640 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfe0880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.774+0000 7ff7a6000640 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeda70 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7ff794006010 tx=0x7ff794032e50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9253a6209a330888 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.774+0000 7ff793400640 1 -- 172.21.15.28:0/1304430034 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff794035040 con 0x55733dfeb680 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.774+0000 7ff793400640 1 -- 172.21.15.28:0/1304430034 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff794031760 con 0x55733dfeb680 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.774+0000 7ff793400640 1 -- 172.21.15.28:0/1304430034 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff794003e10 con 0x55733dfeb680 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.774+0000 7ff79f400640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff78c0b2b50 0x7ff78c0b5010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.774+0000 7ff79f400640 1 -- 172.21.15.28:0/3217104522 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff7980e1c10 con 0x7ff7a8104ea0 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.776+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1304430034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 msgr2=0x55733dfeda70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.776+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeda70 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7ff794006010 tx=0x7ff794032e50 comp rx=0 tx=0).stop 2024-05-20T20:07:30.777 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.776+0000 7ff7a7400640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff78c0b2b50 0x7ff78c0b5010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7a7400640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff78c0b2b50 0x7ff78c0b5010 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7ff7a000a510 tx=0x7ff7a0009650 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:30.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1304430034 shutdown_connections 2024-05-20T20:07:30.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeda70 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe0dc0 0x55733dfeb140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1304430034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfe0880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1304430034 >> 172.21.15.28:0/1304430034 conn(0x55733dfdc9b0 msgr2=0x55733dfdcd90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:30.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1304430034 shutdown_connections 2024-05-20T20:07:30.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.777+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1304430034 wait complete. 2024-05-20T20:07:30.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.778+0000 7ff7b03a2740 1 -- start start 2024-05-20T20:07:30.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.778+0000 7ff7b03a2740 1 Processor -- start 2024-05-20T20:07:30.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfeb070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe0dc0 0x55733dfe8360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfe8970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55733dc1e790 con 0x55733dfe04a0 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55733dee0bc0 con 0x55733dfe0dc0 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55733dfee250 con 0x55733dfeb680 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7b03a2740 10 client.0 _ll_register_callbacks cb 0x55733debe400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7a6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe0dc0 0x55733dfe8360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7a6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe0dc0 0x55733dfe8360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42434/0 (socket says 172.21.15.28:42434) 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7a6000640 1 -- 172.21.15.28:0/891288491 learned_addr learned my addr 172.21.15.28:0/891288491 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfeb070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.779+0000 7ff7a7400640 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfe8970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.780+0000 7ff7a6000640 1 -- 172.21.15.28:0/891288491 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 msgr2=0x55733dfe8970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.780+0000 7ff7a6000640 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfe8970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.780+0000 7ff7a6000640 1 -- 172.21.15.28:0/891288491 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 msgr2=0x55733dfeb070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.780+0000 7ff7a6000640 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfeb070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.780+0000 7ff7a6000640 1 -- 172.21.15.28:0/891288491 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55733dc4d210 con 0x55733dfe0dc0 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.780+0000 7ff7a6000640 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe0dc0 0x55733dfe8360 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7ff794000c00 tx=0x7ff79402a420 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1b6967eedca7f79 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.780+0000 7ff787e00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:30.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.781+0000 7ff787e00640 1 -- 172.21.15.28:0/891288491 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff79404e020 con 0x55733dfe0dc0 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.781+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff794031760 con 0x55733dfe0dc0 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.781+0000 7ff787e00640 1 -- 172.21.15.28:0/891288491 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff794003da0 con 0x55733dfe0dc0 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.781+0000 7ff787e00640 1 -- 172.21.15.28:0/891288491 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff794059050 con 0x55733dfe0dc0 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.781+0000 7ff7b03a2740 10 client.25775 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.781+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55733dd19620 con 0x55733dfe0dc0 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff787e00640 1 -- 172.21.15.28:0/891288491 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff794031760 con 0x55733dfe0dc0 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff787e00640 1 -- 172.21.15.28:0/891288491 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff79403c940 con 0x55733dfe0dc0 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff7b03a2740 10 client.25775 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff7b03a2740 10 client.25775 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff7b03a2740 20 client.25775 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733dffae20 0x55733dffd210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff7b03a2740 4 client.25775 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:30.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55733df4aae0 con 0x55733dffae20 2024-05-20T20:07:30.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.782+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733dffae20 0x55733dffd210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.783+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733dffae20 0x55733dffd210 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a0991e41c6c0f55b server_cookie=a39d2106b3549465 in_seq=0 out_seq=0 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.783+0000 7ff787e00640 10 client.25775 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff787e00640 1 -- 172.21.15.28:0/891288491 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55733df4aae0 con 0x55733dffae20 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff787e00640 10 client.25775 handle_command_reply: tid=0 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 client.25775 shutdown 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe0dc0 msgr2=0x55733dfe8360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe0dc0 0x55733dfe8360 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7ff794000c00 tx=0x7ff79402a420 comp rx=0 tx=0).stop 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 shutdown_connections 2024-05-20T20:07:30.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733dffae20 0x55733dffd210 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe0dc0 0x55733dfe8360 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfe8970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/891288491 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe04a0 0x55733dfeb070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 >> 172.21.15.28:0/891288491 conn(0x55733dfdc9b0 msgr2=0x55733dfd3310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.784+0000 7ff787e00640 0 client.25775 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.785+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 shutdown_connections 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.785+0000 7ff7b03a2740 1 -- 172.21.15.28:0/891288491 wait complete. 2024-05-20T20:07:30.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.785+0000 7ff7b03a2740 20 client.25775 trim_cache size 0 max 16384 2024-05-20T20:07:30.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.817+0000 7ff7b03a2740 1 Processor -- start 2024-05-20T20:07:30.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.817+0000 7ff7b03a2740 1 -- start start 2024-05-20T20:07:30.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe04a0 0x55733dff72b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfd3190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeaba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a81ac470 con 0x55733dfe0dc0 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a81adfb0 con 0x55733dfeb680 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a81acb40 con 0x55733dfe04a0 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7a7400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeaba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7a7400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeaba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42446/0 (socket says 172.21.15.28:42446) 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7a7400640 1 -- 172.21.15.28:0/1358827499 learned_addr learned my addr 172.21.15.28:0/1358827499 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.818+0000 7ff7a6000640 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfd3190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.819+0000 7ff7a7400640 1 -- 172.21.15.28:0/1358827499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe04a0 msgr2=0x55733dff72b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.819+0000 7ff7a7400640 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe04a0 0x55733dff72b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.819+0000 7ff7a7400640 1 -- 172.21.15.28:0/1358827499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 msgr2=0x55733dfd3190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.819+0000 7ff7a7400640 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfd3190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.819+0000 7ff7a7400640 1 -- 172.21.15.28:0/1358827499 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55733dfd26b0 con 0x55733dfeb680 2024-05-20T20:07:30.821 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.820+0000 7ff7a7400640 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeaba0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7ff7a0018900 tx=0x7ff7a0015000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c79bd9fff39e613d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:30.821 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.820+0000 7ff792a00640 1 -- 172.21.15.28:0/1358827499 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff7a00165b0 con 0x55733dfeb680 2024-05-20T20:07:30.821 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.820+0000 7ff792a00640 1 -- 172.21.15.28:0/1358827499 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff7a0016c00 con 0x55733dfeb680 2024-05-20T20:07:30.821 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.820+0000 7ff792a00640 1 -- 172.21.15.28:0/1358827499 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff7a0023d30 con 0x55733dfeb680 2024-05-20T20:07:30.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1358827499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 msgr2=0x55733dfeaba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeaba0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7ff7a0018900 tx=0x7ff7a0015000 comp rx=0 tx=0).stop 2024-05-20T20:07:30.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1358827499 shutdown_connections 2024-05-20T20:07:30.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfeb680 0x55733dfeaba0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfd3190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/1358827499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfe04a0 0x55733dff72b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1358827499 >> 172.21.15.28:0/1358827499 conn(0x55733dfdc9b0 msgr2=0x55733dfd2a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:30.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1358827499 shutdown_connections 2024-05-20T20:07:30.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.821+0000 7ff7b03a2740 1 -- 172.21.15.28:0/1358827499 wait complete. 2024-05-20T20:07:30.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.822+0000 7ff7b03a2740 1 -- start start 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.822+0000 7ff7b03a2740 1 Processor -- start 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe04a0 0x55733dff3c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfeeff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7b03a2740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfef600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a81ad8e0 con 0x55733dfe0dc0 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a81ad210 con 0x55733dfe04a0 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7b03a2740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a81aa470 con 0x55733dfeb680 2024-05-20T20:07:30.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7b03a2740 10 client.0 _ll_register_callbacks cb 0x7ff794040070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:30.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7a6a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe04a0 0x55733dff3c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7a6a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe04a0 0x55733dff3c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42452/0 (socket says 172.21.15.28:42452) 2024-05-20T20:07:30.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7a6a00640 1 -- 172.21.15.28:0/3603811603 learned_addr learned my addr 172.21.15.28:0/3603811603 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:30.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7a6000640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfeeff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.823+0000 7ff7a7400640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfef600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff7a6000640 1 -- 172.21.15.28:0/3603811603 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 msgr2=0x55733dfef600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:30.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff7a6000640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfef600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff7a6000640 1 -- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe04a0 msgr2=0x55733dff3c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:30.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff7a6000640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe04a0 0x55733dff3c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff7a6000640 1 -- 172.21.15.28:0/3603811603 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55733dfefe00 con 0x55733dfe0dc0 2024-05-20T20:07:30.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe04a0 0x55733dff3c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:30.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff7a6000640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfeeff0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7ff79403cce0 tx=0x7ff794005e40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a93886e5d761299b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:30.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.824+0000 7ff787e00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:30.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.825+0000 7ff787e00640 1 -- 172.21.15.28:0/3603811603 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff794061030 con 0x55733dfe0dc0 2024-05-20T20:07:30.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.825+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff7a0016c00 con 0x55733dfe0dc0 2024-05-20T20:07:30.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.825+0000 7ff787e00640 1 -- 172.21.15.28:0/3603811603 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff794031ac0 con 0x55733dfe0dc0 2024-05-20T20:07:30.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.826+0000 7ff787e00640 1 -- 172.21.15.28:0/3603811603 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff79406b050 con 0x55733dfe0dc0 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.826+0000 7ff7b03a2740 10 client.15639 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.826+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55733e065850 con 0x55733dfe0dc0 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.826+0000 7ff787e00640 1 -- 172.21.15.28:0/3603811603 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff7a0016c00 con 0x55733dfe0dc0 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.826+0000 7ff787e00640 1 -- 172.21.15.28:0/3603811603 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff79403c8f0 con 0x55733dfe0dc0 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.827+0000 7ff7b03a2740 10 client.15639 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.827+0000 7ff7b03a2740 10 client.15639 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.827+0000 7ff7b03a2740 20 client.15639 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.827+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733e065fc0 0x55733e0683e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.827+0000 7ff7b03a2740 4 client.15639 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:07:30.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.827+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55733df442c0 con 0x55733e065fc0 2024-05-20T20:07:30.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.827+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733e065fc0 0x55733e0683e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:30.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.828+0000 7ff7a6a00640 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733e065fc0 0x55733e0683e0 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4e31ed962703c62a server_cookie=e0ceac1b3dd34989 in_seq=0 out_seq=0 2024-05-20T20:07:30.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.828+0000 7ff787e00640 10 client.15639 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff787e00640 1 -- 172.21.15.28:0/3603811603 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55733df442c0 con 0x55733e065fc0 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff787e00640 10 client.15639 handle_command_reply: tid=0 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 client.15639 shutdown 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 msgr2=0x55733dfeeff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfeeff0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7ff79403cce0 tx=0x7ff794005e40 comp rx=0 tx=0).stop 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 shutdown_connections 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55733e065fc0 0x55733e0683e0 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55733dfeb680 0x55733dfef600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55733dfe0dc0 0x55733dfeeff0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 --2- 172.21.15.28:0/3603811603 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55733dfe04a0 0x55733dff3c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 >> 172.21.15.28:0/3603811603 conn(0x55733dfdc9b0 msgr2=0x55733dfd3310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:30.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff787e00640 0 client.15639 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:30.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.832+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 shutdown_connections 2024-05-20T20:07:30.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.833+0000 7ff7b03a2740 1 -- 172.21.15.28:0/3603811603 wait complete. 2024-05-20T20:07:30.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.833+0000 7ff7b03a2740 20 client.15639 trim_cache size 0 max 16384 2024-05-20T20:07:30.835 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:07:30.835 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:07:30.835 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:07:30.835 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff78c0b2b50 msgr2=0x7ff78c0b5010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff78c0b2b50 0x7ff78c0b5010 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7ff7a000a510 tx=0x7ff7a0009650 comp rx=0 tx=0).stop 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8104ea0 msgr2=0x7ff7a819b590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8104ea0 0x7ff7a819b590 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7ff79800b3f0 tx=0x7ff79800b8b0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 shutdown_connections 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff7a819fe00 0x7ff7a81a02b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff78c0b2b50 0x7ff78c0b5010 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7a8109f70 0x7ff7a819bad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 --2- 172.21.15.28:0/3217104522 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff7a8104ea0 0x7ff7a819b590 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 >> 172.21.15.28:0/3217104522 conn(0x7ff7a8100740 msgr2=0x7ff7a8101b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 shutdown_connections 2024-05-20T20:07:30.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:30.834+0000 7ff7ad600640 1 -- 172.21.15.28:0/3217104522 wait complete. 2024-05-20T20:07:31.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:30 smithi028 ceph-mon[31178]: pgmap v59: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:30 smithi154 ceph-mon[36314]: pgmap v59: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:30 smithi033 ceph-mon[36179]: pgmap v59: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:31.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 -- 172.21.15.28:0/3782295587 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93501095b0 msgr2=0x7f9350109990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 --2- 172.21.15.28:0/3782295587 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93501095b0 0x7f9350109990 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f9340009990 tx=0x7f9340009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 -- 172.21.15.28:0/3782295587 shutdown_connections 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 --2- 172.21.15.28:0/3782295587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935006bbe0 0x7f93501103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 --2- 172.21.15.28:0/3782295587 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f93501095b0 0x7f9350109990 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 --2- 172.21.15.28:0/3782295587 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f935006b220 0x7f935006b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 -- 172.21.15.28:0/3782295587 >> 172.21.15.28:0/3782295587 conn(0x7f93500793f0 msgr2=0x7f9350079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 -- 172.21.15.28:0/3782295587 shutdown_connections 2024-05-20T20:07:31.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.579+0000 7f9357000700 1 -- 172.21.15.28:0/3782295587 wait complete. 2024-05-20T20:07:31.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 Processor -- start 2024-05-20T20:07:31.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 -- start start 2024-05-20T20:07:31.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935006b220 0x7f93501a0250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f935006bbe0 0x7f93501a0790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93501095b0 0x7f93501a4b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9350111530 con 0x7f935006bbe0 2024-05-20T20:07:31.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f93501113b0 con 0x7f93501095b0 2024-05-20T20:07:31.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.581+0000 7f9357000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f93501116b0 con 0x7f935006b220 2024-05-20T20:07:31.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f935006bbe0 0x7f93501a0790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f935006bbe0 0x7f93501a0790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38148/0 (socket says 172.21.15.28:38148) 2024-05-20T20:07:31.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934f400700 1 -- 172.21.15.28:0/302315632 learned_addr learned my addr 172.21.15.28:0/302315632 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:31.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935006b220 0x7f93501a0250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f9355000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93501095b0 0x7f93501a4b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934fe00700 1 -- 172.21.15.28:0/302315632 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93501095b0 msgr2=0x7f93501a4b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934fe00700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93501095b0 0x7f93501a4b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934fe00700 1 -- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f935006bbe0 msgr2=0x7f93501a0790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934fe00700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f935006bbe0 0x7f93501a0790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f934fe00700 1 -- 172.21.15.28:0/302315632 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f93501a5260 con 0x7f935006b220 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.582+0000 7f9355000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93501095b0 0x7f93501a4b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.583+0000 7f934fe00700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935006b220 0x7f93501a0250 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f9340006010 tx=0x7f93400057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.583+0000 7f934cc00700 1 -- 172.21.15.28:0/302315632 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9340027070 con 0x7f935006b220 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.583+0000 7f934cc00700 1 -- 172.21.15.28:0/302315632 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9340022470 con 0x7f935006b220 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.583+0000 7f934cc00700 1 -- 172.21.15.28:0/302315632 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9340017650 con 0x7f935006b220 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.583+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f93501a54f0 con 0x7f935006b220 2024-05-20T20:07:31.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.583+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f93501acdd0 con 0x7f935006b220 2024-05-20T20:07:31.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.587+0000 7f934cc00700 1 -- 172.21.15.28:0/302315632 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f9340009d10 con 0x7f935006b220 2024-05-20T20:07:31.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.590+0000 7f93618cab80 1 Processor -- start 2024-05-20T20:07:31.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.590+0000 7f93618cab80 1 -- start start 2024-05-20T20:07:31.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.591+0000 7f934cc00700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93340b2a70 0x7f93340b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.591+0000 7f93618cab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x56014090e9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.591+0000 7f93618cab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x560140908b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.591+0000 7f93618cab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560140909140 0x5601409095c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.591+0000 7f93618cab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560140866e40 con 0x560140909140 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.591+0000 7f93618cab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5601408569e0 con 0x56014090e610 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.591+0000 7f93618cab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56014085aa20 con 0x56014090ef30 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934f400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x560140908b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934f400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x560140908b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52082/0 (socket says 172.21.15.28:52082) 2024-05-20T20:07:31.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934f400700 1 -- 172.21.15.28:0/4067756601 learned_addr learned my addr 172.21.15.28:0/4067756601 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:31.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934fe00700 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x56014090e9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934f400700 1 -- 172.21.15.28:0/4067756601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 msgr2=0x56014090e9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:31.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934cc00700 1 -- 172.21.15.28:0/302315632 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f93400e2ed0 con 0x7f935006b220 2024-05-20T20:07:31.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934f400700 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x56014090e9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.592+0000 7f934f400700 1 -- 172.21.15.28:0/4067756601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560140909140 msgr2=0x5601409095c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:07:31.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.593+0000 7f9355000700 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560140909140 0x5601409095c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.593+0000 7f9355000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93340b2a70 0x7f93340b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.593+0000 7f934f400700 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560140909140 0x5601409095c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.593+0000 7f934f400700 1 -- 172.21.15.28:0/4067756601 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56014081f420 con 0x56014090ef30 2024-05-20T20:07:31.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.593+0000 7f9355000700 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560140909140 0x5601409095c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:31.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.593+0000 7f934fe00700 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x56014090e9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:31.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.593+0000 7f934f400700 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x560140908b20 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f933c00fd20 tx=0x7f933c009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7c747ad8148a13fd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:31.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.594+0000 7f9345600700 1 -- 172.21.15.28:0/4067756601 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f933c0183f0 con 0x56014090ef30 2024-05-20T20:07:31.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.594+0000 7f9345600700 1 -- 172.21.15.28:0/4067756601 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f933c01d070 con 0x56014090ef30 2024-05-20T20:07:31.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.594+0000 7f9345600700 1 -- 172.21.15.28:0/4067756601 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f933c0176c0 con 0x56014090ef30 2024-05-20T20:07:31.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f93618cab80 1 -- 172.21.15.28:0/4067756601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 msgr2=0x560140908b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f93618cab80 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x560140908b20 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f933c00fd20 tx=0x7f933c009260 comp rx=0 tx=0).stop 2024-05-20T20:07:31.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f9355000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93340b2a70 0x7f93340b4f30 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x560140909b00 tx=0x7f934800b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:31.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f93618cab80 1 -- 172.21.15.28:0/4067756601 shutdown_connections 2024-05-20T20:07:31.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f93618cab80 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560140909140 0x5601409095c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f93618cab80 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x56014090e9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f93618cab80 1 --2- 172.21.15.28:0/4067756601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x560140908b20 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.595+0000 7f93618cab80 1 -- 172.21.15.28:0/4067756601 >> 172.21.15.28:0/4067756601 conn(0x560140836be0 msgr2=0x560140903690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:31.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.596+0000 7f93618cab80 1 -- 172.21.15.28:0/4067756601 shutdown_connections 2024-05-20T20:07:31.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.596+0000 7f93618cab80 1 -- 172.21.15.28:0/4067756601 wait complete. 2024-05-20T20:07:31.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.596+0000 7f93618cab80 1 -- start start 2024-05-20T20:07:31.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.597+0000 7f93618cab80 1 Processor -- start 2024-05-20T20:07:31.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.597+0000 7f93618cab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140914cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.597+0000 7f93618cab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090e610 0x560140912050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.597+0000 7f93618cab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140912590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f93618cab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56014075cfa0 con 0x56014090ef30 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f93618cab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5601407f7760 con 0x560140909140 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f93618cab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5601409159e0 con 0x56014090e610 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f93618cab80 10 client.0 _ll_register_callbacks cb 0x560140800c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f934fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140914cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f9355000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140912590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f934fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140914cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42484/0 (socket says 172.21.15.28:42484) 2024-05-20T20:07:31.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f934fe00700 1 -- 172.21.15.28:0/3909152455 learned_addr learned my addr 172.21.15.28:0/3909152455 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:31.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f934fe00700 1 -- 172.21.15.28:0/3909152455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090e610 msgr2=0x560140912050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:31.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.598+0000 7f934fe00700 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090e610 0x560140912050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f934fe00700 1 -- 172.21.15.28:0/3909152455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 msgr2=0x560140912590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:31.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f934fe00700 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140912590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f934fe00700 1 -- 172.21.15.28:0/3909152455 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5601407a68d0 con 0x560140909140 2024-05-20T20:07:31.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f9355000700 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140912590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:31.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f934fe00700 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140914cb0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f93400e0630 tx=0x7f9340014d20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=574cdc6f1105e53b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:31.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f933a000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:31.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f933a000700 1 -- 172.21.15.28:0/3909152455 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f93400a9d00 con 0x560140909140 2024-05-20T20:07:31.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f933a000700 1 -- 172.21.15.28:0/3909152455 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f93400e0850 con 0x560140909140 2024-05-20T20:07:31.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.599+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f933c01d070 con 0x560140909140 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.600+0000 7f933a000700 1 -- 172.21.15.28:0/3909152455 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f93400e0a50 con 0x560140909140 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.600+0000 7f93618cab80 10 client.25790 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.600+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560140827060 con 0x560140909140 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.600+0000 7f933a000700 1 -- 172.21.15.28:0/3909152455 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f933c01d070 con 0x560140909140 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.600+0000 7f933a000700 1 -- 172.21.15.28:0/3909152455 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f93400aa450 con 0x560140909140 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.600+0000 7f93618cab80 10 client.25790 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.600+0000 7f93618cab80 10 client.25790 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.601+0000 7f93618cab80 20 client.25790 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:31.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.601+0000 7f93618cab80 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560140922290 0x560140924730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.601+0000 7f93618cab80 4 client.25790 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:31.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.601+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560140868d90 con 0x560140922290 2024-05-20T20:07:31.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.601+0000 7f934f400700 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560140922290 0x560140924730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.602+0000 7f934f400700 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560140922290 0x560140924730 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=11de3a5e621b8ecc server_cookie=aee26be86aafb6e in_seq=0 out_seq=0 2024-05-20T20:07:31.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.602+0000 7f933a000700 10 client.25790 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:07:31.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.602+0000 7f933a000700 1 -- 172.21.15.28:0/3909152455 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x560140868d90 con 0x560140922290 2024-05-20T20:07:31.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.602+0000 7f933a000700 10 client.25790 handle_command_reply: tid=0 2024-05-20T20:07:31.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.602+0000 7f93618cab80 1 client.25790 shutdown 2024-05-20T20:07:31.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 msgr2=0x560140914cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140914cb0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f93400e0630 tx=0x7f9340014d20 comp rx=0 tx=0).stop 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 shutdown_connections 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560140922290 0x560140924730 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140912590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140914cb0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f933a000700 0 client.25790 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 --2- 172.21.15.28:0/3909152455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090e610 0x560140912050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 >> 172.21.15.28:0/3909152455 conn(0x560140836be0 msgr2=0x560140902d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 shutdown_connections 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.603+0000 7f93618cab80 1 -- 172.21.15.28:0/3909152455 wait complete. 2024-05-20T20:07:31.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.604+0000 7f93618cab80 20 client.25790 trim_cache size 0 max 16384 2024-05-20T20:07:31.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 Processor -- start 2024-05-20T20:07:31.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 -- start start 2024-05-20T20:07:31.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140912540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090e610 0x5601409034f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x56014092a000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f93501b4270 con 0x56014090e610 2024-05-20T20:07:31.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f93501b5db0 con 0x560140909140 2024-05-20T20:07:31.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.620+0000 7f93618cab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f93501b4940 con 0x56014090ef30 2024-05-20T20:07:31.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090e610 0x5601409034f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140912540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090e610 0x5601409034f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38168/0 (socket says 172.21.15.28:38168) 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934f400700 1 -- 172.21.15.28:0/1020156626 learned_addr learned my addr 172.21.15.28:0/1020156626 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f9355000700 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x56014092a000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934fe00700 1 -- 172.21.15.28:0/1020156626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 msgr2=0x56014092a000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934fe00700 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x56014092a000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934fe00700 1 -- 172.21.15.28:0/1020156626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090e610 msgr2=0x5601409034f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934fe00700 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090e610 0x5601409034f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934fe00700 1 -- 172.21.15.28:0/1020156626 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5601409044a0 con 0x560140909140 2024-05-20T20:07:31.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f9355000700 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x56014092a000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:31.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.621+0000 7f934f400700 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090e610 0x5601409034f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:31.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.622+0000 7f934fe00700 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140912540 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f93400ce0b0 tx=0x7f9340022ce0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=345b8b8bfadfb379 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:31.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.622+0000 7f9344c00700 1 -- 172.21.15.28:0/1020156626 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f93400b2fc0 con 0x560140909140 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.622+0000 7f9344c00700 1 -- 172.21.15.28:0/1020156626 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f93400ae5a0 con 0x560140909140 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.622+0000 7f9344c00700 1 -- 172.21.15.28:0/1020156626 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f93400e0850 con 0x560140909140 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 -- 172.21.15.28:0/1020156626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 msgr2=0x560140912540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140912540 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f93400ce0b0 tx=0x7f9340022ce0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 -- 172.21.15.28:0/1020156626 shutdown_connections 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56014090ef30 0x56014092a000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560140909140 0x560140912540 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 --2- 172.21.15.28:0/1020156626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090e610 0x5601409034f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 -- 172.21.15.28:0/1020156626 >> 172.21.15.28:0/1020156626 conn(0x560140836be0 msgr2=0x560140911e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 -- 172.21.15.28:0/1020156626 shutdown_connections 2024-05-20T20:07:31.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.623+0000 7f93618cab80 1 -- 172.21.15.28:0/1020156626 wait complete. 2024-05-20T20:07:31.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.624+0000 7f93618cab80 1 -- start start 2024-05-20T20:07:31.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.624+0000 7f93618cab80 1 Processor -- start 2024-05-20T20:07:31.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f93618cab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560140909140 0x56014091b450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f93618cab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x560140929900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f93618cab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140929e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f93618cab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f93501b56e0 con 0x56014090ef30 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f93618cab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f93501b5010 con 0x56014090e610 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f93618cab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f93501b2210 con 0x560140909140 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f93618cab80 10 client.0 _ll_register_callbacks cb 0x560140800c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f934fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560140909140 0x56014091b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f934f400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x560140929900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f934f400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x560140929900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42512/0 (socket says 172.21.15.28:42512) 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f934f400700 1 -- 172.21.15.28:0/73158648 learned_addr learned my addr 172.21.15.28:0/73158648 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:31.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.625+0000 7f9355000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140929e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.626+0000 7f9355000700 1 -- 172.21.15.28:0/73158648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560140909140 msgr2=0x56014091b450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:31.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.626+0000 7f9355000700 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560140909140 0x56014091b450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.626+0000 7f9355000700 1 -- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 msgr2=0x560140929900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:31.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.626+0000 7f9355000700 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x560140929900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.626+0000 7f9355000700 1 -- 172.21.15.28:0/73158648 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56014092a6c0 con 0x56014090ef30 2024-05-20T20:07:31.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.626+0000 7f934f400700 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x560140929900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:31.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.627+0000 7f9355000700 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140929e80 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f9348009fd0 tx=0x7f9348004b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e153878ae084c6cc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:31.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.627+0000 7f933a000700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:31.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.627+0000 7f933a000700 1 -- 172.21.15.28:0/73158648 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9348003a00 con 0x56014090ef30 2024-05-20T20:07:31.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.627+0000 7f933a000700 1 -- 172.21.15.28:0/73158648 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f9348020040 con 0x56014090ef30 2024-05-20T20:07:31.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.627+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f93400ae5a0 con 0x56014090ef30 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.628+0000 7f933a000700 1 -- 172.21.15.28:0/73158648 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f9348030050 con 0x56014090ef30 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.628+0000 7f93618cab80 10 client.15654 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.628+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56014098cf50 con 0x56014090ef30 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.628+0000 7f933a000700 1 -- 172.21.15.28:0/73158648 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f93400ae5a0 con 0x56014090ef30 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f933a000700 1 -- 172.21.15.28:0/73158648 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9348023ce0 con 0x56014090ef30 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f93618cab80 10 client.15654 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f93618cab80 10 client.15654 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:07:31.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f93618cab80 20 client.15654 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:31.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f93618cab80 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x56014098d680 0x56014098faa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:31.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f93618cab80 4 client.15654 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:31.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x560140862570 con 0x56014098d680 2024-05-20T20:07:31.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.629+0000 7f934fe00700 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x56014098d680 0x56014098faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:31.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f934fe00700 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x56014098d680 0x56014098faa0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=591fec74993b5931 server_cookie=60b2783147197c98 in_seq=0 out_seq=0 2024-05-20T20:07:31.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f933a000700 10 client.15654 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:07:31.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f933a000700 1 -- 172.21.15.28:0/73158648 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x560140862570 con 0x56014098d680 2024-05-20T20:07:31.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f933a000700 10 client.15654 handle_command_reply: tid=0 2024-05-20T20:07:31.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 client.15654 shutdown 2024-05-20T20:07:31.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 msgr2=0x560140929e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140929e80 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f9348009fd0 tx=0x7f9348004b40 comp rx=0 tx=0).stop 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 shutdown_connections 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x56014098d680 0x56014098faa0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56014090ef30 0x560140929e80 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560140909140 0x56014091b450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 --2- 172.21.15.28:0/73158648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56014090e610 0x560140929900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 >> 172.21.15.28:0/73158648 conn(0x560140836be0 msgr2=0x56014090f310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.630+0000 7f933a000700 0 client.15654 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.631+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 shutdown_connections 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.631+0000 7f93618cab80 1 -- 172.21.15.28:0/73158648 wait complete. 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.631+0000 7f93618cab80 20 client.15654 trim_cache size 0 max 16384 2024-05-20T20:07:31.633 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:31.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93340b2a70 msgr2=0x7f93340b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93340b2a70 0x7f93340b4f30 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x560140909b00 tx=0x7f934800b040 comp rx=0 tx=0).stop 2024-05-20T20:07:31.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935006b220 msgr2=0x7f93501a0250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:31.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935006b220 0x7f93501a0250 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f9340006010 tx=0x7f93400057c0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 shutdown_connections 2024-05-20T20:07:31.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f93340b2a70 0x7f93340b4f30 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f93501095b0 0x7f93501a4b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f935006b220 0x7f93501a0250 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 --2- 172.21.15.28:0/302315632 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f935006bbe0 0x7f93501a0790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:31.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.632+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 >> 172.21.15.28:0/302315632 conn(0x7f93500793f0 msgr2=0x7f9350101210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:31.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.633+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 shutdown_connections 2024-05-20T20:07:31.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:31.633+0000 7f9357000700 1 -- 172.21.15.28:0/302315632 wait complete. 2024-05-20T20:07:32.691 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:07:33.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:32 smithi028 ceph-mon[31178]: pgmap v60: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-20T20:07:33.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:32 smithi154 ceph-mon[36314]: pgmap v60: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-20T20:07:33.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:32 smithi033 ceph-mon[36179]: pgmap v60: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 0 B/s wr, 8 op/s 2024-05-20T20:07:33.208 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:34.949 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:34 smithi028 ceph-mon[31178]: pgmap v61: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:34 smithi154 ceph-mon[36314]: pgmap v61: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:35.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:34 smithi033 ceph-mon[36179]: pgmap v61: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 -- 172.21.15.28:0/3284481160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa09406b220 msgr2=0x7fa09406b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 --2- 172.21.15.28:0/3284481160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa09406b220 0x7fa09406b6a0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fa080009990 tx=0x7fa080009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 -- 172.21.15.28:0/3284481160 shutdown_connections 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 --2- 172.21.15.28:0/3284481160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa09406bbe0 0x7fa0941103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 --2- 172.21.15.28:0/3284481160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa0941095b0 0x7fa094109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 --2- 172.21.15.28:0/3284481160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa09406b220 0x7fa09406b6a0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 -- 172.21.15.28:0/3284481160 >> 172.21.15.28:0/3284481160 conn(0x7fa0940793f0 msgr2=0x7fa094079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:35.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 -- 172.21.15.28:0/3284481160 shutdown_connections 2024-05-20T20:07:35.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.603+0000 7fa09b600700 1 -- 172.21.15.28:0/3284481160 wait complete. 2024-05-20T20:07:35.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.605+0000 7fa09b600700 1 Processor -- start 2024-05-20T20:07:35.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.605+0000 7fa09b600700 1 -- start start 2024-05-20T20:07:35.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.606+0000 7fa09b600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa09406bbe0 0x7fa09410faa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.606+0000 7fa09b600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa0941095b0 0x7fa09410ffe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.606+0000 7fa09b600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa0941a4e00 0x7fa0941a5280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.606+0000 7fa09b600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa094113760 con 0x7fa0941095b0 2024-05-20T20:07:35.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.606+0000 7fa09b600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa0941135e0 con 0x7fa0941a4e00 2024-05-20T20:07:35.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.606+0000 7fa09b600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa0941138e0 con 0x7fa09406bbe0 2024-05-20T20:07:35.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.606+0000 7fa099600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa0941a4e00 0x7fa0941a5280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa098c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa09406bbe0 0x7fa09410faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa093e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa0941095b0 0x7fa09410ffe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa099600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa0941a4e00 0x7fa0941a5280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42522/0 (socket says 172.21.15.28:42522) 2024-05-20T20:07:35.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa099600700 1 -- 172.21.15.28:0/3985899786 learned_addr learned my addr 172.21.15.28:0/3985899786 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:35.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa093e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa0941095b0 0x7fa09410ffe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38176/0 (socket says 172.21.15.28:38176) 2024-05-20T20:07:35.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa099600700 1 -- 172.21.15.28:0/3985899786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa09406bbe0 msgr2=0x7fa09410faa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa099600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa09406bbe0 0x7fa09410faa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa099600700 1 -- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa0941095b0 msgr2=0x7fa09410ffe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa099600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa0941095b0 0x7fa09410ffe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.607+0000 7fa099600700 1 -- 172.21.15.28:0/3985899786 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa0941a58f0 con 0x7fa0941a4e00 2024-05-20T20:07:35.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.608+0000 7fa099600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa0941a4e00 0x7fa0941a5280 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fa08c00ef30 tx=0x7fa08c00c450 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:35.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.608+0000 7fa091600700 1 -- 172.21.15.28:0/3985899786 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa08c0092e0 con 0x7fa0941a4e00 2024-05-20T20:07:35.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.608+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa0941a5b80 con 0x7fa0941a4e00 2024-05-20T20:07:35.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.608+0000 7fa091600700 1 -- 172.21.15.28:0/3985899786 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa08c004bb0 con 0x7fa0941a4e00 2024-05-20T20:07:35.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.608+0000 7fa091600700 1 -- 172.21.15.28:0/3985899786 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa08c004280 con 0x7fa0941a4e00 2024-05-20T20:07:35.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.608+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa0941ad2d0 con 0x7fa0941a4e00 2024-05-20T20:07:35.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.614+0000 7fa091600700 1 -- 172.21.15.28:0/3985899786 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa08c01b030 con 0x7fa0941a4e00 2024-05-20T20:07:35.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.616+0000 7fa091600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa07c0b2aa0 0x7fa07c0b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.617+0000 7fa091600700 1 -- 172.21.15.28:0/3985899786 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa08c0e1ff0 con 0x7fa0941a4e00 2024-05-20T20:07:35.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.617+0000 7fa0a5f31b80 1 Processor -- start 2024-05-20T20:07:35.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.617+0000 7fa0a5f31b80 1 -- start start 2024-05-20T20:07:35.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.617+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7003290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b6ffd260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b6ffd880 0x5616b6ffdce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5616b6f5be40 con 0x5616b7002eb0 2024-05-20T20:07:35.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5616b6f4b9e0 con 0x5616b70037d0 2024-05-20T20:07:35.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5616b6f4fa20 con 0x5616b6ffd880 2024-05-20T20:07:35.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b6ffd880 0x5616b6ffdce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa098c00700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa07c0b2aa0 0x7fa07c0b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa093e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b6ffd260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b6ffd880 0x5616b6ffdce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52128/0 (socket says 172.21.15.28:52128) 2024-05-20T20:07:35.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 -- 172.21.15.28:0/3265251185 learned_addr learned my addr 172.21.15.28:0/3265251185 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:35.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 -- 172.21.15.28:0/3265251185 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 msgr2=0x5616b6ffd260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:35.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 --2- 172.21.15.28:0/3265251185 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b6ffd260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 -- 172.21.15.28:0/3265251185 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 msgr2=0x5616b7003290 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:07:35.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 --2- 172.21.15.28:0/3265251185 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7003290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.618+0000 7fa099600700 1 -- 172.21.15.28:0/3265251185 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5616b6f14420 con 0x5616b6ffd880 2024-05-20T20:07:35.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.619+0000 7fa099600700 1 --2- 172.21.15.28:0/3265251185 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b6ffd880 0x5616b6ffdce0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fa08c004b80 tx=0x7fa08c0a8e20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=33ae8e9fc371d290 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:35.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.620+0000 7fa086000700 1 -- 172.21.15.28:0/3265251185 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa08c0a9310 con 0x5616b6ffd880 2024-05-20T20:07:35.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.620+0000 7fa098c00700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa07c0b2aa0 0x7fa07c0b4f60 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fa088009680 tx=0x7fa088009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:35.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.620+0000 7fa086000700 1 -- 172.21.15.28:0/3265251185 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa08c0ad6c0 con 0x5616b6ffd880 2024-05-20T20:07:35.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.620+0000 7fa086000700 1 -- 172.21.15.28:0/3265251185 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa08c0ad160 con 0x5616b6ffd880 2024-05-20T20:07:35.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.620+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3265251185 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b6ffd880 msgr2=0x5616b6ffdce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.620+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3265251185 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b6ffd880 0x5616b6ffdce0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fa08c004b80 tx=0x7fa08c0a8e20 comp rx=0 tx=0).stop 2024-05-20T20:07:35.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3265251185 shutdown_connections 2024-05-20T20:07:35.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3265251185 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b6ffd880 0x5616b6ffdce0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3265251185 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7003290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3265251185 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b6ffd260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3265251185 >> 172.21.15.28:0/3265251185 conn(0x5616b6f2bbe0 msgr2=0x5616b6ff8010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3265251185 shutdown_connections 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3265251185 wait complete. 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.621+0000 7fa0a5f31b80 1 -- start start 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.622+0000 7fa0a5f31b80 1 Processor -- start 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.622+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b70095b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.622+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7006950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.622+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b7006e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.622+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5616b6e51fa0 con 0x5616b7002eb0 2024-05-20T20:07:35.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5616b6eec760 con 0x5616b6ffd880 2024-05-20T20:07:35.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5616b700a240 con 0x5616b70037d0 2024-05-20T20:07:35.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa0a5f31b80 10 client.0 _ll_register_callbacks cb 0x5616b6ef5c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:35.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa098c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b70095b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa098c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b70095b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42540/0 (socket says 172.21.15.28:42540) 2024-05-20T20:07:35.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa098c00700 1 -- 172.21.15.28:0/3435874833 learned_addr learned my addr 172.21.15.28:0/3435874833 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:35.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa099600700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b7006e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.623+0000 7fa093e00700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7006950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa098c00700 1 -- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 msgr2=0x5616b7006e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa098c00700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b7006e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa098c00700 1 -- 172.21.15.28:0/3435874833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 msgr2=0x5616b7006950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa098c00700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7006950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa098c00700 1 -- 172.21.15.28:0/3435874833 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5616b6e9b8d0 con 0x5616b6ffd880 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa099600700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b7006e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa093e00700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7006950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:35.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa098c00700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b70095b0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fa0880094b0 tx=0x7fa0880047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7572bd05a9e634ce server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.624+0000 7fa07aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.625+0000 7fa07aa00700 1 -- 172.21.15.28:0/3435874833 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa08801e830 con 0x5616b6ffd880 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.625+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa08c0ad6c0 con 0x5616b6ffd880 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.625+0000 7fa07aa00700 1 -- 172.21.15.28:0/3435874833 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa08801e9d0 con 0x5616b6ffd880 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.625+0000 7fa07aa00700 1 -- 172.21.15.28:0/3435874833 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa08801ec10 con 0x5616b6ffd880 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.625+0000 7fa0a5f31b80 10 client.25808 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.625+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5616b6f1c060 con 0x5616b6ffd880 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.626+0000 7fa07aa00700 1 -- 172.21.15.28:0/3435874833 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa08c0ad6c0 con 0x5616b6ffd880 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa07aa00700 1 -- 172.21.15.28:0/3435874833 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa0880263f0 con 0x5616b6ffd880 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa0a5f31b80 10 client.25808 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa0a5f31b80 10 client.25808 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:07:35.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa0a5f31b80 20 client.25808 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:35.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7028e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa0a5f31b80 4 client.25808 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:35.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5616b6f5dd90 con 0x5616b70269f0 2024-05-20T20:07:35.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.627+0000 7fa093e00700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7028e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.629+0000 7fa093e00700 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7028e60 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=15e723451bdcd910 server_cookie=62212723549de604 in_seq=0 out_seq=0 2024-05-20T20:07:35.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.629+0000 7fa07aa00700 10 client.25808 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:35.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa07aa00700 1 -- 172.21.15.28:0/3435874833 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5616b6f5dd90 con 0x5616b70269f0 2024-05-20T20:07:35.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa07aa00700 10 client.25808 handle_command_reply: tid=0 2024-05-20T20:07:35.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa0a5f31b80 1 client.25808 shutdown 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 msgr2=0x5616b70095b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b70095b0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fa0880094b0 tx=0x7fa0880047c0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 shutdown_connections 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b7006e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b70095b0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.631+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7028e60 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.632+0000 7fa07aa00700 0 client.25808 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:35.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.632+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/3435874833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b7006950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.632+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 >> 172.21.15.28:0/3435874833 conn(0x5616b6f2bbe0 msgr2=0x5616b6ffc250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:35.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.632+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 shutdown_connections 2024-05-20T20:07:35.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.633+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/3435874833 wait complete. 2024-05-20T20:07:35.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.633+0000 7fa0a5f31b80 20 client.25808 trim_cache size 0 max 16384 2024-05-20T20:07:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 Processor -- start 2024-05-20T20:07:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 -- start start 2024-05-20T20:07:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b6ffd260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b70010d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b6ff79e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa094074610 con 0x5616b7002eb0 2024-05-20T20:07:35.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa094199620 con 0x5616b6ffd880 2024-05-20T20:07:35.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.649+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa094074ce0 con 0x5616b70037d0 2024-05-20T20:07:35.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.650+0000 7fa093e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b70010d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.650+0000 7fa093e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b70010d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38200/0 (socket says 172.21.15.28:38200) 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.650+0000 7fa093e00700 1 -- 172.21.15.28:0/2407513496 learned_addr learned my addr 172.21.15.28:0/2407513496 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.650+0000 7fa099600700 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b6ff79e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.650+0000 7fa098c00700 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b6ffd260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.651+0000 7fa093e00700 1 -- 172.21.15.28:0/2407513496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 msgr2=0x5616b6ff79e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.651+0000 7fa093e00700 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b6ff79e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.651+0000 7fa093e00700 1 -- 172.21.15.28:0/2407513496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 msgr2=0x5616b6ffd260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.651+0000 7fa093e00700 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b6ffd260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.651+0000 7fa093e00700 1 -- 172.21.15.28:0/2407513496 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5616b7002b00 con 0x5616b7002eb0 2024-05-20T20:07:35.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.652+0000 7fa093e00700 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b70010d0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fa080009990 tx=0x7fa080005260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a309794379cd4c70 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:35.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.652+0000 7fa085600700 1 -- 172.21.15.28:0/2407513496 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa080005350 con 0x5616b7002eb0 2024-05-20T20:07:35.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.652+0000 7fa085600700 1 -- 172.21.15.28:0/2407513496 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa0800054f0 con 0x5616b7002eb0 2024-05-20T20:07:35.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.652+0000 7fa085600700 1 -- 172.21.15.28:0/2407513496 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa080003680 con 0x5616b7002eb0 2024-05-20T20:07:35.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.653+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2407513496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 msgr2=0x5616b70010d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.653+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b70010d0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fa080009990 tx=0x7fa080005260 comp rx=0 tx=0).stop 2024-05-20T20:07:35.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.653+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2407513496 shutdown_connections 2024-05-20T20:07:35.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.653+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b70037d0 0x5616b6ff79e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.653+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b6ffd880 0x5616b6ffd260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.653+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2407513496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b7002eb0 0x5616b70010d0 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.653+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2407513496 >> 172.21.15.28:0/2407513496 conn(0x5616b6f2bbe0 msgr2=0x5616b70009c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:35.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.654+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2407513496 shutdown_connections 2024-05-20T20:07:35.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.654+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2407513496 wait complete. 2024-05-20T20:07:35.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.654+0000 7fa0a5f31b80 1 -- start start 2024-05-20T20:07:35.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 1 Processor -- start 2024-05-20T20:07:35.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b6ffd880 0x5616b7020cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b7002eb0 0x5616b702de80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b702e3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa094198f50 con 0x5616b6ffd880 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa0940753b0 con 0x5616b70037d0 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa094072550 con 0x5616b7002eb0 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.655+0000 7fa0a5f31b80 10 client.0 _ll_register_callbacks cb 0x5616b6ef5c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.656+0000 7fa098c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b6ffd880 0x5616b7020cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.656+0000 7fa093e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b7002eb0 0x5616b702de80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.656+0000 7fa093e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b7002eb0 0x5616b702de80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52144/0 (socket says 172.21.15.28:52144) 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.656+0000 7fa093e00700 1 -- 172.21.15.28:0/2696157844 learned_addr learned my addr 172.21.15.28:0/2696157844 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.656+0000 7fa099600700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b702e3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa098c00700 1 -- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b7002eb0 msgr2=0x5616b702de80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:35.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa098c00700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b7002eb0 0x5616b702de80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa098c00700 1 -- 172.21.15.28:0/2696157844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 msgr2=0x5616b702e3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:35.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa098c00700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b702e3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa098c00700 1 -- 172.21.15.28:0/2696157844 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5616b702ec10 con 0x5616b6ffd880 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa099600700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b702e3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa093e00700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b7002eb0 0x5616b702de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa098c00700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b6ffd880 0x5616b7020cf0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fa088026a00 tx=0x7fa088026620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9017a4d4c03103eb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.657+0000 7fa07aa00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.658+0000 7fa07aa00700 1 -- 172.21.15.28:0/2696157844 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa088035040 con 0x5616b6ffd880 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.658+0000 7fa07aa00700 1 -- 172.21.15.28:0/2696157844 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa088026b40 con 0x5616b6ffd880 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.658+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa0800054f0 con 0x5616b6ffd880 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.658+0000 7fa07aa00700 1 -- 172.21.15.28:0/2696157844 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa088046050 con 0x5616b6ffd880 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.658+0000 7fa0a5f31b80 10 client.15669 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:35.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.658+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5616b7092c00 con 0x5616b6ffd880 2024-05-20T20:07:35.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa07aa00700 1 -- 172.21.15.28:0/2696157844 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa0800054f0 con 0x5616b6ffd880 2024-05-20T20:07:35.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa07aa00700 1 -- 172.21.15.28:0/2696157844 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa088041030 con 0x5616b6ffd880 2024-05-20T20:07:35.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa0a5f31b80 10 client.15669 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:35.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa0a5f31b80 10 client.15669 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:07:35.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa0a5f31b80 20 client.15669 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:35.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7095390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:35.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa0a5f31b80 4 client.15669 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:35.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.659+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5616b6f57570 con 0x5616b70269f0 2024-05-20T20:07:35.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.661+0000 7fa093e00700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7095390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:35.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.661+0000 7fa093e00700 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7095390 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=be0d707ad07444b1 server_cookie=a5b8e0e5828c7f8 in_seq=0 out_seq=0 2024-05-20T20:07:35.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.661+0000 7fa07aa00700 10 client.15669 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.661+0000 7fa07aa00700 1 -- 172.21.15.28:0/2696157844 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5616b6f57570 con 0x5616b70269f0 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.661+0000 7fa07aa00700 10 client.15669 handle_command_reply: tid=0 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.661+0000 7fa0a5f31b80 1 client.15669 shutdown 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b6ffd880 msgr2=0x5616b7020cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b6ffd880 0x5616b7020cf0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fa088026a00 tx=0x7fa088026620 comp rx=0 tx=0).stop 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 shutdown_connections 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616b70037d0 0x5616b702e3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x5616b70269f0 0x5616b7095390 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616b6ffd880 0x5616b7020cf0 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 --2- 172.21.15.28:0/2696157844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616b7002eb0 0x5616b702de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 >> 172.21.15.28:0/2696157844 conn(0x5616b6f2bbe0 msgr2=0x5616b6ff79e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:35.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa07aa00700 0 client.15669 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:07:35.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.662+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 shutdown_connections 2024-05-20T20:07:35.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.663+0000 7fa0a5f31b80 1 -- 172.21.15.28:0/2696157844 wait complete. 2024-05-20T20:07:35.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.663+0000 7fa0a5f31b80 20 client.15669 trim_cache size 0 max 16384 2024-05-20T20:07:35.665 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa07c0b2aa0 msgr2=0x7fa07c0b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa07c0b2aa0 0x7fa07c0b4f60 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fa088009680 tx=0x7fa088009260 comp rx=0 tx=0).stop 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa0941a4e00 msgr2=0x7fa0941a5280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa0941a4e00 0x7fa0941a5280 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fa08c00ef30 tx=0x7fa08c00c450 comp rx=0 tx=0).stop 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 shutdown_connections 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa07c0b2aa0 0x7fa07c0b4f60 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa0941a4e00 0x7fa0941a5280 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa09406bbe0 0x7fa09410faa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 --2- 172.21.15.28:0/3985899786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa0941095b0 0x7fa09410ffe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:35.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.664+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 >> 172.21.15.28:0/3985899786 conn(0x7fa0940793f0 msgr2=0x7fa0941011e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:35.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.666+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 shutdown_connections 2024-05-20T20:07:35.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:35.666+0000 7fa09b600700 1 -- 172.21.15.28:0/3985899786 wait complete. 2024-05-20T20:07:36.563 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:07:37.064 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:36 smithi154 ceph-mon[36314]: pgmap v62: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:37.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:36 smithi033 ceph-mon[36179]: pgmap v62: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:37.158 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:36 smithi028 ceph-mon[31178]: pgmap v62: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:39.092 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:38 smithi028 ceph-mon[31178]: pgmap v63: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:38 smithi154 ceph-mon[36314]: pgmap v63: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:39.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:38 smithi033 ceph-mon[36179]: pgmap v63: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:39.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 -- 172.21.15.28:0/3068102658 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 msgr2=0x7ff830102800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 --2- 172.21.15.28:0/3068102658 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 0x7ff830102800 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7ff8200098f0 tx=0x7ff820009c00 comp rx=0 tx=0).stop 2024-05-20T20:07:39.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 -- 172.21.15.28:0/3068102658 shutdown_connections 2024-05-20T20:07:39.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 --2- 172.21.15.28:0/3068102658 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff830107830 0x7ff830107ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 --2- 172.21.15.28:0/3068102658 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 0x7ff830102800 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 --2- 172.21.15.28:0/3068102658 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff830102d40 0x7ff830107210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 -- 172.21.15.28:0/3068102658 >> 172.21.15.28:0/3068102658 conn(0x7ff8300fde60 msgr2=0x7ff830100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:39.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 -- 172.21.15.28:0/3068102658 shutdown_connections 2024-05-20T20:07:39.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.385+0000 7ff82fe00700 1 -- 172.21.15.28:0/3068102658 wait complete. 2024-05-20T20:07:39.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.386+0000 7ff82fe00700 1 Processor -- start 2024-05-20T20:07:39.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.387+0000 7ff82fe00700 1 -- start start 2024-05-20T20:07:39.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.387+0000 7ff82fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 0x7ff830074d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.387+0000 7ff82fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff830102d40 0x7ff83006de50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.387+0000 7ff82fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff830107830 0x7ff83006e390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.387+0000 7ff82fe00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff830119c70 con 0x7ff830102420 2024-05-20T20:07:39.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.387+0000 7ff82fe00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff830119af0 con 0x7ff830107830 2024-05-20T20:07:39.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.387+0000 7ff82fe00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff830119df0 con 0x7ff830102d40 2024-05-20T20:07:39.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.388+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 0x7ff830074d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.388+0000 7ff82f400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff830107830 0x7ff83006e390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.388+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 0x7ff830074d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53696/0 (socket says 172.21.15.28:53696) 2024-05-20T20:07:39.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.388+0000 7ff82ea00700 1 -- 172.21.15.28:0/2392016106 learned_addr learned my addr 172.21.15.28:0/2392016106 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:39.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.388+0000 7ff82f400700 1 -- 172.21.15.28:0/2392016106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff830102d40 msgr2=0x7ff83006de50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.388+0000 7ff82f400700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff830102d40 0x7ff83006de50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.389+0000 7ff82f400700 1 -- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 msgr2=0x7ff830074d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.389+0000 7ff82f400700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 0x7ff830074d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.389+0000 7ff82f400700 1 -- 172.21.15.28:0/2392016106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff83006e9d0 con 0x7ff830107830 2024-05-20T20:07:39.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.389+0000 7ff82f400700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff830107830 0x7ff83006e390 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7ff82800efc0 tx=0x7ff82800c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:39.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.389+0000 7ff827400700 1 -- 172.21.15.28:0/2392016106 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8280092e0 con 0x7ff830107830 2024-05-20T20:07:39.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.389+0000 7ff827400700 1 -- 172.21.15.28:0/2392016106 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff828004bb0 con 0x7ff830107830 2024-05-20T20:07:39.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.389+0000 7ff827400700 1 -- 172.21.15.28:0/2392016106 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff828004060 con 0x7ff830107830 2024-05-20T20:07:39.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.390+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff83006ec60 con 0x7ff830107830 2024-05-20T20:07:39.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.390+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff830072b90 con 0x7ff830107830 2024-05-20T20:07:39.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.395+0000 7ff827400700 1 -- 172.21.15.28:0/2392016106 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ff82801b030 con 0x7ff830107830 2024-05-20T20:07:39.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.396+0000 7ff8402c9b80 1 Processor -- start 2024-05-20T20:07:39.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.397+0000 7ff8402c9b80 1 -- start start 2024-05-20T20:07:39.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.397+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747c7c0 0x55ea0747cba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.397+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea07476d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.397+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea074777f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.397+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ea073d4e40 con 0x55ea0747c7c0 2024-05-20T20:07:39.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.397+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ea073c49e0 con 0x55ea07477390 2024-05-20T20:07:39.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.397+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ea073c8a20 con 0x55ea0747d0e0 2024-05-20T20:07:39.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.398+0000 7ff827400700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8140b29f0 0x7ff8140b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.398+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea07476d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.398+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea07476d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56256/0 (socket says 172.21.15.28:56256) 2024-05-20T20:07:39.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.398+0000 7ff82ea00700 1 -- 172.21.15.28:0/3657604034 learned_addr learned my addr 172.21.15.28:0/3657604034 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:39.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.398+0000 7ff82e000700 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747c7c0 0x55ea0747cba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.398+0000 7ff827400700 1 -- 172.21.15.28:0/2392016106 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff8280e2050 con 0x7ff830107830 2024-05-20T20:07:39.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.399+0000 7ff82f400700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8140b29f0 0x7ff8140b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.399+0000 7ff82e000700 1 -- 172.21.15.28:0/3657604034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 msgr2=0x55ea07476d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:39.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.399+0000 7ff82e000700 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea07476d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.399+0000 7ff82e000700 1 -- 172.21.15.28:0/3657604034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 msgr2=0x55ea074777f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:07:39.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.399+0000 7ff82e000700 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea074777f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.399+0000 7ff82e000700 1 -- 172.21.15.28:0/3657604034 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ea0738d420 con 0x55ea0747c7c0 2024-05-20T20:07:39.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.400+0000 7ff82e000700 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747c7c0 0x55ea0747cba0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7ff83006f650 tx=0x7ff81c009ed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=39796989be1d6eb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:39.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.400+0000 7ff82f400700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8140b29f0 0x7ff8140b4eb0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7ff8280dfdb0 tx=0x7ff828009040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:39.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.400+0000 7ff81be00700 1 -- 172.21.15.28:0/3657604034 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff81c004230 con 0x55ea0747c7c0 2024-05-20T20:07:39.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.400+0000 7ff81be00700 1 -- 172.21.15.28:0/3657604034 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff81c00ed40 con 0x55ea0747c7c0 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.400+0000 7ff81be00700 1 -- 172.21.15.28:0/3657604034 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff81c005020 con 0x55ea0747c7c0 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.401+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3657604034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747c7c0 msgr2=0x55ea0747cba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.401+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747c7c0 0x55ea0747cba0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7ff83006f650 tx=0x7ff81c009ed0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.402+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3657604034 shutdown_connections 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.402+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea074777f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.402+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747c7c0 0x55ea0747cba0 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.402+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3657604034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea07476d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.402+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3657604034 >> 172.21.15.28:0/3657604034 conn(0x55ea073a4be0 msgr2=0x55ea07471810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:39.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.402+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3657604034 shutdown_connections 2024-05-20T20:07:39.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.402+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3657604034 wait complete. 2024-05-20T20:07:39.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.403+0000 7ff8402c9b80 1 -- start start 2024-05-20T20:07:39.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.403+0000 7ff8402c9b80 1 Processor -- start 2024-05-20T20:07:39.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07482df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea07480190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea074806d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ea072cafa0 con 0x55ea0747d0e0 2024-05-20T20:07:39.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ea07365760 con 0x55ea07477390 2024-05-20T20:07:39.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ea07483c00 con 0x55ea0747c7c0 2024-05-20T20:07:39.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff8402c9b80 10 client.0 _ll_register_callbacks cb 0x55ea0736ec70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:39.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.404+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea07480190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82e000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea074806d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea07480190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56264/0 (socket says 172.21.15.28:56264) 2024-05-20T20:07:39.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82ea00700 1 -- 172.21.15.28:0/3328603902 learned_addr learned my addr 172.21.15.28:0/3328603902 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:39.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82ea00700 1 -- 172.21.15.28:0/3328603902 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 msgr2=0x55ea07482df0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:07:39.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82ea00700 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07482df0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82ea00700 1 -- 172.21.15.28:0/3328603902 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 msgr2=0x55ea074806d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:39.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82ea00700 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea074806d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82ea00700 1 -- 172.21.15.28:0/3328603902 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ea073148d0 con 0x55ea0747c7c0 2024-05-20T20:07:39.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.405+0000 7ff82e000700 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea074806d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.406+0000 7ff82ea00700 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea07480190 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7ff820009230 tx=0x7ff82000f690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=22f95b07b5ada6a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.406+0000 7ff818c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.406+0000 7ff818c00700 1 -- 172.21.15.28:0/3328603902 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff820005240 con 0x55ea0747c7c0 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.406+0000 7ff818c00700 1 -- 172.21.15.28:0/3328603902 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff8200053e0 con 0x55ea0747c7c0 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.407+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff81c00ed40 con 0x55ea0747c7c0 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.407+0000 7ff818c00700 1 -- 172.21.15.28:0/3328603902 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff820030050 con 0x55ea0747c7c0 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.407+0000 7ff8402c9b80 10 client.25777 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:39.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.407+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55ea07395060 con 0x55ea0747c7c0 2024-05-20T20:07:39.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.407+0000 7ff818c00700 1 -- 172.21.15.28:0/3328603902 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff81c00ed40 con 0x55ea0747c7c0 2024-05-20T20:07:39.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.408+0000 7ff818c00700 1 -- 172.21.15.28:0/3328603902 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff82002b030 con 0x55ea0747c7c0 2024-05-20T20:07:39.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.408+0000 7ff8402c9b80 10 client.25777 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:39.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.408+0000 7ff8402c9b80 10 client.25777 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:07:39.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.408+0000 7ff8402c9b80 20 client.25777 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:39.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.408+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea07490760 0x55ea07492c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.408+0000 7ff8402c9b80 4 client.25777 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:39.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.408+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ea073d6d90 con 0x55ea07490760 2024-05-20T20:07:39.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.409+0000 7ff82e000700 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea07490760 0x55ea07492c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.409+0000 7ff82e000700 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea07490760 0x55ea07492c00 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e90b4bb125033558 server_cookie=4ef2ab677fdcd9ff in_seq=0 out_seq=0 2024-05-20T20:07:39.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.409+0000 7ff818c00700 10 client.25777 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.410+0000 7ff818c00700 1 -- 172.21.15.28:0/3328603902 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55ea073d6d90 con 0x55ea07490760 2024-05-20T20:07:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.410+0000 7ff818c00700 10 client.25777 handle_command_reply: tid=0 2024-05-20T20:07:39.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.410+0000 7ff8402c9b80 1 client.25777 shutdown 2024-05-20T20:07:39.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 msgr2=0x55ea07480190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea07480190 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7ff820009230 tx=0x7ff82000f690 comp rx=0 tx=0).stop 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 shutdown_connections 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea07490760 0x55ea07492c00 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea074806d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07482df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/3328603902 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea07480190 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 >> 172.21.15.28:0/3328603902 conn(0x55ea073a4be0 msgr2=0x55ea07470fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff818c00700 0 client.25777 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 shutdown_connections 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 1 -- 172.21.15.28:0/3328603902 wait complete. 2024-05-20T20:07:39.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.411+0000 7ff8402c9b80 20 client.25777 trim_cache size 0 max 16384 2024-05-20T20:07:39.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.443+0000 7ff8402c9b80 1 Processor -- start 2024-05-20T20:07:39.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.443+0000 7ff8402c9b80 1 -- start start 2024-05-20T20:07:39.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07476d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea0747dd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea07471740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff8301bcb50 con 0x55ea0747d0e0 2024-05-20T20:07:39.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff8301be690 con 0x55ea07477390 2024-05-20T20:07:39.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff8301bd220 con 0x55ea0747c7c0 2024-05-20T20:07:39.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea0747dd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff82ea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea0747dd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56280/0 (socket says 172.21.15.28:56280) 2024-05-20T20:07:39.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.444+0000 7ff82ea00700 1 -- 172.21.15.28:0/2832840527 learned_addr learned my addr 172.21.15.28:0/2832840527 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:39.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07476d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea07471740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 -- 172.21.15.28:0/2832840527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 msgr2=0x55ea0747dd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:39.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea0747dd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 -- 172.21.15.28:0/2832840527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 msgr2=0x55ea07471740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:39.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea07471740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 -- 172.21.15.28:0/2832840527 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ea07483ae0 con 0x55ea07477390 2024-05-20T20:07:39.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.445+0000 7ff82e000700 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07476d70 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x55ea07493440 tx=0x7ff81c004b50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9e9a82fa69b27a24 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:39.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.446+0000 7ff81b400700 1 -- 172.21.15.28:0/2832840527 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff81c004050 con 0x55ea07477390 2024-05-20T20:07:39.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.446+0000 7ff81b400700 1 -- 172.21.15.28:0/2832840527 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff81c00e910 con 0x55ea07477390 2024-05-20T20:07:39.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.446+0000 7ff81b400700 1 -- 172.21.15.28:0/2832840527 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff81c003980 con 0x55ea07477390 2024-05-20T20:07:39.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 -- 172.21.15.28:0/2832840527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 msgr2=0x55ea07476d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07476d70 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x55ea07493440 tx=0x7ff81c004b50 comp rx=0 tx=0).stop 2024-05-20T20:07:39.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 -- 172.21.15.28:0/2832840527 shutdown_connections 2024-05-20T20:07:39.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea0747d0e0 0x55ea07471740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea07477390 0x55ea07476d70 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/2832840527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747c7c0 0x55ea0747dd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 -- 172.21.15.28:0/2832840527 >> 172.21.15.28:0/2832840527 conn(0x55ea073a4be0 msgr2=0x55ea074805e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:39.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.447+0000 7ff8402c9b80 1 -- 172.21.15.28:0/2832840527 shutdown_connections 2024-05-20T20:07:39.448 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.448+0000 7ff8402c9b80 1 -- 172.21.15.28:0/2832840527 wait complete. 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.448+0000 7ff8402c9b80 1 -- start start 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 1 Processor -- start 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea07477390 0x55ea07489070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea0747c7c0 0x55ea074895b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea074978e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff8301bdfc0 con 0x55ea07477390 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff8301bd8f0 con 0x55ea0747c7c0 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff8301baaf0 con 0x55ea0747d0e0 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.449+0000 7ff8402c9b80 10 client.0 _ll_register_callbacks cb 0x55ea0736ec70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea07477390 0x55ea07489070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea07477390 0x55ea07489070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53732/0 (socket says 172.21.15.28:53732) 2024-05-20T20:07:39.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 -- 172.21.15.28:0/1186502355 learned_addr learned my addr 172.21.15.28:0/1186502355 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:39.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea074978e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82ea00700 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea0747c7c0 0x55ea074895b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 -- 172.21.15.28:0/1186502355 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 msgr2=0x55ea074978e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:39.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea074978e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 -- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea0747c7c0 msgr2=0x55ea074895b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:39.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.450+0000 7ff82e000700 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea0747c7c0 0x55ea074895b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.451+0000 7ff82e000700 1 -- 172.21.15.28:0/1186502355 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ea07498000 con 0x55ea07477390 2024-05-20T20:07:39.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.451+0000 7ff82e000700 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea07477390 0x55ea07489070 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7ff81c009f10 tx=0x7ff81c009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c62bbc91aa265252 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:39.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.451+0000 7ff818c00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:39.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.452+0000 7ff818c00700 1 -- 172.21.15.28:0/1186502355 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff81c004530 con 0x55ea07477390 2024-05-20T20:07:39.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.452+0000 7ff818c00700 1 -- 172.21.15.28:0/1186502355 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff81c003680 con 0x55ea07477390 2024-05-20T20:07:39.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.452+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff81c00e910 con 0x55ea07477390 2024-05-20T20:07:39.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.452+0000 7ff818c00700 1 -- 172.21.15.28:0/1186502355 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff81c003880 con 0x55ea07477390 2024-05-20T20:07:39.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.452+0000 7ff8402c9b80 10 client.15687 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:39.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.452+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55ea074fad60 con 0x55ea07477390 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff818c00700 1 -- 172.21.15.28:0/1186502355 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff81c00e910 con 0x55ea07477390 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff818c00700 1 -- 172.21.15.28:0/1186502355 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff81c03d050 con 0x55ea07477390 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff8402c9b80 10 client.15687 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff8402c9b80 10 client.15687 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff8402c9b80 20 client.15687 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea074fb1a0 0x55ea074fd5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff8402c9b80 4 client.15687 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:39.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.453+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55ea073d0570 con 0x55ea074fb1a0 2024-05-20T20:07:39.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.454+0000 7ff82ea00700 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea074fb1a0 0x55ea074fd5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:39.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.454+0000 7ff82ea00700 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea074fb1a0 0x55ea074fd5c0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=194632fd3a62d15b server_cookie=4f048b35aa8df59d in_seq=0 out_seq=0 2024-05-20T20:07:39.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.454+0000 7ff818c00700 10 client.15687 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff818c00700 1 -- 172.21.15.28:0/1186502355 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55ea073d0570 con 0x55ea074fb1a0 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff818c00700 10 client.15687 handle_command_reply: tid=0 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 client.15687 shutdown 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea07477390 msgr2=0x55ea07489070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea07477390 0x55ea07489070 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7ff81c009f10 tx=0x7ff81c009260 comp rx=0 tx=0).stop 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 shutdown_connections 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55ea074fb1a0 0x55ea074fd5c0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ea0747d0e0 0x55ea074978e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ea07477390 0x55ea07489070 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 --2- 172.21.15.28:0/1186502355 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ea0747c7c0 0x55ea074895b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 >> 172.21.15.28:0/1186502355 conn(0x55ea073a4be0 msgr2=0x55ea0747b320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:39.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff818c00700 0 client.15687 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:07:39.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.455+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 shutdown_connections 2024-05-20T20:07:39.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.456+0000 7ff8402c9b80 1 -- 172.21.15.28:0/1186502355 wait complete. 2024-05-20T20:07:39.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.456+0000 7ff8402c9b80 20 client.15687 trim_cache size 0 max 16384 2024-05-20T20:07:39.458 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:39.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8140b29f0 msgr2=0x7ff8140b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8140b29f0 0x7ff8140b4eb0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7ff8280dfdb0 tx=0x7ff828009040 comp rx=0 tx=0).stop 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff830107830 msgr2=0x7ff83006e390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff830107830 0x7ff83006e390 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7ff82800efc0 tx=0x7ff82800c3a0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 shutdown_connections 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8140b29f0 0x7ff8140b4eb0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff830107830 0x7ff83006e390 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff830102420 0x7ff830074d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 --2- 172.21.15.28:0/2392016106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff830102d40 0x7ff83006de50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.457+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 >> 172.21.15.28:0/2392016106 conn(0x7ff8300fde60 msgr2=0x7ff8301051e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.458+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 shutdown_connections 2024-05-20T20:07:39.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:39.458+0000 7ff82fe00700 1 -- 172.21.15.28:0/2392016106 wait complete. 2024-05-20T20:07:40.214 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:07:40.684 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:41.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:40 smithi154 ceph-mon[36314]: pgmap v64: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:40 smithi033 ceph-mon[36179]: pgmap v64: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:41.186 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:40 smithi028 ceph-mon[31178]: pgmap v64: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:41.503 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:07:42.224 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:42.879 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:42 smithi028 ceph-mon[31178]: pgmap v65: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 341 B/s wr, 8 op/s 2024-05-20T20:07:43.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 -- 172.21.15.28:0/67792943 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006b220 msgr2=0x7fac5006b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 --2- 172.21.15.28:0/67792943 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006b220 0x7fac5006b6a0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fac3c009990 tx=0x7fac3c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 -- 172.21.15.28:0/67792943 shutdown_connections 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 --2- 172.21.15.28:0/67792943 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac5006bbe0 0x7fac501103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 --2- 172.21.15.28:0/67792943 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac501095b0 0x7fac50109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 --2- 172.21.15.28:0/67792943 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006b220 0x7fac5006b6a0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 -- 172.21.15.28:0/67792943 >> 172.21.15.28:0/67792943 conn(0x7fac500793f0 msgr2=0x7fac50079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 -- 172.21.15.28:0/67792943 shutdown_connections 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.057+0000 7fac58200700 1 -- 172.21.15.28:0/67792943 wait complete. 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.059+0000 7fac58200700 1 Processor -- start 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.059+0000 7fac58200700 1 -- start start 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.060+0000 7fac58200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac5006b220 0x7fac5019be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.060+0000 7fac58200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006bbe0 0x7fac5019c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.060+0000 7fac58200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac501095b0 0x7fac501a06f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.060+0000 7fac58200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fac50113720 con 0x7fac5006bbe0 2024-05-20T20:07:43.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.060+0000 7fac58200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fac501135a0 con 0x7fac501095b0 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.060+0000 7fac58200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fac501138a0 con 0x7fac5006b220 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac5006b220 0x7fac5019be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac5006b220 0x7fac5019be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56306/0 (socket says 172.21.15.28:56306) 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 -- 172.21.15.28:0/4256336871 learned_addr learned my addr 172.21.15.28:0/4256336871 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac56200700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac501095b0 0x7fac501a06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac54e00700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006bbe0 0x7fac5019c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 -- 172.21.15.28:0/4256336871 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac501095b0 msgr2=0x7fac501a06f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac501095b0 0x7fac501a06f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 -- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006bbe0 msgr2=0x7fac5019c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006bbe0 0x7fac5019c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac55800700 1 -- 172.21.15.28:0/4256336871 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fac501a0e30 con 0x7fac5006b220 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.061+0000 7fac54e00700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006bbe0 0x7fac5019c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.062+0000 7fac55800700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac5006b220 0x7fac5019be20 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7fac4400e9a0 tx=0x7fac4400ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:43.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.062+0000 7fac4e000700 1 -- 172.21.15.28:0/4256336871 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac44024070 con 0x7fac5006b220 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.062+0000 7fac4e000700 1 -- 172.21.15.28:0/4256336871 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fac4400cc70 con 0x7fac5006b220 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.062+0000 7fac4e000700 1 -- 172.21.15.28:0/4256336871 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac4401fc60 con 0x7fac5006b220 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.062+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fac501a1120 con 0x7fac5006b220 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.063+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fac5010a310 con 0x7fac5006b220 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.068+0000 7fac4e000700 1 -- 172.21.15.28:0/4256336871 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fac44027070 con 0x7fac5006b220 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.069+0000 7fac62a94b80 1 Processor -- start 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.069+0000 7fac62a94b80 1 -- start start 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.070+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fd8290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.070+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd87d0 0x558e03fd2260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.070+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.070+0000 7fac62a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558e03f30e40 con 0x558e03fd7eb0 2024-05-20T20:07:43.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.070+0000 7fac62a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558e03f209e0 con 0x558e03fd87d0 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.070+0000 7fac62a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558e03f24a20 con 0x558e03fd2880 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.071+0000 7fac56200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fd8290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.071+0000 7fac55800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd87d0 0x558e03fd2260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.071+0000 7fac54e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.071+0000 7fac56200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fd8290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53770/0 (socket says 172.21.15.28:53770) 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.071+0000 7fac56200700 1 -- 172.21.15.28:0/929452470 learned_addr learned my addr 172.21.15.28:0/929452470 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.072+0000 7fac56200700 1 -- 172.21.15.28:0/929452470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 msgr2=0x558e03fd2ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.072+0000 7fac56200700 1 --2- 172.21.15.28:0/929452470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.072+0000 7fac56200700 1 -- 172.21.15.28:0/929452470 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd87d0 msgr2=0x558e03fd2260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.072+0000 7fac56200700 1 --2- 172.21.15.28:0/929452470 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd87d0 0x558e03fd2260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.072+0000 7fac56200700 1 -- 172.21.15.28:0/929452470 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558e03ee9420 con 0x558e03fd7eb0 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.072+0000 7fac4e000700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac380b2a70 0x7fac380b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.072+0000 7fac4e000700 1 -- 172.21.15.28:0/4256336871 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fac440ea870 con 0x7fac5006b220 2024-05-20T20:07:43.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.073+0000 7fac56200700 1 --2- 172.21.15.28:0/929452470 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fd8290 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fac4800a730 tx=0x7fac480078d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4343af429440fc51 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.073+0000 7fac55800700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac380b2a70 0x7fac380b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.074+0000 7fac42a00700 1 -- 172.21.15.28:0/929452470 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac48003a00 con 0x558e03fd7eb0 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.074+0000 7fac55800700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac380b2a70 0x7fac380b4f30 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fac440e8190 tx=0x7fac4401f6c0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.074+0000 7fac42a00700 1 -- 172.21.15.28:0/929452470 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fac48003ba0 con 0x558e03fd7eb0 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.074+0000 7fac42a00700 1 -- 172.21.15.28:0/929452470 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac48003a00 con 0x558e03fd7eb0 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.076+0000 7fac62a94b80 1 -- 172.21.15.28:0/929452470 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 msgr2=0x558e03fd8290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.076+0000 7fac62a94b80 1 --2- 172.21.15.28:0/929452470 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fd8290 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fac4800a730 tx=0x7fac480078d0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.076+0000 7fac62a94b80 1 -- 172.21.15.28:0/929452470 shutdown_connections 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.076+0000 7fac62a94b80 1 --2- 172.21.15.28:0/929452470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.076+0000 7fac62a94b80 1 --2- 172.21.15.28:0/929452470 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fd8290 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.076+0000 7fac62a94b80 1 --2- 172.21.15.28:0/929452470 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd87d0 0x558e03fd2260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.076+0000 7fac62a94b80 1 -- 172.21.15.28:0/929452470 >> 172.21.15.28:0/929452470 conn(0x558e03f00be0 msgr2=0x558e03fcd010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.077+0000 7fac62a94b80 1 -- 172.21.15.28:0/929452470 shutdown_connections 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.077+0000 7fac62a94b80 1 -- 172.21.15.28:0/929452470 wait complete. 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.078+0000 7fac62a94b80 1 -- start start 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.078+0000 7fac62a94b80 1 Processor -- start 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.079+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd2880 0x558e03fde410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.079+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fdb7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.079+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd87d0 0x558e03fdbcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.079+0000 7fac62a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558e03e26fa0 con 0x558e03fd7eb0 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.079+0000 7fac62a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558e03ec1760 con 0x558e03fd2880 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.079+0000 7fac62a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558e03fdf230 con 0x558e03fd87d0 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.079+0000 7fac62a94b80 10 client.0 _ll_register_callbacks cb 0x558e03ecac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.080+0000 7fac56200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd2880 0x558e03fde410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.080+0000 7fac54e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fdb7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.080+0000 7fac56200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd2880 0x558e03fde410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46802/0 (socket says 172.21.15.28:46802) 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.080+0000 7fac56200700 1 -- 172.21.15.28:0/1080756019 learned_addr learned my addr 172.21.15.28:0/1080756019 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac56200700 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd87d0 0x558e03fdbcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac56200700 1 -- 172.21.15.28:0/1080756019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd87d0 msgr2=0x558e03fdbcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac56200700 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd87d0 0x558e03fdbcf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac56200700 1 -- 172.21.15.28:0/1080756019 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 msgr2=0x558e03fdb7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac56200700 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fdb7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac56200700 1 -- 172.21.15.28:0/1080756019 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558e03e708d0 con 0x558e03fd2880 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac56200700 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd2880 0x558e03fde410 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fac48007700 tx=0x7fac48004780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5741a48e502fdccb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac37400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.081+0000 7fac37400700 1 -- 172.21.15.28:0/1080756019 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac48003a00 con 0x558e03fd2880 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.082+0000 7fac37400700 1 -- 172.21.15.28:0/1080756019 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fac48019070 con 0x558e03fd2880 2024-05-20T20:07:43.106 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.082+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fac48003ba0 con 0x558e03fd2880 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.082+0000 7fac37400700 1 -- 172.21.15.28:0/1080756019 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fac48004510 con 0x558e03fd2880 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.082+0000 7fac62a94b80 10 client.25832 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.082+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558e03ef1060 con 0x558e03fd2880 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac37400700 1 -- 172.21.15.28:0/1080756019 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fac48003ba0 con 0x558e03fd2880 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac37400700 1 -- 172.21.15.28:0/1080756019 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fac48020970 con 0x558e03fd2880 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac62a94b80 10 client.25832 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac62a94b80 10 client.25832 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac62a94b80 20 client.25832 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e03feb970 0x558e03fedd60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac62a94b80 4 client.25832 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.083+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558e03f32d90 con 0x558e03feb970 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.084+0000 7fac54e00700 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e03feb970 0x558e03fedd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.085+0000 7fac54e00700 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e03feb970 0x558e03fedd60 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c0776cb4bf4d49 server_cookie=31fc40db881bc885 in_seq=0 out_seq=0 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.085+0000 7fac37400700 10 client.25832 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac37400700 1 -- 172.21.15.28:0/1080756019 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558e03f32d90 con 0x558e03feb970 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac37400700 10 client.25832 handle_command_reply: tid=0 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 client.25832 shutdown 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd2880 msgr2=0x558e03fde410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd2880 0x558e03fde410 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fac48007700 tx=0x7fac48004780 comp rx=0 tx=0).stop 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 shutdown_connections 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e03feb970 0x558e03fedd60 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd87d0 0x558e03fdbcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd2880 0x558e03fde410 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1080756019 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd7eb0 0x558e03fdb7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 >> 172.21.15.28:0/1080756019 conn(0x558e03f00be0 msgr2=0x558e03fd1250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac37400700 0 client.25832 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.086+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 shutdown_connections 2024-05-20T20:07:43.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.087+0000 7fac62a94b80 1 -- 172.21.15.28:0/1080756019 wait complete. 2024-05-20T20:07:43.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.087+0000 7fac62a94b80 20 client.25832 trim_cache size 0 max 16384 2024-05-20T20:07:43.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 Processor -- start 2024-05-20T20:07:43.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 -- start start 2024-05-20T20:07:43.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03fcd0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fac501abd70 con 0x558e03fd87d0 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fac501ad8b0 con 0x558e03fd7eb0 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac62a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fac501ac440 con 0x558e03fd2880 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.119+0000 7fac56200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac56200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56348/0 (socket says 172.21.15.28:56348) 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac56200700 1 -- 172.21.15.28:0/799383496 learned_addr learned my addr 172.21.15.28:0/799383496 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac56200700 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac54e00700 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03fcd0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac54e00700 1 -- 172.21.15.28:0/799383496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 msgr2=0x558e03fd2260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac54e00700 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac54e00700 1 -- 172.21.15.28:0/799383496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 msgr2=0x558e03ff3300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac54e00700 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac54e00700 1 -- 172.21.15.28:0/799383496 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558e03fd32f0 con 0x558e03fd7eb0 2024-05-20T20:07:43.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.120+0000 7fac56200700 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:43.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.121+0000 7fac54e00700 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03fcd0f0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fac50104540 tx=0x7fac3c01a040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=362cfbc62dd815c0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:43.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.122+0000 7fac42000700 1 -- 172.21.15.28:0/799383496 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac3c00b920 con 0x558e03fd7eb0 2024-05-20T20:07:43.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.122+0000 7fac42000700 1 -- 172.21.15.28:0/799383496 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fac3c003680 con 0x558e03fd7eb0 2024-05-20T20:07:43.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.122+0000 7fac42000700 1 -- 172.21.15.28:0/799383496 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac3c00f6c0 con 0x558e03fd7eb0 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 -- 172.21.15.28:0/799383496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 msgr2=0x558e03fcd0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03fcd0f0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fac50104540 tx=0x7fac3c01a040 comp rx=0 tx=0).stop 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 -- 172.21.15.28:0/799383496 shutdown_connections 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03fd2260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 --2- 172.21.15.28:0/799383496 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03fcd0f0 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 -- 172.21.15.28:0/799383496 >> 172.21.15.28:0/799383496 conn(0x558e03f00be0 msgr2=0x558e03fcc9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:43.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 -- 172.21.15.28:0/799383496 shutdown_connections 2024-05-20T20:07:43.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.123+0000 7fac62a94b80 1 -- 172.21.15.28:0/799383496 wait complete. 2024-05-20T20:07:43.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.124+0000 7fac62a94b80 1 -- start start 2024-05-20T20:07:43.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.124+0000 7fac62a94b80 1 Processor -- start 2024-05-20T20:07:43.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03ff3010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03ff3550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac62a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac62a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fac501ad1e0 con 0x558e03fd87d0 2024-05-20T20:07:43.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac62a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fac501acb10 con 0x558e03fd7eb0 2024-05-20T20:07:43.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac62a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fac501a9d30 con 0x558e03fd2880 2024-05-20T20:07:43.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac62a94b80 10 client.0 _ll_register_callbacks cb 0x558e03fdc1b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:43.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:42 smithi154 ceph-mon[36314]: pgmap v65: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 341 B/s wr, 8 op/s 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac56200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03ff3010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac56200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03ff3010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56356/0 (socket says 172.21.15.28:56356) 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac56200700 1 -- 172.21.15.28:0/1017901985 learned_addr learned my addr 172.21.15.28:0/1017901985 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.125+0000 7fac56200700 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.126+0000 7fac54e00700 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03ff3550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.126+0000 7fac56200700 1 -- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 msgr2=0x558e03ff3550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.126+0000 7fac56200700 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03ff3550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.126+0000 7fac56200700 1 -- 172.21.15.28:0/1017901985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 msgr2=0x558e03ff3a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:43.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.126+0000 7fac56200700 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.127+0000 7fac56200700 1 -- 172.21.15.28:0/1017901985 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558e040553e0 con 0x558e03fd2880 2024-05-20T20:07:43.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.127+0000 7fac56200700 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03ff3010 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fac48007d90 tx=0x7fac480043b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dec4d4901daed844 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:43.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.127+0000 7fac37400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:07:43.128 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:42 smithi033 ceph-mon[36179]: pgmap v65: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 341 B/s wr, 8 op/s 2024-05-20T20:07:43.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.127+0000 7fac37400700 1 -- 172.21.15.28:0/1017901985 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac4802e080 con 0x558e03fd2880 2024-05-20T20:07:43.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.127+0000 7fac37400700 1 -- 172.21.15.28:0/1017901985 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fac48019040 con 0x558e03fd2880 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.127+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fac3c003680 con 0x558e03fd2880 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.128+0000 7fac37400700 1 -- 172.21.15.28:0/1017901985 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fac48043050 con 0x558e03fd2880 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.128+0000 7fac62a94b80 10 client.25798 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.128+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558e040566c0 con 0x558e03fd2880 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.128+0000 7fac37400700 1 -- 172.21.15.28:0/1017901985 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fac3c003680 con 0x558e03fd2880 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.128+0000 7fac37400700 1 -- 172.21.15.28:0/1017901985 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fac4801fb00 con 0x558e03fd2880 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.129+0000 7fac62a94b80 10 client.25798 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.129+0000 7fac62a94b80 10 client.25798 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.129+0000 7fac62a94b80 20 client.25798 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.129+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e04056cb0 0x558e040590d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.129+0000 7fac62a94b80 4 client.25798 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:07:43.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.129+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x558e03f2c570 con 0x558e04056cb0 2024-05-20T20:07:43.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.129+0000 7fac54e00700 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e04056cb0 0x558e040590d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:43.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.130+0000 7fac54e00700 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e04056cb0 0x558e040590d0 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d9d77062b6cc958f server_cookie=cb4c7c21e7a4a8da in_seq=0 out_seq=0 2024-05-20T20:07:43.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.130+0000 7fac37400700 10 client.25798 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:43.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.130+0000 7fac37400700 1 -- 172.21.15.28:0/1017901985 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x558e03f2c570 con 0x558e04056cb0 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.130+0000 7fac37400700 10 client.25798 handle_command_reply: tid=0 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.130+0000 7fac62a94b80 1 client.25798 shutdown 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 msgr2=0x558e03ff3010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03ff3010 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fac48007d90 tx=0x7fac480043b0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 shutdown_connections 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558e04056cb0 0x558e040590d0 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558e03fd87d0 0x558e03ff3a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558e03fd2880 0x558e03ff3010 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 --2- 172.21.15.28:0/1017901985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558e03fd7eb0 0x558e03ff3550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 >> 172.21.15.28:0/1017901985 conn(0x558e03f00be0 msgr2=0x558e03fd8bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac37400700 0 client.25798 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 shutdown_connections 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 1 -- 172.21.15.28:0/1017901985 wait complete. 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.131+0000 7fac62a94b80 20 client.25798 trim_cache size 0 max 16384 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:07:43.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac380b2a70 msgr2=0x7fac380b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac380b2a70 0x7fac380b4f30 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fac440e8190 tx=0x7fac4401f6c0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac5006b220 msgr2=0x7fac5019be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac5006b220 0x7fac5019be20 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7fac4400e9a0 tx=0x7fac4400ecb0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 shutdown_connections 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac380b2a70 0x7fac380b4f30 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac501095b0 0x7fac501a06f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac5006b220 0x7fac5019be20 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 --2- 172.21.15.28:0/4256336871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac5006bbe0 0x7fac5019c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:43.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.132+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 >> 172.21.15.28:0/4256336871 conn(0x7fac500793f0 msgr2=0x7fac5010e720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:43.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.133+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 shutdown_connections 2024-05-20T20:07:43.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:43.133+0000 7fac58200700 1 -- 172.21.15.28:0/4256336871 wait complete. 2024-05-20T20:07:44.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 -- 172.21.15.28:0/3688463399 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff70410ce30 msgr2=0x7ff70410f2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 --2- 172.21.15.28:0/3688463399 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff70410ce30 0x7ff70410f2d0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc009ca0 tx=0x7ff6fc02a950 comp rx=0 tx=0).stop 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 -- 172.21.15.28:0/3688463399 shutdown_connections 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 --2- 172.21.15.28:0/3688463399 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff704104330 0x7ff70410c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 --2- 172.21.15.28:0/3688463399 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff70410ce30 0x7ff70410f2d0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 --2- 172.21.15.28:0/3688463399 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff704103a10 0x7ff704103df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 -- 172.21.15.28:0/3688463399 >> 172.21.15.28:0/3688463399 conn(0x7ff7040fd270 msgr2=0x7ff7040ff690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.032+0000 7ff709600640 1 -- 172.21.15.28:0/3688463399 shutdown_connections 2024-05-20T20:07:44.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.033+0000 7ff709600640 1 -- 172.21.15.28:0/3688463399 wait complete. 2024-05-20T20:07:44.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.034+0000 7ff709600640 1 Processor -- start 2024-05-20T20:07:44.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.034+0000 7ff709600640 1 -- start start 2024-05-20T20:07:44.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff709600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff704103a10 0x7ff704199070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff709600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff704104330 0x7ff7041995b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff709600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff70410ce30 0x7ff70419d8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff709600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff704112b30 con 0x7ff704103a10 2024-05-20T20:07:44.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff709600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff7041129b0 con 0x7ff704104330 2024-05-20T20:07:44.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff709600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff704112cb0 con 0x7ff70410ce30 2024-05-20T20:07:44.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff703e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff704103a10 0x7ff704199070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff708c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff70410ce30 0x7ff70419d8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff708c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff70410ce30 0x7ff70419d8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56372/0 (socket says 172.21.15.28:56372) 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff708c00640 1 -- 172.21.15.28:0/3590383421 learned_addr learned my addr 172.21.15.28:0/3590383421 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.035+0000 7ff703400640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff704104330 0x7ff7041995b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.036+0000 7ff703e00640 1 -- 172.21.15.28:0/3590383421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff70410ce30 msgr2=0x7ff70419d8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.036+0000 7ff703e00640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff70410ce30 0x7ff70419d8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.036+0000 7ff703e00640 1 -- 172.21.15.28:0/3590383421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff704104330 msgr2=0x7ff7041995b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.036+0000 7ff703e00640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff704104330 0x7ff7041995b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.036+0000 7ff703e00640 1 -- 172.21.15.28:0/3590383421 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff70419e010 con 0x7ff704103a10 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.036+0000 7ff703400640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff704104330 0x7ff7041995b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:44.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.037+0000 7ff703e00640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff704103a10 0x7ff704199070 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7ff6f4009bc0 tx=0x7ff6f400b690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:44.038 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.037+0000 7ff700c00640 1 -- 172.21.15.28:0/3590383421 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6f401e070 con 0x7ff704103a10 2024-05-20T20:07:44.038 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.037+0000 7ff700c00640 1 -- 172.21.15.28:0/3590383421 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6f40198c0 con 0x7ff704103a10 2024-05-20T20:07:44.038 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.037+0000 7ff700c00640 1 -- 172.21.15.28:0/3590383421 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6f40187d0 con 0x7ff704103a10 2024-05-20T20:07:44.038 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.037+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff70419e300 con 0x7ff704103a10 2024-05-20T20:07:44.038 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.037+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff704101590 con 0x7ff704103a10 2024-05-20T20:07:44.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.043+0000 7ff70da09740 1 Processor -- start 2024-05-20T20:07:44.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.043+0000 7ff70da09740 1 -- start start 2024-05-20T20:07:44.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.043+0000 7ff700c00640 1 -- 172.21.15.28:0/3590383421 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ff6f4021070 con 0x7ff704103a10 2024-05-20T20:07:44.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff70da09740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c88230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff70da09740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c88770 0x561f16c92af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.045 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff70da09740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c93030 0x561f16c95420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff70da09740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561f16bf8b90 con 0x561f16c88770 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff70da09740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561f16be8730 con 0x561f16c87e50 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff70da09740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561f16bec770 con 0x561f16c93030 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff708c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c88230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff708c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c88230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46846/0 (socket says 172.21.15.28:46846) 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff708c00640 1 -- 172.21.15.28:0/2629113503 learned_addr learned my addr 172.21.15.28:0/2629113503 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.044+0000 7ff703400640 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c93030 0x561f16c95420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.045+0000 7ff703e00640 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c88770 0x561f16c92af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.045+0000 7ff708c00640 1 -- 172.21.15.28:0/2629113503 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c93030 msgr2=0x561f16c95420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.045+0000 7ff708c00640 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c93030 0x561f16c95420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.045+0000 7ff708c00640 1 -- 172.21.15.28:0/2629113503 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c88770 msgr2=0x561f16c92af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.045+0000 7ff708c00640 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c88770 0x561f16c92af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.045+0000 7ff708c00640 1 -- 172.21.15.28:0/2629113503 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561f16b818b0 con 0x561f16c87e50 2024-05-20T20:07:44.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.045+0000 7ff703e00640 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c88770 0x561f16c92af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:44.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.046+0000 7ff700c00640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff6e80b29d0 0x7ff6e80b4e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.046+0000 7ff708c00640 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c88230 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc02f410 tx=0x7ff6fc02f440 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3d3bb254bf83cdae server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.046+0000 7ff700c00640 1 -- 172.21.15.28:0/3590383421 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff6f40e5d30 con 0x7ff704103a10 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.046+0000 7ff6f9200640 1 -- 172.21.15.28:0/2629113503 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6fc048070 con 0x561f16c87e50 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.046+0000 7ff6f9200640 1 -- 172.21.15.28:0/2629113503 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6fc003e20 con 0x561f16c87e50 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.046+0000 7ff6f9200640 1 -- 172.21.15.28:0/2629113503 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6fc034040 con 0x561f16c87e50 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.046+0000 7ff703e00640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff6e80b29d0 0x7ff6e80b4e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 -- 172.21.15.28:0/2629113503 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 msgr2=0x561f16c88230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c88230 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc02f410 tx=0x7ff6fc02f440 comp rx=0 tx=0).stop 2024-05-20T20:07:44.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff703e00640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff6e80b29d0 0x7ff6e80b4e90 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x561f16c893f0 tx=0x7ff6f40108b0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 -- 172.21.15.28:0/2629113503 shutdown_connections 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c93030 0x561f16c95420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c88770 0x561f16c92af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 --2- 172.21.15.28:0/2629113503 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c88230 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 -- 172.21.15.28:0/2629113503 >> 172.21.15.28:0/2629113503 conn(0x561f16c84360 msgr2=0x561f16c84740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 -- 172.21.15.28:0/2629113503 shutdown_connections 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.047+0000 7ff70da09740 1 -- 172.21.15.28:0/2629113503 wait complete. 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.048+0000 7ff70da09740 1 -- start start 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.048+0000 7ff70da09740 1 Processor -- start 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff70da09740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c92af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff70da09740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16c8fde0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff70da09740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16c90520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff70da09740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561f168ce790 con 0x561f16c93030 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff70da09740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561f16b90bc0 con 0x561f16c87e50 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff70da09740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561f16c95c00 con 0x561f16c88770 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff70da09740 10 client.0 _ll_register_callbacks cb 0x561f16b6e400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.049+0000 7ff708c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c92af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c92af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46852/0 (socket says 172.21.15.28:46852) 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 -- 172.21.15.28:0/670154657 learned_addr learned my addr 172.21.15.28:0/670154657 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16c90520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff703400640 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16c8fde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 -- 172.21.15.28:0/670154657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 msgr2=0x561f16c8fde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16c8fde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 -- 172.21.15.28:0/670154657 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 msgr2=0x561f16c90520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16c90520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 -- 172.21.15.28:0/670154657 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561f168fd210 con 0x561f16c87e50 2024-05-20T20:07:44.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff708c00640 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c92af0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc02fdb0 tx=0x7ff6fc02fcb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8fece5a3d3ffac95 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:44.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.050+0000 7ff6ed600640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:44.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.051+0000 7ff6ed600640 1 -- 172.21.15.28:0/670154657 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6fc048040 con 0x561f16c87e50 2024-05-20T20:07:44.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.051+0000 7ff6ed600640 1 -- 172.21.15.28:0/670154657 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff6fc003c00 con 0x561f16c87e50 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.051+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff6fc003e20 con 0x561f16c87e50 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.051+0000 7ff6ed600640 1 -- 172.21.15.28:0/670154657 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff6fc004040 con 0x561f16c87e50 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.051+0000 7ff70da09740 10 client.25850 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.051+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561f169c9620 con 0x561f16c87e50 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff6ed600640 1 -- 172.21.15.28:0/670154657 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6fc003e20 con 0x561f16c87e50 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff6ed600640 1 -- 172.21.15.28:0/670154657 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff6fc059a40 con 0x561f16c87e50 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff70da09740 10 client.25850 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff70da09740 10 client.25850 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff70da09740 20 client.25850 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff70da09740 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16ca2850 0x561f16ca4c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff70da09740 4 client.25850 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:44.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.052+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561f16bfaae0 con 0x561f16ca2850 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.053+0000 7ff703400640 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16ca2850 0x561f16ca4c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.053+0000 7ff703400640 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16ca2850 0x561f16ca4c40 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1e81642a87a95660 server_cookie=30bd880d1368b00d in_seq=0 out_seq=0 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.053+0000 7ff6ed600640 10 client.25850 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.054+0000 7ff6ed600640 1 -- 172.21.15.28:0/670154657 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561f16bfaae0 con 0x561f16ca2850 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.054+0000 7ff6ed600640 10 client.25850 handle_command_reply: tid=0 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.054+0000 7ff70da09740 1 client.25850 shutdown 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 msgr2=0x561f16c92af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c92af0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc02fdb0 tx=0x7ff6fc02fcb0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 shutdown_connections 2024-05-20T20:07:44.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16ca2850 0x561f16ca4c40 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16c90520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16c8fde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 --2- 172.21.15.28:0/670154657 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16c92af0 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 >> 172.21.15.28:0/670154657 conn(0x561f16c84360 msgr2=0x561f16c8a0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff6ed600640 0 client.25850 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 shutdown_connections 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 1 -- 172.21.15.28:0/670154657 wait complete. 2024-05-20T20:07:44.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.055+0000 7ff70da09740 20 client.25850 trim_cache size 0 max 16384 2024-05-20T20:07:44.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.087+0000 7ff70da09740 1 Processor -- start 2024-05-20T20:07:44.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.087+0000 7ff70da09740 1 -- start start 2024-05-20T20:07:44.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff70da09740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c87e50 0x561f16c83c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff70da09740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c88770 0x561f16c92620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff70da09740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca7880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff70da09740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff7041ac2c0 con 0x561f16c93030 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff70da09740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff7041ade00 con 0x561f16c88770 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff70da09740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff7041ac990 con 0x561f16c87e50 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff708c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c87e50 0x561f16c83c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff703400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c88770 0x561f16c92620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff708c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c87e50 0x561f16c83c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56410/0 (socket says 172.21.15.28:56410) 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff708c00640 1 -- 172.21.15.28:0/1978084687 learned_addr learned my addr 172.21.15.28:0/1978084687 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.088+0000 7ff708c00640 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca7880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff703400640 1 -- 172.21.15.28:0/1978084687 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c87e50 msgr2=0x561f16c83c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff703400640 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c87e50 0x561f16c83c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff703400640 1 -- 172.21.15.28:0/1978084687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 msgr2=0x561f16ca7880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff703400640 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca7880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff703400640 1 -- 172.21.15.28:0/1978084687 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561f16c90890 con 0x561f16c88770 2024-05-20T20:07:44.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff703400640 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c88770 0x561f16c92620 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x561f16c95c30 tx=0x7ff6f0005f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=26163d1963b21e97 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:44.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff6efe00640 1 -- 172.21.15.28:0/1978084687 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6f0024070 con 0x561f16c88770 2024-05-20T20:07:44.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff6efe00640 1 -- 172.21.15.28:0/1978084687 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6f0005020 con 0x561f16c88770 2024-05-20T20:07:44.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.089+0000 7ff6efe00640 1 -- 172.21.15.28:0/1978084687 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6f0004b20 con 0x561f16c88770 2024-05-20T20:07:44.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.090+0000 7ff70da09740 1 -- 172.21.15.28:0/1978084687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c88770 msgr2=0x561f16c92620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.090+0000 7ff70da09740 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c88770 0x561f16c92620 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x561f16c95c30 tx=0x7ff6f0005f90 comp rx=0 tx=0).stop 2024-05-20T20:07:44.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.090+0000 7ff70da09740 1 -- 172.21.15.28:0/1978084687 shutdown_connections 2024-05-20T20:07:44.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.090+0000 7ff70da09740 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c88770 0x561f16c92620 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.090+0000 7ff70da09740 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca7880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.090+0000 7ff70da09740 1 --2- 172.21.15.28:0/1978084687 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c87e50 0x561f16c83c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.091+0000 7ff70da09740 1 -- 172.21.15.28:0/1978084687 >> 172.21.15.28:0/1978084687 conn(0x561f16c84360 msgr2=0x561f16c8a0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.091+0000 7ff70da09740 1 -- 172.21.15.28:0/1978084687 shutdown_connections 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.091+0000 7ff70da09740 1 -- 172.21.15.28:0/1978084687 wait complete. 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.091+0000 7ff70da09740 1 -- start start 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.091+0000 7ff70da09740 1 Processor -- start 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff70da09740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16ca4860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff70da09740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16ca4da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff70da09740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca52e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff70da09740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff7041ad730 con 0x561f16c93030 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff70da09740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff7041ad060 con 0x561f16c87e50 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff70da09740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff7041aa2a0 con 0x561f16c88770 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff70da09740 10 client.0 _ll_register_callbacks cb 0x7ff6fc046070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.092+0000 7ff708c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16ca4860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16ca4860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:46874/0 (socket says 172.21.15.28:46874) 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 -- 172.21.15.28:0/3637071160 learned_addr learned my addr 172.21.15.28:0/3637071160 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca52e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff703400640 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16ca4da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 -- 172.21.15.28:0/3637071160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 msgr2=0x561f16ca4da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16ca4da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 -- 172.21.15.28:0/3637071160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 msgr2=0x561f16ca52e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca52e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 -- 172.21.15.28:0/3637071160 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561f16d0b080 con 0x561f16c87e50 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.093+0000 7ff708c00640 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16ca4860 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc009dc0 tx=0x7ff6fc004a60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bd3cd6a5d56fcd49 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.094+0000 7ff6ed600640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.094+0000 7ff6ed600640 1 -- 172.21.15.28:0/3637071160 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6fc048070 con 0x561f16c87e50 2024-05-20T20:07:44.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.094+0000 7ff6ed600640 1 -- 172.21.15.28:0/3637071160 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ff6fc040040 con 0x561f16c87e50 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.094+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff6f0005020 con 0x561f16c87e50 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.094+0000 7ff6ed600640 1 -- 172.21.15.28:0/3637071160 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff6fc06d050 con 0x561f16c87e50 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.094+0000 7ff70da09740 10 client.25856 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.094+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561f16d0c2b0 con 0x561f16c87e50 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff6ed600640 1 -- 172.21.15.28:0/3637071160 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff6f0005020 con 0x561f16c87e50 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff6ed600640 1 -- 172.21.15.28:0/3637071160 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff6fc030650 con 0x561f16c87e50 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff70da09740 10 client.25856 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff70da09740 10 client.25856 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff70da09740 20 client.25856 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff70da09740 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16d0c860 0x561f16d0ec80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff70da09740 4 client.25856 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.095+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x561f16bf42c0 con 0x561f16d0c860 2024-05-20T20:07:44.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.096+0000 7ff703400640 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16d0c860 0x561f16d0ec80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:44.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.096+0000 7ff703400640 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16d0c860 0x561f16d0ec80 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c05ac7cb8e7e391f server_cookie=5926b7cf9541b286 in_seq=0 out_seq=0 2024-05-20T20:07:44.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.096+0000 7ff6ed600640 10 client.25856 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:44.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.100+0000 7ff6ed600640 1 -- 172.21.15.28:0/3637071160 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x561f16bf42c0 con 0x561f16d0c860 2024-05-20T20:07:44.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.100+0000 7ff6ed600640 10 client.25856 handle_command_reply: tid=0 2024-05-20T20:07:44.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.101+0000 7ff70da09740 1 client.25856 shutdown 2024-05-20T20:07:44.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.101+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 msgr2=0x561f16ca4860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.101+0000 7ff70da09740 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16ca4860 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7ff6fc009dc0 tx=0x7ff6fc004a60 comp rx=0 tx=0).stop 2024-05-20T20:07:44.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 shutdown_connections 2024-05-20T20:07:44.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561f16d0c860 0x561f16d0ec80 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561f16c93030 0x561f16ca52e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561f16c88770 0x561f16ca4da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 --2- 172.21.15.28:0/3637071160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561f16c87e50 0x561f16ca4860 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 >> 172.21.15.28:0/3637071160 conn(0x561f16c84360 msgr2=0x561f16c8a950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:44.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff6ed600640 0 client.25856 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:44.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 shutdown_connections 2024-05-20T20:07:44.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.102+0000 7ff70da09740 1 -- 172.21.15.28:0/3637071160 wait complete. 2024-05-20T20:07:44.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.103+0000 7ff70da09740 20 client.25856 trim_cache size 0 max 16384 2024-05-20T20:07:44.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:07:44.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:07:44.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:07:44.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff6e80b29d0 msgr2=0x7ff6e80b4e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff6e80b29d0 0x7ff6e80b4e90 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x561f16c893f0 tx=0x7ff6f40108b0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff704103a10 msgr2=0x7ff704199070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:44.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff704103a10 0x7ff704199070 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7ff6f4009bc0 tx=0x7ff6f400b690 comp rx=0 tx=0).stop 2024-05-20T20:07:44.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 shutdown_connections 2024-05-20T20:07:44.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff704104330 0x7ff7041995b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff6e80b29d0 0x7ff6e80b4e90 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff70410ce30 0x7ff70419d8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 --2- 172.21.15.28:0/3590383421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff704103a10 0x7ff704199070 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 >> 172.21.15.28:0/3590383421 conn(0x7ff7040fd270 msgr2=0x7ff7040feaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 shutdown_connections 2024-05-20T20:07:44.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:44.104+0000 7ff709600640 1 -- 172.21.15.28:0/3590383421 wait complete. 2024-05-20T20:07:45.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:44 smithi154 ceph-mon[36314]: pgmap v66: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:45.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:44 smithi033 ceph-mon[36179]: pgmap v66: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:45.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:44 smithi028 ceph-mon[31178]: pgmap v66: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:46 smithi154 ceph-mon[36314]: pgmap v67: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 8 op/s 2024-05-20T20:07:47.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:46 smithi033 ceph-mon[36179]: pgmap v67: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 8 op/s 2024-05-20T20:07:47.241 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:46 smithi028 ceph-mon[31178]: pgmap v67: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 8 op/s 2024-05-20T20:07:48.655 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:07:48.656 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.750+0000 7fae01400700 1 -- 172.21.15.28:0/1167474613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc108470 msgr2=0x7fadfc06b250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.750+0000 7fae01400700 1 --2- 172.21.15.28:0/1167474613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc108470 0x7fadfc06b250 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7fadec009990 tx=0x7fadec009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.751+0000 7fae01400700 1 -- 172.21.15.28:0/1167474613 shutdown_connections 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.751+0000 7fae01400700 1 --2- 172.21.15.28:0/1167474613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc105f80 0x7fadfc106410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.751+0000 7fae01400700 1 --2- 172.21.15.28:0/1167474613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc108470 0x7fadfc06b250 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.752+0000 7fae01400700 1 --2- 172.21.15.28:0/1167474613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fadfc06b790 0x7fadfc06bbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.752+0000 7fae01400700 1 -- 172.21.15.28:0/1167474613 >> 172.21.15.28:0/1167474613 conn(0x7fadfc0793f0 msgr2=0x7fadfc079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.752+0000 7fae01400700 1 -- 172.21.15.28:0/1167474613 shutdown_connections 2024-05-20T20:07:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.752+0000 7fae01400700 1 -- 172.21.15.28:0/1167474613 wait complete. 2024-05-20T20:07:48.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.754+0000 7fae01400700 1 Processor -- start 2024-05-20T20:07:48.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.754+0000 7fae01400700 1 -- start start 2024-05-20T20:07:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.754+0000 7fae01400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc06b790 0x7fadfc1a02d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fae01400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc105f80 0x7fadfc1a0810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fae01400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fadfc108470 0x7fadfc1a4ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fae01400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fadfc119d20 con 0x7fadfc06b790 2024-05-20T20:07:48.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fae01400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fadfc119ba0 con 0x7fadfc105f80 2024-05-20T20:07:48.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fae01400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fadfc119ea0 con 0x7fadfc108470 2024-05-20T20:07:48.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fadfa800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc06b790 0x7fadfc1a02d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:48.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fadfa800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc06b790 0x7fadfc1a02d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:46682/0 (socket says 172.21.15.28:46682) 2024-05-20T20:07:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.755+0000 7fadfa800700 1 -- 172.21.15.28:0/1811554416 learned_addr learned my addr 172.21.15.28:0/1811554416 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.756+0000 7fadf9e00700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc105f80 0x7fadfc1a0810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.756+0000 7fadf9e00700 1 -- 172.21.15.28:0/1811554416 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fadfc108470 msgr2=0x7fadfc1a4ba0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:07:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.756+0000 7fadf9e00700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fadfc108470 0x7fadfc1a4ba0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.756+0000 7fadf9e00700 1 -- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc06b790 msgr2=0x7fadfc1a02d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.757+0000 7fadf9e00700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc06b790 0x7fadfc1a02d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.757+0000 7fadf9e00700 1 -- 172.21.15.28:0/1811554416 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fadfc1a52e0 con 0x7fadfc105f80 2024-05-20T20:07:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.757+0000 7fadfa800700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc06b790 0x7fadfc1a02d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.758+0000 7fadf9e00700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc105f80 0x7fadfc1a0810 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fade800d750 tx=0x7fade800da60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.758+0000 7fadf3400700 1 -- 172.21.15.28:0/1811554416 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fade8017070 con 0x7fadfc105f80 2024-05-20T20:07:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.758+0000 7fadf3400700 1 -- 172.21.15.28:0/1811554416 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fade8005020 con 0x7fadfc105f80 2024-05-20T20:07:48.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.758+0000 7fadf3400700 1 -- 172.21.15.28:0/1811554416 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fade800fa10 con 0x7fadfc105f80 2024-05-20T20:07:48.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.759+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fadfc1a55d0 con 0x7fadfc105f80 2024-05-20T20:07:48.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.760+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fadfc1ace50 con 0x7fadfc105f80 2024-05-20T20:07:48.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.762+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fadfc06a230 con 0x7fadfc105f80 2024-05-20T20:07:48.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.767+0000 7fadf3400700 1 -- 172.21.15.28:0/1811554416 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fade8004a30 con 0x7fadfc105f80 2024-05-20T20:07:48.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.770+0000 7fadf3400700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fade00b2af0 0x7fade00b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:48.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.770+0000 7fadf3400700 1 -- 172.21.15.28:0/1811554416 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fade80e2140 con 0x7fadfc105f80 2024-05-20T20:07:48.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.771+0000 7fadfa800700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fade00b2af0 0x7fade00b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:48.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.772+0000 7fadfa800700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fade00b2af0 0x7fade00b4fb0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fadec009750 tx=0x7fadec00b560 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:48.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:48.773+0000 7fadf3400700 1 -- 172.21.15.28:0/1811554416 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fade8020020 con 0x7fadfc105f80 2024-05-20T20:07:49.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:48 smithi028 ceph-mon[31178]: pgmap v68: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 8 op/s 2024-05-20T20:07:49.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:48 smithi154 ceph-mon[36314]: pgmap v68: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 8 op/s 2024-05-20T20:07:49.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:48 smithi033 ceph-mon[36179]: pgmap v68: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 85 B/s wr, 8 op/s 2024-05-20T20:07:49.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.155+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fadfc1a1630 con 0x7fadfc105f80 2024-05-20T20:07:49.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.156+0000 7fadf3400700 1 -- 172.21.15.28:0/1811554416 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fade80a9ad0 con 0x7fadfc105f80 2024-05-20T20:07:49.158 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:07:49.158 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.161+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fade00b2af0 msgr2=0x7fade00b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.161+0000 7fae01400700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fade00b2af0 0x7fade00b4fb0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fadec009750 tx=0x7fadec00b560 comp rx=0 tx=0).stop 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.161+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc105f80 msgr2=0x7fadfc1a0810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.161+0000 7fae01400700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc105f80 0x7fadfc1a0810 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fade800d750 tx=0x7fade800da60 comp rx=0 tx=0).stop 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 shutdown_connections 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fadfc108470 0x7fadfc1a4ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fadfc06b790 0x7fadfc1a02d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:49.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fade00b2af0 0x7fade00b4fb0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:49.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 --2- 172.21.15.28:0/1811554416 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fadfc105f80 0x7fadfc1a0810 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:49.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 >> 172.21.15.28:0/1811554416 conn(0x7fadfc0793f0 msgr2=0x7fadfc101430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:49.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 shutdown_connections 2024-05-20T20:07:49.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:49.162+0000 7fae01400700 1 -- 172.21.15.28:0/1811554416 wait complete. 2024-05-20T20:07:49.164 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:07:50.010 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:50.048 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id f564f325 --release --await' 2024-05-20T20:07:50.048 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id f564f325 --release --await 2024-05-20T20:07:50.117 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:49 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1811554416' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:50.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:49 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1811554416' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:50.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:49 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1811554416' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:07:51.011 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:51.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:50 smithi028 ceph-mon[31178]: pgmap v69: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:51.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:50 smithi154 ceph-mon[36314]: pgmap v69: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:51.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:50 smithi033 ceph-mon[36179]: pgmap v69: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 341 B/s wr, 13 op/s 2024-05-20T20:07:51.938 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:51 smithi028 ceph-mon[31178]: pgmap v70: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:52.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:51 smithi154 ceph-mon[36314]: pgmap v70: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:52.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:51 smithi033 ceph-mon[36179]: pgmap v70: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:07:52.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.853+0000 7fe6c1400700 1 -- 172.21.15.28:0/3835595720 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc10da60 msgr2=0x7fe6bc10ff20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.853+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3835595720 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc10da60 0x7fe6bc10ff20 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fe6b4009f90 tx=0x7fe6b400b330 comp rx=0 tx=0).stop 2024-05-20T20:07:52.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.854+0000 7fe6c1400700 1 -- 172.21.15.28:0/3835595720 shutdown_connections 2024-05-20T20:07:52.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.854+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3835595720 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc10da60 0x7fe6bc10ff20 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.854+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3835595720 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6bc104630 0x7fe6bc104a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.854+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3835595720 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc104f50 0x7fe6bc10d440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.854+0000 7fe6c1400700 1 -- 172.21.15.28:0/3835595720 >> 172.21.15.28:0/3835595720 conn(0x7fe6bc0fde60 msgr2=0x7fe6bc100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:52.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.854+0000 7fe6c1400700 1 -- 172.21.15.28:0/3835595720 shutdown_connections 2024-05-20T20:07:52.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.855+0000 7fe6c1400700 1 -- 172.21.15.28:0/3835595720 wait complete. 2024-05-20T20:07:52.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.857+0000 7fe6c1400700 1 Processor -- start 2024-05-20T20:07:52.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.858+0000 7fe6c1400700 1 -- start start 2024-05-20T20:07:52.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.859+0000 7fe6c1400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc104630 0x7fe6bc074c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.859+0000 7fe6c1400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6bc104f50 0x7fe6bc0751c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.859+0000 7fe6c1400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc10da60 0x7fe6bc06df60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.860+0000 7fe6c1400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc119ca0 con 0x7fe6bc104f50 2024-05-20T20:07:52.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.860+0000 7fe6c1400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc119b20 con 0x7fe6bc104630 2024-05-20T20:07:52.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.860+0000 7fe6c1400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc119e20 con 0x7fe6bc10da60 2024-05-20T20:07:52.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.860+0000 7fe6c0a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc10da60 0x7fe6bc06df60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.860+0000 7fe6c0a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc10da60 0x7fe6bc06df60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:46122/0 (socket says 172.21.15.28:46122) 2024-05-20T20:07:52.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.860+0000 7fe6c0a00700 1 -- 172.21.15.28:0/3861324127 learned_addr learned my addr 172.21.15.28:0/3861324127 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:52.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.861+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc104630 0x7fe6bc074c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.861+0000 7fe6c0a00700 1 -- 172.21.15.28:0/3861324127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc104630 msgr2=0x7fe6bc074c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.861+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc104630 0x7fe6bc074c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.861+0000 7fe6c0a00700 1 -- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6bc104f50 msgr2=0x7fe6bc0751c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:07:52.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.861+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6bc104f50 0x7fe6bc0751c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.861+0000 7fe6c0a00700 1 -- 172.21.15.28:0/3861324127 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe6bc06e860 con 0x7fe6bc10da60 2024-05-20T20:07:52.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.862+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc10da60 0x7fe6bc06df60 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fe6b4009f60 tx=0x7fe6b40077e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:52.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.863+0000 7fe6b8c00700 1 -- 172.21.15.28:0/3861324127 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6b4007bc0 con 0x7fe6bc10da60 2024-05-20T20:07:52.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.863+0000 7fe6b8c00700 1 -- 172.21.15.28:0/3861324127 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe6b40092e0 con 0x7fe6bc10da60 2024-05-20T20:07:52.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.863+0000 7fe6b8c00700 1 -- 172.21.15.28:0/3861324127 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6b4005040 con 0x7fe6bc10da60 2024-05-20T20:07:52.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.863+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe6bc06eaf0 con 0x7fe6bc10da60 2024-05-20T20:07:52.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.863+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe6bc072980 con 0x7fe6bc10da60 2024-05-20T20:07:52.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.869+0000 7fe6cd291b80 1 Processor -- start 2024-05-20T20:07:52.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6b8c00700 1 -- 172.21.15.28:0/3861324127 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fe6b4003a00 con 0x7fe6bc10da60 2024-05-20T20:07:52.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6cd291b80 1 -- start start 2024-05-20T20:07:52.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fb480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5f5530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5f5fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555ced553ec0 con 0x555ced5fb9c0 2024-05-20T20:07:52.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555ced543a60 con 0x555ced5fb0a0 2024-05-20T20:07:52.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555ced547aa0 con 0x555ced5f5b50 2024-05-20T20:07:52.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fb480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fb480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:32804/0 (socket says 172.21.15.28:32804) 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.870+0000 7fe6bb400700 1 -- 172.21.15.28:0/1653025755 learned_addr learned my addr 172.21.15.28:0/1653025755 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.871+0000 7fe6bb400700 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5f5fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.871+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5f5530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.871+0000 7fe6bb400700 1 -- 172.21.15.28:0/1653025755 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 msgr2=0x555ced5f5fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.871+0000 7fe6bb400700 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5f5fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.871+0000 7fe6bb400700 1 -- 172.21.15.28:0/1653025755 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 msgr2=0x555ced5f5530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.871+0000 7fe6bb400700 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5f5530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.871+0000 7fe6bb400700 1 -- 172.21.15.28:0/1653025755 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555ced50c4a0 con 0x555ced5fb0a0 2024-05-20T20:07:52.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.872+0000 7fe6bb400700 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fb480 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fe6a8013740 tx=0x7fe6a8009790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=88a7cf645fa4b845 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:52.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.872+0000 7fe6b1600700 1 -- 172.21.15.28:0/1653025755 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6a8013e20 con 0x555ced5fb0a0 2024-05-20T20:07:52.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.872+0000 7fe6b1600700 1 -- 172.21.15.28:0/1653025755 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe6a8011890 con 0x555ced5fb0a0 2024-05-20T20:07:52.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.872+0000 7fe6b1600700 1 -- 172.21.15.28:0/1653025755 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6a80106c0 con 0x555ced5fb0a0 2024-05-20T20:07:52.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.873+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1653025755 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 msgr2=0x555ced5fb480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.873+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fb480 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fe6a8013740 tx=0x7fe6a8009790 comp rx=0 tx=0).stop 2024-05-20T20:07:52.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.873+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1653025755 shutdown_connections 2024-05-20T20:07:52.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.873+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5f5fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.873+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fb480 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.873+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1653025755 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5f5530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.873+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1653025755 >> 172.21.15.28:0/1653025755 conn(0x555ced523c60 msgr2=0x555ced5f01d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:52.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.874+0000 7fe6b8c00700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe6a00b2a70 0x7fe6a00b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.874+0000 7fe6bb400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe6a00b2a70 0x7fe6a00b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.875+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1653025755 shutdown_connections 2024-05-20T20:07:52.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.875+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1653025755 wait complete. 2024-05-20T20:07:52.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.875+0000 7fe6cd291b80 1 -- start start 2024-05-20T20:07:52.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.876+0000 7fe6b8c00700 1 -- 172.21.15.28:0/3861324127 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fe6b40e3090 con 0x7fe6bc10da60 2024-05-20T20:07:52.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.876+0000 7fe6bb400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe6a00b2a70 0x7fe6a00b4f30 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fe6a8005850 tx=0x7fe6a8022040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:52.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.876+0000 7fe6cd291b80 1 Processor -- start 2024-05-20T20:07:52.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.876+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced6016b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.876+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fea50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.876+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced5fef90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555ced44a020 con 0x555ced5f5b50 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555ced4e47e0 con 0x555ced5fb0a0 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555ced6024e0 con 0x555ced5fb9c0 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6cd291b80 10 client.0 _ll_register_callbacks cb 0x555ced4edcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6c0a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced5fef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6c0a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced5fef90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:46136/0 (socket says 172.21.15.28:46136) 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6c0a00700 1 -- 172.21.15.28:0/2049490703 learned_addr learned my addr 172.21.15.28:0/2049490703 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:52.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.877+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced6016b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.878+0000 7fe6c0a00700 1 -- 172.21.15.28:0/2049490703 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 msgr2=0x555ced5fea50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.878+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fea50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.878+0000 7fe6c0a00700 1 -- 172.21.15.28:0/2049490703 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 msgr2=0x555ced6016b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.878+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced6016b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.878+0000 7fe6c0a00700 1 -- 172.21.15.28:0/2049490703 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555ced493950 con 0x555ced5fb9c0 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.878+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced6016b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.879+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced5fef90 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fe6b40090a0 tx=0x7fe6b40042e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=49c7af2f025f02dd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.879+0000 7fe6a6000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.879+0000 7fe6a6000700 1 -- 172.21.15.28:0/2049490703 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6b40a9fd0 con 0x555ced5fb9c0 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.879+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe6a8011890 con 0x555ced5fb9c0 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.879+0000 7fe6a6000700 1 -- 172.21.15.28:0/2049490703 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fe6b40a4df0 con 0x555ced5fb9c0 2024-05-20T20:07:52.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.880+0000 7fe6a6000700 1 -- 172.21.15.28:0/2049490703 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe6b40aabc0 con 0x555ced5fb9c0 2024-05-20T20:07:52.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.880+0000 7fe6cd291b80 10 client.25831 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:52.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.880+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555ced5140e0 con 0x555ced5fb9c0 2024-05-20T20:07:52.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.880+0000 7fe6a6000700 1 -- 172.21.15.28:0/2049490703 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe6a8011890 con 0x555ced5fb9c0 2024-05-20T20:07:52.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6a6000700 1 -- 172.21.15.28:0/2049490703 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe6b40b2e00 con 0x555ced5fb9c0 2024-05-20T20:07:52.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6cd291b80 10 client.25831 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:52.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6cd291b80 10 client.25831 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:52.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6cd291b80 20 client.25831 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:52.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced60ed90 0x555ced611180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6cd291b80 4 client.25831 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:52.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555ced555e10 con 0x555ced60ed90 2024-05-20T20:07:52.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.881+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced60ed90 0x555ced611180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.882+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced60ed90 0x555ced611180 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e5354f4e090193cb server_cookie=37a8cd2b77c94d77 in_seq=0 out_seq=0 2024-05-20T20:07:52.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.882+0000 7fe6a6000700 10 client.25831 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:52.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.883+0000 7fe6a6000700 1 -- 172.21.15.28:0/2049490703 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x555ced555e10 con 0x555ced60ed90 2024-05-20T20:07:52.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.883+0000 7fe6a6000700 10 client.25831 handle_command_reply: tid=0 2024-05-20T20:07:52.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.883+0000 7fe6cd291b80 1 client.25831 shutdown 2024-05-20T20:07:52.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.883+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 msgr2=0x555ced5fef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.883+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced5fef90 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fe6b40090a0 tx=0x7fe6b40042e0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 shutdown_connections 2024-05-20T20:07:52.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced60ed90 0x555ced611180 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced5fef90 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced6016b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/2049490703 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 >> 172.21.15.28:0/2049490703 conn(0x555ced523c60 msgr2=0x555ced5f4520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:52.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6a6000700 0 client.25831 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:52.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 shutdown_connections 2024-05-20T20:07:52.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.884+0000 7fe6cd291b80 1 -- 172.21.15.28:0/2049490703 wait complete. 2024-05-20T20:07:52.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.885+0000 7fe6cd291b80 20 client.25831 trim_cache size 0 max 16384 2024-05-20T20:07:52.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 Processor -- start 2024-05-20T20:07:52.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 -- start start 2024-05-20T20:07:52.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5fd340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fd880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5efcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc1bc950 con 0x555ced5fb9c0 2024-05-20T20:07:52.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc1be490 con 0x555ced5fb0a0 2024-05-20T20:07:52.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.907+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc1bd020 con 0x555ced5f5b50 2024-05-20T20:07:52.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fd880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6c0a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5efcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fd880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:32826/0 (socket says 172.21.15.28:32826) 2024-05-20T20:07:52.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6bb400700 1 -- 172.21.15.28:0/853765727 learned_addr learned my addr 172.21.15.28:0/853765727 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:52.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5fd340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6bbe00700 1 -- 172.21.15.28:0/853765727 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 msgr2=0x555ced5fd880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fd880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.908+0000 7fe6bbe00700 1 -- 172.21.15.28:0/853765727 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 msgr2=0x555ced5efcb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.909+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5efcb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.909+0000 7fe6bbe00700 1 -- 172.21.15.28:0/853765727 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555ced5f4b90 con 0x555ced5f5b50 2024-05-20T20:07:52.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.909+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5efcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:52.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.909+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5fd340 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fe6b40a6a60 tx=0x7fe6ac0137c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=675803d7e96a56b0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:52.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.909+0000 7fe6b0c00700 1 -- 172.21.15.28:0/853765727 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6ac004220 con 0x555ced5f5b50 2024-05-20T20:07:52.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.909+0000 7fe6b0c00700 1 -- 172.21.15.28:0/853765727 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe6ac013e50 con 0x555ced5f5b50 2024-05-20T20:07:52.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.909+0000 7fe6b0c00700 1 -- 172.21.15.28:0/853765727 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6ac01f6b0 con 0x555ced5f5b50 2024-05-20T20:07:52.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 -- 172.21.15.28:0/853765727 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 msgr2=0x555ced5fd340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5fd340 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fe6b40a6a60 tx=0x7fe6ac0137c0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 -- 172.21.15.28:0/853765727 shutdown_connections 2024-05-20T20:07:52.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5fb9c0 0x555ced5efcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5f5b50 0x555ced5fd340 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/853765727 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced5fd880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 -- 172.21.15.28:0/853765727 >> 172.21.15.28:0/853765727 conn(0x555ced523c60 msgr2=0x555ced5f4eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:52.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.910+0000 7fe6cd291b80 1 -- 172.21.15.28:0/853765727 shutdown_connections 2024-05-20T20:07:52.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.911+0000 7fe6cd291b80 1 -- 172.21.15.28:0/853765727 wait complete. 2024-05-20T20:07:52.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.911+0000 7fe6cd291b80 1 -- start start 2024-05-20T20:07:52.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.911+0000 7fe6cd291b80 1 Processor -- start 2024-05-20T20:07:52.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.912+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced607950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.912+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced607e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.912+0000 7fe6cd291b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced604740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.912+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc1bddc0 con 0x555ced5f5b50 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.912+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc1bd6f0 con 0x555ced5fb0a0 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.912+0000 7fe6cd291b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe6bc1ba8f0 con 0x555ced5fb9c0 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.912+0000 7fe6cd291b80 10 client.0 _ll_register_callbacks cb 0x555ced4edcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.913+0000 7fe6bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced607e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.913+0000 7fe6bb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced607e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:32830/0 (socket says 172.21.15.28:32830) 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.913+0000 7fe6bb400700 1 -- 172.21.15.28:0/1396803402 learned_addr learned my addr 172.21.15.28:0/1396803402 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:52.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.913+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced607950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.913+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced604740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6bbe00700 1 -- 172.21.15.28:0/1396803402 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 msgr2=0x555ced604740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced604740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6bbe00700 1 -- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 msgr2=0x555ced607e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:52.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced607e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6bbe00700 1 -- 172.21.15.28:0/1396803402 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555ced604e50 con 0x555ced5f5b50 2024-05-20T20:07:52.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6c0a00700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced604740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:52.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6bb400700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced607e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:52.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6bbe00700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced607950 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7fe6b40a6a60 tx=0x7fe6ac0040d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=83cadd4b67e3ab68 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:52.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.914+0000 7fe6a6000700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:52.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.915+0000 7fe6a6000700 1 -- 172.21.15.28:0/1396803402 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe6ac004500 con 0x555ced5f5b50 2024-05-20T20:07:52.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.915+0000 7fe6a6000700 1 -- 172.21.15.28:0/1396803402 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fe6ac005240 con 0x555ced5f5b50 2024-05-20T20:07:52.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.915+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe6ac013e50 con 0x555ced5f5b50 2024-05-20T20:07:52.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.915+0000 7fe6a6000700 1 -- 172.21.15.28:0/1396803402 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe6ac005440 con 0x555ced5f5b50 2024-05-20T20:07:52.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.915+0000 7fe6cd291b80 10 client.15741 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:52.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.915+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555ced67a3b0 con 0x555ced5f5b50 2024-05-20T20:07:52.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6a6000700 1 -- 172.21.15.28:0/1396803402 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe6ac02c070 con 0x555ced5f5b50 2024-05-20T20:07:52.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6a6000700 1 -- 172.21.15.28:0/1396803402 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe6ac00f040 con 0x555ced5f5b50 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6cd291b80 10 client.15741 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6cd291b80 10 client.15741 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6cd291b80 20 client.15741 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced67a950 0x555ced67cd40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6cd291b80 4 client.15741 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.916+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x7fe6ac013e50 con 0x555ced67a950 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.917+0000 7fe6bb400700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced67a950 0x555ced67cd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:52.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.917+0000 7fe6bb400700 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced67a950 0x555ced67cd40 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9197b76a38b8a386 server_cookie=238ecb82596b0159 in_seq=0 out_seq=0 2024-05-20T20:07:52.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.917+0000 7fe6a6000700 10 client.15741 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:52.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.919+0000 7fe6a6000700 1 -- 172.21.15.28:0/1396803402 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x7fe6ac013e50 con 0x555ced67a950 2024-05-20T20:07:52.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.919+0000 7fe6a6000700 10 client.15741 handle_command_reply: tid=0 2024-05-20T20:07:52.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.919+0000 7fe6cd291b80 1 client.15741 shutdown 2024-05-20T20:07:52.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.919+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 msgr2=0x555ced607950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.919+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced607950 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7fe6b40a6a60 tx=0x7fe6ac0040d0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 shutdown_connections 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555ced67a950 0x555ced67cd40 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ced5fb9c0 0x555ced604740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ced5f5b50 0x555ced607950 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 --2- 172.21.15.28:0/1396803402 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ced5fb0a0 0x555ced607e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 >> 172.21.15.28:0/1396803402 conn(0x555ced523c60 msgr2=0x555ced5f4e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6a6000700 0 client.15741 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 shutdown_connections 2024-05-20T20:07:52.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 1 -- 172.21.15.28:0/1396803402 wait complete. 2024-05-20T20:07:52.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.920+0000 7fe6cd291b80 20 client.15741 trim_cache size 0 max 16384 2024-05-20T20:07:52.921 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:07:52.921 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:07:52.921 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "3d9edc3d-5956-4d2e-ac6d-02b5e73f7ad3", 2024-05-20T20:07:52.921 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:07:52.921 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.921+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe6a00b2a70 msgr2=0x7fe6a00b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.921+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe6a00b2a70 0x7fe6a00b4f30 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fe6a8005850 tx=0x7fe6a8022040 comp rx=0 tx=0).stop 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.921+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc10da60 msgr2=0x7fe6bc06df60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.921+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc10da60 0x7fe6bc06df60 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fe6b4009f60 tx=0x7fe6b40077e0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 shutdown_connections 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe6bc10da60 0x7fe6bc06df60 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe6a00b2a70 0x7fe6a00b4f30 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe6bc104630 0x7fe6bc074c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 --2- 172.21.15.28:0/3861324127 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe6bc104f50 0x7fe6bc0751c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:52.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 >> 172.21.15.28:0/3861324127 conn(0x7fe6bc0fde60 msgr2=0x7fe6bc0feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:52.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 shutdown_connections 2024-05-20T20:07:52.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:52.922+0000 7fe6c1400700 1 -- 172.21.15.28:0/3861324127 wait complete. 2024-05-20T20:07:53.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:53 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:07:53.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:53 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:53.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:53 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:07:53.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:53 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:53.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:53 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:07:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:53 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:07:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:53 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:53 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:07:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:53 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:53.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:53 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:07:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:53 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:07:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:53 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:53 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:07:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:53 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:07:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:53 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:07:54.307 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:07:54.398 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:54 smithi028 ceph-mon[31178]: pgmap v71: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:54.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:54 smithi154 ceph-mon[36314]: pgmap v71: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:54 smithi033 ceph-mon[36179]: pgmap v71: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:07:54.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.983+0000 7f785d200700 1 -- 172.21.15.28:0/3991857281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 msgr2=0x7f7858105c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:54.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.983+0000 7f785d200700 1 --2- 172.21.15.28:0/3991857281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 0x7f7858105c00 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f7844009990 tx=0x7f7844009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:54.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.984+0000 7f785d200700 1 -- 172.21.15.28:0/3991857281 shutdown_connections 2024-05-20T20:07:54.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.984+0000 7f785d200700 1 --2- 172.21.15.28:0/3991857281 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7858106140 0x7f7858112650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:54.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.984+0000 7f785d200700 1 --2- 172.21.15.28:0/3991857281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f785810b780 0x7f785810bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:54.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.984+0000 7f785d200700 1 --2- 172.21.15.28:0/3991857281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 0x7f7858105c00 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:54.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.984+0000 7f785d200700 1 -- 172.21.15.28:0/3991857281 >> 172.21.15.28:0/3991857281 conn(0x7f7858101270 msgr2=0x7f7858103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:54.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.984+0000 7f785d200700 1 -- 172.21.15.28:0/3991857281 shutdown_connections 2024-05-20T20:07:54.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.984+0000 7f785d200700 1 -- 172.21.15.28:0/3991857281 wait complete. 2024-05-20T20:07:54.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.985+0000 7f785d200700 1 Processor -- start 2024-05-20T20:07:54.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.986+0000 7f785d200700 1 -- start start 2024-05-20T20:07:54.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.986+0000 7f785d200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 0x7f785819bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:54.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.986+0000 7f785d200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7858106140 0x7f785819c410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:54.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.986+0000 7f785d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f785810b780 0x7f78581a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:54.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.986+0000 7f785d200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f78581159c0 con 0x7f7858106140 2024-05-20T20:07:54.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.986+0000 7f785d200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7858115840 con 0x7f7858105780 2024-05-20T20:07:54.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.986+0000 7f785d200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7858115b40 con 0x7f785810b780 2024-05-20T20:07:54.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.987+0000 7f7856800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f785810b780 0x7f78581a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:54.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.987+0000 7f7856800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f785810b780 0x7f78581a0750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:46174/0 (socket says 172.21.15.28:46174) 2024-05-20T20:07:54.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.987+0000 7f7856800700 1 -- 172.21.15.28:0/3492910614 learned_addr learned my addr 172.21.15.28:0/3492910614 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:54.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.987+0000 7f7855e00700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 0x7f785819bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:54.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.987+0000 7f7855400700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7858106140 0x7f785819c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:54.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7856800700 1 -- 172.21.15.28:0/3492910614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 msgr2=0x7f785819bed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:54.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7856800700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 0x7f785819bed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:54.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7856800700 1 -- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7858106140 msgr2=0x7f785819c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:54.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7856800700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7858106140 0x7f785819c410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:54.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7856800700 1 -- 172.21.15.28:0/3492910614 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f78581a0e90 con 0x7f785810b780 2024-05-20T20:07:54.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7855e00700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 0x7f785819bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:54.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7855400700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7858106140 0x7f785819c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:07:54.991 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:54.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.988+0000 7f7856800700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f785810b780 0x7f78581a0750 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f785000b970 tx=0x7f785000bd30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:54.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.989+0000 7f784ea00700 1 -- 172.21.15.28:0/3492910614 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7850019070 con 0x7f785810b780 2024-05-20T20:07:54.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.989+0000 7f784ea00700 1 -- 172.21.15.28:0/3492910614 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7850014440 con 0x7f785810b780 2024-05-20T20:07:54.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.989+0000 7f784ea00700 1 -- 172.21.15.28:0/3492910614 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f785000ce60 con 0x7f785810b780 2024-05-20T20:07:54.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.989+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f78581a1180 con 0x7f785810b780 2024-05-20T20:07:54.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.989+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f78581a8a00 con 0x7f785810b780 2024-05-20T20:07:54.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.994+0000 7f784ea00700 1 -- 172.21.15.28:0/3492910614 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f785001c070 con 0x7f785810b780 2024-05-20T20:07:54.997 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.996+0000 7f7867becb80 1 Processor -- start 2024-05-20T20:07:54.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.996+0000 7f7867becb80 1 -- start start 2024-05-20T20:07:54.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.997+0000 7f7867becb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f47a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:54.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.997+0000 7f7867becb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f4ce0 0x5597f57ee7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:54.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.997+0000 7f7867becb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57ef280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:54.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.997+0000 7f7867becb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5597f5743f40 con 0x5597f57eee00 2024-05-20T20:07:54.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.997+0000 7f7867becb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5597f5733ae0 con 0x5597f57f43c0 2024-05-20T20:07:54.998 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.997+0000 7f7867becb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5597f5737b20 con 0x5597f57f4ce0 2024-05-20T20:07:54.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.997+0000 7f784ea00700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f783c0b2b40 0x7f783c0b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:54.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f47a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:54.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f47a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:32848/0 (socket says 172.21.15.28:32848) 2024-05-20T20:07:54.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855400700 1 -- 172.21.15.28:0/145424392 learned_addr learned my addr 172.21.15.28:0/145424392 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:54.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855e00700 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f4ce0 0x5597f57ee7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:54.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f784ea00700 1 -- 172.21.15.28:0/3492910614 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f78500e18c0 con 0x7f785810b780 2024-05-20T20:07:55.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855e00700 1 -- 172.21.15.28:0/145424392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 msgr2=0x5597f57f47a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855e00700 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f47a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855e00700 1 -- 172.21.15.28:0/145424392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 msgr2=0x5597f57ef280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855e00700 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57ef280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.998+0000 7f7855e00700 1 -- 172.21.15.28:0/145424392 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5597f56fc520 con 0x5597f57f4ce0 2024-05-20T20:07:55.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:54.999+0000 7f7855400700 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f47a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:55.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.000+0000 7f7856800700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f783c0b2b40 0x7f783c0b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.001+0000 7f7856800700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f783c0b2b40 0x7f783c0b5000 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f78500df130 tx=0x7f7850004be0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:55.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.002+0000 7f7855e00700 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f4ce0 0x5597f57ee7e0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f7848009cc0 tx=0x7f78480113e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3241d5de377964ab server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:55.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.002+0000 7f7843400700 1 -- 172.21.15.28:0/145424392 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7848011ce0 con 0x5597f57f4ce0 2024-05-20T20:07:55.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.003+0000 7f7843400700 1 -- 172.21.15.28:0/145424392 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7848011e80 con 0x5597f57f4ce0 2024-05-20T20:07:55.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.003+0000 7f7843400700 1 -- 172.21.15.28:0/145424392 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f784800f810 con 0x5597f57f4ce0 2024-05-20T20:07:55.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.003+0000 7f7867becb80 1 -- 172.21.15.28:0/145424392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f4ce0 msgr2=0x5597f57ee7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:55.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.003+0000 7f7867becb80 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f4ce0 0x5597f57ee7e0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f7848009cc0 tx=0x7f78480113e0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.004+0000 7f7867becb80 1 -- 172.21.15.28:0/145424392 shutdown_connections 2024-05-20T20:07:55.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.004+0000 7f7867becb80 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57ef280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.004+0000 7f7867becb80 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f47a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.004+0000 7f7867becb80 1 --2- 172.21.15.28:0/145424392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f4ce0 0x5597f57ee7e0 secure :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f7848009cc0 tx=0x7f78480113e0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.004+0000 7f7867becb80 1 -- 172.21.15.28:0/145424392 >> 172.21.15.28:0/145424392 conn(0x5597f5713ce0 msgr2=0x5597f57e0e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:55.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.005+0000 7f7867becb80 1 -- 172.21.15.28:0/145424392 shutdown_connections 2024-05-20T20:07:55.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.005+0000 7f7867becb80 1 -- 172.21.15.28:0/145424392 wait complete. 2024-05-20T20:07:55.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.006+0000 7f7867becb80 1 -- start start 2024-05-20T20:07:55.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.007+0000 7f7867becb80 1 Processor -- start 2024-05-20T20:07:55.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.007+0000 7f7867becb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57fac80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7867becb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f7f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7867becb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f88a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7867becb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5597f563a0a0 con 0x5597f57eee00 2024-05-20T20:07:55.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7867becb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5597f56d4860 con 0x5597f57f43c0 2024-05-20T20:07:55.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7867becb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5597f580c7a0 con 0x5597f57f8440 2024-05-20T20:07:55.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7867becb80 10 client.0 _ll_register_callbacks cb 0x5597f56ddd70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:55.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7855400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57fac80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7855e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f7f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7855e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f7f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:32864/0 (socket says 172.21.15.28:32864) 2024-05-20T20:07:55.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.008+0000 7f7855e00700 1 -- 172.21.15.28:0/34625161 learned_addr learned my addr 172.21.15.28:0/34625161 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:55.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.009+0000 7f7855400700 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f88a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.010+0000 7f7855400700 1 -- 172.21.15.28:0/34625161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 msgr2=0x5597f57f88a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.010+0000 7f7855400700 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f88a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.010+0000 7f7855400700 1 -- 172.21.15.28:0/34625161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 msgr2=0x5597f57f7f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.010+0000 7f7855400700 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f7f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.010+0000 7f7855400700 1 -- 172.21.15.28:0/34625161 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5597f56839d0 con 0x5597f57eee00 2024-05-20T20:07:55.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.010+0000 7f7855400700 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57fac80 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f7844009960 tx=0x7f7844005940 comp rx=0 tx=0).ready entity=mon.0 client_cookie=67326c1fa1e911b4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:55.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.011+0000 7f7837e00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:55.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.011+0000 7f7837e00700 1 -- 172.21.15.28:0/34625161 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7844003880 con 0x5597f57eee00 2024-05-20T20:07:55.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.011+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5597f5745e90 con 0x5597f57eee00 2024-05-20T20:07:55.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.011+0000 7f7837e00700 1 -- 172.21.15.28:0/34625161 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7844003680 con 0x5597f57eee00 2024-05-20T20:07:55.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.011+0000 7f7837e00700 1 -- 172.21.15.28:0/34625161 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f7844003880 con 0x5597f57eee00 2024-05-20T20:07:55.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.012+0000 7f7867becb80 10 client.15750 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:55.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.012+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5597f5704160 con 0x5597f57eee00 2024-05-20T20:07:55.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.012+0000 7f7837e00700 1 -- 172.21.15.28:0/34625161 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x5597f5745e90 con 0x5597f57eee00 2024-05-20T20:07:55.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.013+0000 7f7837e00700 1 -- 172.21.15.28:0/34625161 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7844027070 con 0x5597f57eee00 2024-05-20T20:07:55.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.013+0000 7f7867becb80 10 client.15750 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:55.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.013+0000 7f7867becb80 10 client.15750 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:07:55.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.013+0000 7f7867becb80 20 client.15750 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:55.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.013+0000 7f7867becb80 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f5819830 0x5597f581bc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.013+0000 7f7867becb80 4 client.15750 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:55.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.013+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5597f573f670 con 0x5597f5819830 2024-05-20T20:07:55.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.015+0000 7f7855e00700 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f5819830 0x5597f581bc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.016+0000 7f7837e00700 10 client.15750 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:55.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.016+0000 7f7855e00700 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f5819830 0x5597f581bc20 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=75e2b8a518078fdc server_cookie=ca49a4d228a47ab0 in_seq=0 out_seq=0 2024-05-20T20:07:55.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.017+0000 7f7837e00700 1 -- 172.21.15.28:0/34625161 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5597f573f670 con 0x5597f5819830 2024-05-20T20:07:55.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.017+0000 7f7837e00700 10 client.15750 handle_command_reply: tid=0 2024-05-20T20:07:55.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.017+0000 7f7867becb80 1 client.15750 shutdown 2024-05-20T20:07:55.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.017+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 msgr2=0x5597f57fac80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.017+0000 7f7867becb80 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57fac80 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f7844009960 tx=0x7f7844005940 comp rx=0 tx=0).stop 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 shutdown_connections 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f5819830 0x5597f581bc20 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f88a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f57fac80 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 --2- 172.21.15.28:0/34625161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f57f7f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 >> 172.21.15.28:0/34625161 conn(0x5597f5713ce0 msgr2=0x5597f57e0cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:55.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7837e00700 0 client.15750 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:55.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 shutdown_connections 2024-05-20T20:07:55.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.018+0000 7f7867becb80 1 -- 172.21.15.28:0/34625161 wait complete. 2024-05-20T20:07:55.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.019+0000 7f7867becb80 20 client.15750 trim_cache size 0 max 16384 2024-05-20T20:07:55.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.061+0000 7f7867becb80 1 Processor -- start 2024-05-20T20:07:55.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.062+0000 7f7867becb80 1 -- start start 2024-05-20T20:07:55.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7867becb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57eee00 0x5597f57ee7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7867becb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57f43c0 0x5597f57f60d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7867becb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f37e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7867becb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f785806ffe0 con 0x5597f57f43c0 2024-05-20T20:07:55.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7867becb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7858071b20 con 0x5597f57eee00 2024-05-20T20:07:55.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7867becb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f78580706b0 con 0x5597f57f8440 2024-05-20T20:07:55.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7855400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57eee00 0x5597f57ee7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7855400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57eee00 0x5597f57ee7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:32866/0 (socket says 172.21.15.28:32866) 2024-05-20T20:07:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7855400700 1 -- 172.21.15.28:0/2909808978 learned_addr learned my addr 172.21.15.28:0/2909808978 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.063+0000 7f7855e00700 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57f43c0 0x5597f57f60d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.064+0000 7f7855400700 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f37e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.064+0000 7f7855400700 1 -- 172.21.15.28:0/2909808978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 msgr2=0x5597f57f37e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.064+0000 7f7855400700 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f37e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.064+0000 7f7855400700 1 -- 172.21.15.28:0/2909808978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57f43c0 msgr2=0x5597f57f60d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.066+0000 7f7855400700 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57f43c0 0x5597f57f60d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.066+0000 7f7855400700 1 -- 172.21.15.28:0/2909808978 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5597f581c0f0 con 0x5597f57eee00 2024-05-20T20:07:55.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.066+0000 7f7855e00700 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57f43c0 0x5597f57f60d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:55.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.066+0000 7f7855400700 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57eee00 0x5597f57ee7e0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f784400b2e0 tx=0x7f7844009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5a6c7b5e267e682b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:55.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.066+0000 7f7842a00700 1 -- 172.21.15.28:0/2909808978 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f784400fae0 con 0x5597f57eee00 2024-05-20T20:07:55.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.066+0000 7f7842a00700 1 -- 172.21.15.28:0/2909808978 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f784400fe50 con 0x5597f57eee00 2024-05-20T20:07:55.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.066+0000 7f7842a00700 1 -- 172.21.15.28:0/2909808978 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f78440296d0 con 0x5597f57eee00 2024-05-20T20:07:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.067+0000 7f7867becb80 1 -- 172.21.15.28:0/2909808978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57eee00 msgr2=0x5597f57ee7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:55.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.067+0000 7f7867becb80 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57eee00 0x5597f57ee7e0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f784400b2e0 tx=0x7f7844009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.070+0000 7f7867becb80 1 -- 172.21.15.28:0/2909808978 shutdown_connections 2024-05-20T20:07:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.070+0000 7f7867becb80 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f57f37e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.070+0000 7f7867becb80 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57eee00 0x5597f57ee7e0 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.070+0000 7f7867becb80 1 --2- 172.21.15.28:0/2909808978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57f43c0 0x5597f57f60d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.070+0000 7f7867becb80 1 -- 172.21.15.28:0/2909808978 >> 172.21.15.28:0/2909808978 conn(0x5597f5713ce0 msgr2=0x5597f57f30d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:55.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.070+0000 7f7867becb80 1 -- 172.21.15.28:0/2909808978 shutdown_connections 2024-05-20T20:07:55.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.071+0000 7f7867becb80 1 -- 172.21.15.28:0/2909808978 wait complete. 2024-05-20T20:07:55.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.071+0000 7f7867becb80 1 -- start start 2024-05-20T20:07:55.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.071+0000 7f7867becb80 1 Processor -- start 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7867becb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f5811c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7867becb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f5812150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7867becb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f580ea00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7867becb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7858071450 con 0x5597f57eee00 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7867becb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7858070d80 con 0x5597f57f43c0 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7867becb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f785807bb40 con 0x5597f57f8440 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7867becb80 10 client.0 _ll_register_callbacks cb 0x5597f56ddd70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:55.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.072+0000 7f7855400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f5811c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.073+0000 7f7855400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f5811c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:46782/0 (socket says 172.21.15.28:46782) 2024-05-20T20:07:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.073+0000 7f7855400700 1 -- 172.21.15.28:0/29228375 learned_addr learned my addr 172.21.15.28:0/29228375 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.073+0000 7f7855400700 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f580ea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.073+0000 7f7855e00700 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f5812150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.074+0000 7f7855400700 1 -- 172.21.15.28:0/29228375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 msgr2=0x5597f580ea00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.074+0000 7f7855400700 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f580ea00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.074+0000 7f7855400700 1 -- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 msgr2=0x5597f5812150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:55.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.074+0000 7f7855400700 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f5812150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.074+0000 7f7855400700 1 -- 172.21.15.28:0/29228375 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5597f580f180 con 0x5597f57eee00 2024-05-20T20:07:55.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.074+0000 7f7855400700 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f5811c10 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f7844009900 tx=0x7f784400f8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5a112ff95485ed9b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:55.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.075+0000 7f7837e00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:07:55.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.075+0000 7f7837e00700 1 -- 172.21.15.28:0/29228375 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f784404a060 con 0x5597f57eee00 2024-05-20T20:07:55.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.075+0000 7f7837e00700 1 -- 172.21.15.28:0/29228375 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7844003c20 con 0x5597f57eee00 2024-05-20T20:07:55.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.075+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f784400fe50 con 0x5597f57eee00 2024-05-20T20:07:55.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.075+0000 7f7837e00700 1 -- 172.21.15.28:0/29228375 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f784400f670 con 0x5597f57eee00 2024-05-20T20:07:55.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.076+0000 7f7867becb80 10 client.15756 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:55.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.076+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5597f5884dd0 con 0x5597f57eee00 2024-05-20T20:07:55.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.076+0000 7f7837e00700 1 -- 172.21.15.28:0/29228375 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f784400fe50 con 0x5597f57eee00 2024-05-20T20:07:55.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.076+0000 7f7837e00700 1 -- 172.21.15.28:0/29228375 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7844027040 con 0x5597f57eee00 2024-05-20T20:07:55.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.077+0000 7f7867becb80 10 client.15756 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:55.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.077+0000 7f7867becb80 10 client.15756 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:07:55.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.077+0000 7f7867becb80 20 client.15756 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:55.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.077+0000 7f7867becb80 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f58853d0 0x5597f58877c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:55.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.077+0000 7f7867becb80 4 client.15756 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "f564f325", "release": true, "await": true}] 2024-05-20T20:07:55.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.077+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "f564f325", "release": true, "await": true}) -- 0x5597f55b6350 con 0x5597f58853d0 2024-05-20T20:07:55.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.077+0000 7f7855e00700 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f58853d0 0x5597f58877c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:55.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.078+0000 7f7855e00700 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f58853d0 0x5597f58877c0 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=775bb1a1ec690181 server_cookie=332817f34dd93ea9 in_seq=0 out_seq=0 2024-05-20T20:07:55.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.078+0000 7f7837e00700 10 client.15756 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:55.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.080+0000 7f7837e00700 1 -- 172.21.15.28:0/29228375 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+589 (crc 0 0 0) 0x5597f55b6350 con 0x5597f58853d0 2024-05-20T20:07:55.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.080+0000 7f7837e00700 10 client.15756 handle_command_reply: tid=0 2024-05-20T20:07:55.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.080+0000 7f7867becb80 1 client.15756 shutdown 2024-05-20T20:07:55.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.080+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 msgr2=0x5597f5811c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:55.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.080+0000 7f7867becb80 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f5811c10 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f7844009900 tx=0x7f784400f8c0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 shutdown_connections 2024-05-20T20:07:55.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5597f58853d0 0x5597f58877c0 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5597f57f8440 0x5597f580ea00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5597f57eee00 0x5597f5811c10 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 --2- 172.21.15.28:0/29228375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5597f57f43c0 0x5597f5812150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7837e00700 0 client.15756 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 >> 172.21.15.28:0/29228375 conn(0x5597f5713ce0 msgr2=0x5597f57e0cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 shutdown_connections 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 1 -- 172.21.15.28:0/29228375 wait complete. 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.081+0000 7f7867becb80 20 client.15756 trim_cache size 0 max 16384 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 4, 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:07:55.083 INFO:teuthology.orchestra.run.smithi028.stdout: "f564f325": { 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 4, 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 146.3, 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:07:55.084 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:07:55.085 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:07:55.085 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:55.085 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:55.085 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:55.085 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:55.085 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:07:55.085 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f783c0b2b40 msgr2=0x7f783c0b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f783c0b2b40 0x7f783c0b5000 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f78500df130 tx=0x7f7850004be0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f785810b780 msgr2=0x7f78581a0750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f785810b780 0x7f78581a0750 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f785000b970 tx=0x7f785000bd30 comp rx=0 tx=0).stop 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 shutdown_connections 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f785810b780 0x7f78581a0750 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f783c0b2b40 0x7f783c0b5000 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7858105780 0x7f785819bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 --2- 172.21.15.28:0/3492910614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7858106140 0x7f785819c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:55.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.083+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 >> 172.21.15.28:0/3492910614 conn(0x7f7858101270 msgr2=0x7f7858101890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:55.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.084+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 shutdown_connections 2024-05-20T20:07:55.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:55.084+0000 7f785d200700 1 -- 172.21.15.28:0/3492910614 wait complete. 2024-05-20T20:07:56.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:56 smithi028 ceph-mon[31178]: pgmap v72: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:07:56.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:56 smithi154 ceph-mon[36314]: pgmap v72: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:07:56.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:56 smithi033 ceph-mon[36179]: pgmap v72: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:07:56.755 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set 'f564f325', total seconds elapsed: 41.6 2024-05-20T20:07:56.756 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 26.35 2024-05-20T20:07:56.756 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:07:57.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 -- 172.21.15.28:0/2663336154 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc108a80 msgr2=0x7f98fc108e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 --2- 172.21.15.28:0/2663336154 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc108a80 0x7f98fc108e60 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f98ec009cd0 tx=0x7f98ec02a8f0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 -- 172.21.15.28:0/2663336154 shutdown_connections 2024-05-20T20:07:57.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 --2- 172.21.15.28:0/2663336154 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f98fc102a70 0x7f98fc102ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 --2- 172.21.15.28:0/2663336154 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f98fc103430 0x7f98fc10f940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 --2- 172.21.15.28:0/2663336154 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc108a80 0x7f98fc108e60 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 -- 172.21.15.28:0/2663336154 >> 172.21.15.28:0/2663336154 conn(0x7f98fc0fe500 msgr2=0x7f98fc100920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:57.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 -- 172.21.15.28:0/2663336154 shutdown_connections 2024-05-20T20:07:57.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.285+0000 7f9902000640 1 -- 172.21.15.28:0/2663336154 wait complete. 2024-05-20T20:07:57.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.287+0000 7f9902000640 1 Processor -- start 2024-05-20T20:07:57.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.287+0000 7f9902000640 1 -- start start 2024-05-20T20:07:57.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.287+0000 7f9902000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f98fc102a70 0x7f98fc1099a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f9902000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc103430 0x7f98fc109ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f9902000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f98fc108a80 0x7f98fc10a420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f9902000640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc112d90 con 0x7f98fc102a70 2024-05-20T20:07:57.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f9902000640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc112c10 con 0x7f98fc108a80 2024-05-20T20:07:57.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f9902000640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc112f10 con 0x7f98fc103430 2024-05-20T20:07:57.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc103430 0x7f98fc109ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc103430 0x7f98fc109ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38296/0 (socket says 172.21.15.28:38296) 2024-05-20T20:07:57.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f98f2a00640 1 -- 172.21.15.28:0/1052341443 learned_addr learned my addr 172.21.15.28:0/1052341443 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:57.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f98fb400640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f98fc102a70 0x7f98fc1099a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.288+0000 7f98fbe00640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f98fc108a80 0x7f98fc10a420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.289+0000 7f98fbe00640 1 -- 172.21.15.28:0/1052341443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc103430 msgr2=0x7f98fc109ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.289+0000 7f98fbe00640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc103430 0x7f98fc109ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.289+0000 7f98fbe00640 1 -- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f98fc102a70 msgr2=0x7f98fc1099a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.289+0000 7f98fbe00640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f98fc102a70 0x7f98fc1099a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.289+0000 7f98fbe00640 1 -- 172.21.15.28:0/1052341443 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f98fc1ab9f0 con 0x7f98fc108a80 2024-05-20T20:07:57.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.289+0000 7f98fb400640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f98fc102a70 0x7f98fc1099a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:07:57.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.290+0000 7f98fbe00640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f98fc108a80 0x7f98fc10a420 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f98f400e9a0 tx=0x7f98f400ee60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:57.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.290+0000 7f98f8c00640 1 -- 172.21.15.28:0/1052341443 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98f400ccc0 con 0x7f98fc108a80 2024-05-20T20:07:57.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.290+0000 7f98f8c00640 1 -- 172.21.15.28:0/1052341443 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f98f400ce60 con 0x7f98fc108a80 2024-05-20T20:07:57.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.290+0000 7f98f8c00640 1 -- 172.21.15.28:0/1052341443 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98f4010450 con 0x7f98fc108a80 2024-05-20T20:07:57.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.290+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f98fc1abce0 con 0x7f98fc108a80 2024-05-20T20:07:57.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.291+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f98fc1ac1e0 con 0x7f98fc108a80 2024-05-20T20:07:57.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.296+0000 7f9904f7c740 1 Processor -- start 2024-05-20T20:07:57.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.301+0000 7f98f8c00640 1 -- 172.21.15.28:0/1052341443 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f98f4003710 con 0x7f98fc108a80 2024-05-20T20:07:57.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.301+0000 7f9904f7c740 1 -- start start 2024-05-20T20:07:57.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.302+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x555982194070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.302+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.302+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821a1260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.302+0000 7f9904f7c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555982104b90 con 0x55598219ee70 2024-05-20T20:07:57.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.302+0000 7f9904f7c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5559820f4730 con 0x5559821945b0 2024-05-20T20:07:57.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.302+0000 7f9904f7c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5559820f8770 con 0x555982193c90 2024-05-20T20:07:57.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.303+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x555982194070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.303+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x555982194070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38304/0 (socket says 172.21.15.28:38304) 2024-05-20T20:07:57.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.303+0000 7f98f2a00640 1 -- 172.21.15.28:0/1837852904 learned_addr learned my addr 172.21.15.28:0/1837852904 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:57.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.303+0000 7f98fb400640 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98f2a00640 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821a1260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98f8c00640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f98e00b2ae0 0x7f98e00b4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98fb400640 1 -- 172.21.15.28:0/1837852904 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 msgr2=0x555982194070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:57.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98fb400640 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x555982194070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98fb400640 1 -- 172.21.15.28:0/1837852904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 msgr2=0x5559821a1260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:57.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98fb400640 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821a1260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98fb400640 1 -- 172.21.15.28:0/1837852904 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55598208d8b0 con 0x5559821945b0 2024-05-20T20:07:57.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98f2a00640 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821a1260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:57.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.304+0000 7f98f8c00640 1 -- 172.21.15.28:0/1052341443 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f98f40e2680 con 0x7f98fc108a80 2024-05-20T20:07:57.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.306+0000 7f98fbe00640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f98e00b2ae0 0x7f98e00b4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.306+0000 7f98fb400640 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219e930 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f98ec006010 tx=0x7f98ec005f70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=33f1863e9b838225 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:57.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.306+0000 7f98ebe00640 1 -- 172.21.15.28:0/1837852904 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98ec030a00 con 0x5559821945b0 2024-05-20T20:07:57.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.310+0000 7f98ebe00640 1 -- 172.21.15.28:0/1837852904 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f98ec036030 con 0x5559821945b0 2024-05-20T20:07:57.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.310+0000 7f98ebe00640 1 -- 172.21.15.28:0/1837852904 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98ec004750 con 0x5559821945b0 2024-05-20T20:07:57.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.311+0000 7f98fbe00640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f98e00b2ae0 0x7f98e00b4fa0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f98f40a9670 tx=0x7f98f400ef90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:57.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.311+0000 7f9904f7c740 1 -- 172.21.15.28:0/1837852904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 msgr2=0x55598219e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.312+0000 7f9904f7c740 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219e930 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f98ec006010 tx=0x7f98ec005f70 comp rx=0 tx=0).stop 2024-05-20T20:07:57.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.313+0000 7f9904f7c740 1 -- 172.21.15.28:0/1837852904 shutdown_connections 2024-05-20T20:07:57.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.313+0000 7f9904f7c740 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219e930 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.313+0000 7f9904f7c740 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821a1260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.313+0000 7f9904f7c740 1 --2- 172.21.15.28:0/1837852904 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x555982194070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.313+0000 7f9904f7c740 1 -- 172.21.15.28:0/1837852904 >> 172.21.15.28:0/1837852904 conn(0x5559821901a0 msgr2=0x555982190580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:57.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.313+0000 7f9904f7c740 1 -- 172.21.15.28:0/1837852904 shutdown_connections 2024-05-20T20:07:57.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.313+0000 7f9904f7c740 1 -- 172.21.15.28:0/1837852904 wait complete. 2024-05-20T20:07:57.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.314+0000 7f9904f7c740 1 -- start start 2024-05-20T20:07:57.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.314+0000 7f9904f7c740 1 Processor -- start 2024-05-20T20:07:57.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598219e830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219bb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x55598219c130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f9904f7c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555981dda790 con 0x55598219ee70 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f9904f7c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55598209cbc0 con 0x5559821945b0 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f9904f7c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5559821a1a40 con 0x555982193c90 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f9904f7c740 10 client.0 _ll_register_callbacks cb 0x55598207a400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f98fb400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219bb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f98fb400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219bb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58002/0 (socket says 172.21.15.28:58002) 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.316+0000 7f98fb400640 1 -- 172.21.15.28:0/2352617644 learned_addr learned my addr 172.21.15.28:0/2352617644 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:57.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.317+0000 7f98fb400640 1 -- 172.21.15.28:0/2352617644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 msgr2=0x55598219e830 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:07:57.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.317+0000 7f98fb400640 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598219e830 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.317+0000 7f98fb400640 1 -- 172.21.15.28:0/2352617644 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 msgr2=0x55598219c130 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:07:57.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.317+0000 7f98fb400640 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x55598219c130 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.317+0000 7f98fb400640 1 -- 172.21.15.28:0/2352617644 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555981e09210 con 0x5559821945b0 2024-05-20T20:07:57.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.317+0000 7f98fb400640 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219bb20 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f98ec031040 tx=0x7f98ec0099d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d66c26d55325e6a6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:57.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.317+0000 7f98e8c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:57.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.318+0000 7f98e8c00640 1 -- 172.21.15.28:0/2352617644 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98ec052020 con 0x5559821945b0 2024-05-20T20:07:57.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.318+0000 7f98e8c00640 1 -- 172.21.15.28:0/2352617644 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f98ec003970 con 0x5559821945b0 2024-05-20T20:07:57.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.319+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f98ec036030 con 0x5559821945b0 2024-05-20T20:07:57.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.319+0000 7f98e8c00640 1 -- 172.21.15.28:0/2352617644 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f98ec003b70 con 0x5559821945b0 2024-05-20T20:07:57.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.319+0000 7f9904f7c740 10 client.25904 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:57.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.319+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555981ed5620 con 0x5559821945b0 2024-05-20T20:07:57.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.320+0000 7f98e8c00640 1 -- 172.21.15.28:0/2352617644 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f98ec036030 con 0x5559821945b0 2024-05-20T20:07:57.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.320+0000 7f98e8c00640 1 -- 172.21.15.28:0/2352617644 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f98ec043070 con 0x5559821945b0 2024-05-20T20:07:57.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.321+0000 7f9904f7c740 10 client.25904 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:57.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.321+0000 7f9904f7c740 10 client.25904 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:57.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.321+0000 7f9904f7c740 20 client.25904 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:57.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.321+0000 7f9904f7c740 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559821ae5d0 0x5559821b09c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.321+0000 7f9904f7c740 4 client.25904 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:57.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.321+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555982106ae0 con 0x5559821ae5d0 2024-05-20T20:07:57.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.322+0000 7f98f2a00640 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559821ae5d0 0x5559821b09c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.322+0000 7f98f2a00640 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559821ae5d0 0x5559821b09c0 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9ed5b5e445dcf005 server_cookie=43838e9cd6029a96 in_seq=0 out_seq=0 2024-05-20T20:07:57.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.322+0000 7f98e8c00640 10 client.25904 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:57.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f98e8c00640 1 -- 172.21.15.28:0/2352617644 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x555982106ae0 con 0x5559821ae5d0 2024-05-20T20:07:57.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f98e8c00640 10 client.25904 handle_command_reply: tid=0 2024-05-20T20:07:57.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 client.25904 shutdown 2024-05-20T20:07:57.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 msgr2=0x55598219bb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219bb20 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f98ec031040 tx=0x7f98ec0099d0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 shutdown_connections 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559821ae5d0 0x5559821b09c0 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5559821945b0 0x55598219bb20 secure :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f98ec031040 tx=0x7f98ec0099d0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x55598219c130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f9904f7c740 1 --2- 172.21.15.28:0/2352617644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598219e830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.324+0000 7f98e8c00640 0 client.25904 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.325+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 >> 172.21.15.28:0/2352617644 conn(0x5559821901a0 msgr2=0x555982195ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.325+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 shutdown_connections 2024-05-20T20:07:57.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.325+0000 7f9904f7c740 1 -- 172.21.15.28:0/2352617644 wait complete. 2024-05-20T20:07:57.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.325+0000 7f9904f7c740 20 client.25904 trim_cache size 0 max 16384 2024-05-20T20:07:57.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.358+0000 7f9904f7c740 1 Processor -- start 2024-05-20T20:07:57.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.359+0000 7f9904f7c740 1 -- start start 2024-05-20T20:07:57.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.359+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598218fa90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.359+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x55598219ba20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.359+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55598219e360 0x55598219e7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.359+0000 7f9904f7c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc1b4c10 con 0x55598219ee70 2024-05-20T20:07:57.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.359+0000 7f9904f7c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc1b6750 con 0x55598219e360 2024-05-20T20:07:57.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.359+0000 7f9904f7c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc1b52e0 con 0x555982193c90 2024-05-20T20:07:57.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.360+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598218fa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.360+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598218fa90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38326/0 (socket says 172.21.15.28:38326) 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.360+0000 7f98f2a00640 1 -- 172.21.15.28:0/827058950 learned_addr learned my addr 172.21.15.28:0/827058950 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.360+0000 7f98f2a00640 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55598219e360 0x55598219e7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.361+0000 7f98f2a00640 1 -- 172.21.15.28:0/827058950 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55598219e360 msgr2=0x55598219e7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.361+0000 7f98f2a00640 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55598219e360 0x55598219e7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.361+0000 7f98f2a00640 1 -- 172.21.15.28:0/827058950 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 msgr2=0x55598219ba20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.361+0000 7f98f2a00640 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x55598219ba20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.361+0000 7f98f2a00640 1 -- 172.21.15.28:0/827058950 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55598218e6b0 con 0x555982193c90 2024-05-20T20:07:57.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.361+0000 7f98f2a00640 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598218fa90 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x55598219c730 tx=0x7f98e40095d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=79bf9b753df1d598 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:57.363 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.362+0000 7f98eb400640 1 -- 172.21.15.28:0/827058950 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98e4010b80 con 0x555982193c90 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.362+0000 7f98eb400640 1 -- 172.21.15.28:0/827058950 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f98e4010d20 con 0x555982193c90 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.362+0000 7f98eb400640 1 -- 172.21.15.28:0/827058950 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98e4004c30 con 0x555982193c90 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 -- 172.21.15.28:0/827058950 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 msgr2=0x55598218fa90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598218fa90 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x55598219c730 tx=0x7f98e40095d0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 -- 172.21.15.28:0/827058950 shutdown_connections 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55598219e360 0x55598219e7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x55598219ba20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 --2- 172.21.15.28:0/827058950 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555982193c90 0x55598218fa90 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 -- 172.21.15.28:0/827058950 >> 172.21.15.28:0/827058950 conn(0x5559821901a0 msgr2=0x555982197160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:57.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 -- 172.21.15.28:0/827058950 shutdown_connections 2024-05-20T20:07:57.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.363+0000 7f9904f7c740 1 -- 172.21.15.28:0/827058950 wait complete. 2024-05-20T20:07:57.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.364+0000 7f9904f7c740 1 -- start start 2024-05-20T20:07:57.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.364+0000 7f9904f7c740 1 Processor -- start 2024-05-20T20:07:57.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555982193c90 0x5559821b4050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.366 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55598219e360 0x5559821b13e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.366 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f9904f7c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821b1920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.366 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f9904f7c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc1b6080 con 0x55598219ee70 2024-05-20T20:07:57.366 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f9904f7c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc1b59b0 con 0x555982193c90 2024-05-20T20:07:57.366 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f9904f7c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f98fc1b2bf0 con 0x55598219e360 2024-05-20T20:07:57.366 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f9904f7c740 10 client.0 _ll_register_callbacks cb 0x7f98ec009b20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:57.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555982193c90 0x5559821b4050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f98f2a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555982193c90 0x5559821b4050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58022/0 (socket says 172.21.15.28:58022) 2024-05-20T20:07:57.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.365+0000 7f98f2a00640 1 -- 172.21.15.28:0/3507765418 learned_addr learned my addr 172.21.15.28:0/3507765418 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:57.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98f2a00640 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821b1920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98f2a00640 1 -- 172.21.15.28:0/3507765418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55598219e360 msgr2=0x5559821b13e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:57.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98fb400640 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55598219e360 0x5559821b13e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98f2a00640 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55598219e360 0x5559821b13e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.368 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98f2a00640 1 -- 172.21.15.28:0/3507765418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 msgr2=0x5559821b1920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:57.368 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98f2a00640 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821b1920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.368 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98f2a00640 1 -- 172.21.15.28:0/3507765418 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5559821b20d0 con 0x555982193c90 2024-05-20T20:07:57.368 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.366+0000 7f98f2a00640 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555982193c90 0x5559821b4050 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f98e4010980 tx=0x7f98e4020040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e78e14cf23bbb07d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:57.368 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.367+0000 7f98e8c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:57.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.367+0000 7f98e8c00640 1 -- 172.21.15.28:0/3507765418 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98e402b6f0 con 0x555982193c90 2024-05-20T20:07:57.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.367+0000 7f98e8c00640 1 -- 172.21.15.28:0/3507765418 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f98e402b890 con 0x555982193c90 2024-05-20T20:07:57.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.367+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f98e4010d20 con 0x555982193c90 2024-05-20T20:07:57.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.367+0000 7f98e8c00640 1 -- 172.21.15.28:0/3507765418 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f98e402ba90 con 0x555982193c90 2024-05-20T20:07:57.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.367+0000 7f9904f7c740 10 client.25910 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:57.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.367+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555982218da0 con 0x555982193c90 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f98e8c00640 1 -- 172.21.15.28:0/3507765418 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f98e4010d20 con 0x555982193c90 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f98e8c00640 1 -- 172.21.15.28:0/3507765418 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f98e402acd0 con 0x555982193c90 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f9904f7c740 10 client.25910 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f9904f7c740 10 client.25910 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f9904f7c740 20 client.25910 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f9904f7c740 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559822192b0 0x55598221b6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f9904f7c740 4 client.25910 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.368+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5559821002c0 con 0x5559822192b0 2024-05-20T20:07:57.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.369+0000 7f98fb400640 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559822192b0 0x55598221b6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.370+0000 7f98fb400640 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559822192b0 0x55598221b6a0 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=83fe0a83a6d2a772 server_cookie=34a76e9165fb23de in_seq=0 out_seq=0 2024-05-20T20:07:57.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.370+0000 7f98e8c00640 10 client.25910 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:57.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f98e8c00640 1 -- 172.21.15.28:0/3507765418 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5559821002c0 con 0x5559822192b0 2024-05-20T20:07:57.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f98e8c00640 10 client.25910 handle_command_reply: tid=0 2024-05-20T20:07:57.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 client.25910 shutdown 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555982193c90 msgr2=0x5559821b4050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555982193c90 0x5559821b4050 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f98e4010980 tx=0x7f98e4020040 comp rx=0 tx=0).stop 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 shutdown_connections 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5559822192b0 0x55598221b6a0 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55598219ee70 0x5559821b1920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55598219e360 0x5559821b13e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 --2- 172.21.15.28:0/3507765418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555982193c90 0x5559821b4050 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 >> 172.21.15.28:0/3507765418 conn(0x5559821901a0 msgr2=0x55598218ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:57.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.374+0000 7f98e8c00640 0 client.25910 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:57.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.375+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 shutdown_connections 2024-05-20T20:07:57.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.375+0000 7f9904f7c740 1 -- 172.21.15.28:0/3507765418 wait complete. 2024-05-20T20:07:57.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.375+0000 7f9904f7c740 20 client.25910 trim_cache size 0 max 16384 2024-05-20T20:07:57.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:07:57.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:07:57.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:07:57.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f98e00b2ae0 msgr2=0x7f98e00b4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f98e00b2ae0 0x7f98e00b4fa0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f98f40a9670 tx=0x7f98f400ef90 comp rx=0 tx=0).stop 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f98fc108a80 msgr2=0x7f98fc10a420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f98fc108a80 0x7f98fc10a420 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f98f400e9a0 tx=0x7f98f400ee60 comp rx=0 tx=0).stop 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 shutdown_connections 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f98fc108a80 0x7f98fc10a420 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f98e00b2ae0 0x7f98e00b4fa0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f98fc103430 0x7f98fc109ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 --2- 172.21.15.28:0/1052341443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f98fc102a70 0x7f98fc1099a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.376+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 >> 172.21.15.28:0/1052341443 conn(0x7f98fc0fe500 msgr2=0x7f98fc0fe990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:57.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.378+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 shutdown_connections 2024-05-20T20:07:57.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:07:57.378+0000 7f9902000640 1 -- 172.21.15.28:0/1052341443 wait complete. 2024-05-20T20:07:57.882 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.985+0000 7f80d2a00700 1 -- 172.21.15.28:0/2862670705 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 msgr2=0x7f80cc06b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.985+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2862670705 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 0x7f80cc06b6a0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f80b8009990 tx=0x7f80b8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.985+0000 7f80d2a00700 1 -- 172.21.15.28:0/2862670705 shutdown_connections 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.985+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2862670705 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f80cc06bbe0 0x7f80cc1103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.985+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2862670705 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f80cc1095b0 0x7f80cc109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.985+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2862670705 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 0x7f80cc06b6a0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.985+0000 7f80d2a00700 1 -- 172.21.15.28:0/2862670705 >> 172.21.15.28:0/2862670705 conn(0x7f80cc0793f0 msgr2=0x7f80cc079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:57.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.986+0000 7f80d2a00700 1 -- 172.21.15.28:0/2862670705 shutdown_connections 2024-05-20T20:07:57.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.986+0000 7f80d2a00700 1 -- 172.21.15.28:0/2862670705 wait complete. 2024-05-20T20:07:57.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.987+0000 7f80d2a00700 1 Processor -- start 2024-05-20T20:07:57.989 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.988+0000 7f80d2a00700 1 -- start start 2024-05-20T20:07:57.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80d2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 0x7f80cc1a08e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80d2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f80cc06bbe0 0x7f80cc1a0e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80d2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f80cc1095b0 0x7f80cc10f720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:57.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80cbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 0x7f80cc1a08e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80cbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 0x7f80cc1a08e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38356/0 (socket says 172.21.15.28:38356) 2024-05-20T20:07:57.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f80cc113710 con 0x7f80cc06bbe0 2024-05-20T20:07:57.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80cbe00700 1 -- 172.21.15.28:0/2315308148 learned_addr learned my addr 172.21.15.28:0/2315308148 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:57.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f80cc113590 con 0x7f80cc1095b0 2024-05-20T20:07:57.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80d0a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f80cc1095b0 0x7f80cc10f720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.989+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f80cc113890 con 0x7f80cc06b220 2024-05-20T20:07:57.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.990+0000 7f80cb400700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f80cc06bbe0 0x7f80cc1a0e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:57.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.990+0000 7f80d0a00700 1 -- 172.21.15.28:0/2315308148 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 msgr2=0x7f80cc1a08e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.990+0000 7f80d0a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 0x7f80cc1a08e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.990+0000 7f80d0a00700 1 -- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f80cc06bbe0 msgr2=0x7f80cc1a0e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:57.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.990+0000 7f80d0a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f80cc06bbe0 0x7f80cc1a0e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:57.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.990+0000 7f80d0a00700 1 -- 172.21.15.28:0/2315308148 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f80cc10fff0 con 0x7f80cc1095b0 2024-05-20T20:07:57.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.992+0000 7f80d0a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f80cc1095b0 0x7f80cc10f720 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f80c400efc0 tx=0x7f80c400c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:57.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.992+0000 7f80c8c00700 1 -- 172.21.15.28:0/2315308148 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80c40092e0 con 0x7f80cc1095b0 2024-05-20T20:07:57.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.992+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f80cc110280 con 0x7f80cc1095b0 2024-05-20T20:07:57.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.992+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f80cc19c560 con 0x7f80cc1095b0 2024-05-20T20:07:57.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.992+0000 7f80c8c00700 1 -- 172.21.15.28:0/2315308148 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f80c4004bb0 con 0x7f80cc1095b0 2024-05-20T20:07:57.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.992+0000 7f80c8c00700 1 -- 172.21.15.28:0/2315308148 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80c4004020 con 0x7f80cc1095b0 2024-05-20T20:07:57.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.998+0000 7f80dd42db80 1 Processor -- start 2024-05-20T20:07:58.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.998+0000 7f80c8c00700 1 -- 172.21.15.28:0/2315308148 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f80c401b030 con 0x7f80cc1095b0 2024-05-20T20:07:58.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.999+0000 7f80dd42db80 1 -- start start 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.999+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c627290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:57.999+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c621260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80dd42db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e65c57fe40 con 0x55e65c6277d0 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80dd42db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e65c56f9e0 con 0x55e65c621880 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80dd42db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e65c573a20 con 0x55e65c626eb0 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80cb400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c627290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80cb400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c627290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38362/0 (socket says 172.21.15.28:38362) 2024-05-20T20:07:58.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80cb400700 1 -- 172.21.15.28:0/137336525 learned_addr learned my addr 172.21.15.28:0/137336525 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:58.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.000+0000 7f80cbe00700 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c621260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80cb400700 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80c8c00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f80b00b2af0 0x7f80b00b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80c8c00700 1 -- 172.21.15.28:0/2315308148 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f80c40e1fe0 con 0x7f80cc1095b0 2024-05-20T20:07:58.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80cb400700 1 -- 172.21.15.28:0/137336525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 msgr2=0x55e65c621ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:58.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80cb400700 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80cb400700 1 -- 172.21.15.28:0/137336525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 msgr2=0x55e65c621260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:58.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80cb400700 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c621260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.001+0000 7f80cb400700 1 -- 172.21.15.28:0/137336525 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e65c538420 con 0x55e65c626eb0 2024-05-20T20:07:58.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.002+0000 7f80cb400700 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c627290 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f80b8015000 tx=0x7f80b800b260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6467ee04b4d8ca09 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:58.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.002+0000 7f80d0a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f80b00b2af0 0x7f80b00b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.004 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.002+0000 7f80c1600700 1 -- 172.21.15.28:0/137336525 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80b800b920 con 0x55e65c626eb0 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.002+0000 7f80c1600700 1 -- 172.21.15.28:0/137336525 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f80b8022040 con 0x55e65c626eb0 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.003+0000 7f80c1600700 1 -- 172.21.15.28:0/137336525 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80b8003e00 con 0x55e65c626eb0 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.003+0000 7f80dd42db80 1 -- 172.21.15.28:0/137336525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 msgr2=0x55e65c627290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.003+0000 7f80dd42db80 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c627290 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f80b8015000 tx=0x7f80b800b260 comp rx=0 tx=0).stop 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.003+0000 7f80d0a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f80b00b2af0 0x7f80b00b4fb0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f80c40df740 tx=0x7f80c4009f70 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 -- 172.21.15.28:0/137336525 shutdown_connections 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c627290 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 --2- 172.21.15.28:0/137336525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c621260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 -- 172.21.15.28:0/137336525 >> 172.21.15.28:0/137336525 conn(0x55e65c54fbe0 msgr2=0x55e65c61c010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:58.005 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 -- 172.21.15.28:0/137336525 shutdown_connections 2024-05-20T20:07:58.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 -- 172.21.15.28:0/137336525 wait complete. 2024-05-20T20:07:58.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.004+0000 7f80dd42db80 1 -- start start 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 1 Processor -- start 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 0x55e65c62d410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c626eb0 0x55e65c62a7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c6277d0 0x55e65c62acf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e65c475fa0 con 0x55e65c626eb0 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e65c510760 con 0x55e65c6277d0 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e65c62e230 con 0x55e65c621880 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.005+0000 7f80dd42db80 10 client.0 _ll_register_callbacks cb 0x55e65c519c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:58.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.006+0000 7f80cb400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 0x55e65c62d410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.006+0000 7f80cb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c6277d0 0x55e65c62acf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.006+0000 7f80cb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c6277d0 0x55e65c62acf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58066/0 (socket says 172.21.15.28:58066) 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.006+0000 7f80cb400700 1 -- 172.21.15.28:0/3577102206 learned_addr learned my addr 172.21.15.28:0/3577102206 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.006+0000 7f80cbe00700 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c626eb0 0x55e65c62a7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80cb400700 1 -- 172.21.15.28:0/3577102206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 msgr2=0x55e65c62d410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80cb400700 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 0x55e65c62d410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80cb400700 1 -- 172.21.15.28:0/3577102206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c626eb0 msgr2=0x55e65c62a7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80cb400700 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c626eb0 0x55e65c62a7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80cb400700 1 -- 172.21.15.28:0/3577102206 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e65c4bf8d0 con 0x55e65c6277d0 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80cb400700 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c6277d0 0x55e65c62acf0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f80b8005850 tx=0x7f80b8009510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6ac02b1fb19f4b59 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80b6000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80b6000700 1 -- 172.21.15.28:0/3577102206 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80b802d020 con 0x55e65c6277d0 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80b6000700 1 -- 172.21.15.28:0/3577102206 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f80b800b920 con 0x55e65c6277d0 2024-05-20T20:07:58.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.007+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f80b8022040 con 0x55e65c6277d0 2024-05-20T20:07:58.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.008+0000 7f80b6000700 1 -- 172.21.15.28:0/3577102206 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f80b8040050 con 0x55e65c6277d0 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.008+0000 7f80dd42db80 10 client.25922 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.008+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e65c540060 con 0x55e65c6277d0 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.008+0000 7f80b6000700 1 -- 172.21.15.28:0/3577102206 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f80b8029480 con 0x55e65c6277d0 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.008+0000 7f80b6000700 1 -- 172.21.15.28:0/3577102206 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f80b8027070 con 0x55e65c6277d0 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.009+0000 7f80dd42db80 10 client.25922 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.009+0000 7f80dd42db80 10 client.25922 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.009+0000 7f80dd42db80 20 client.25922 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:58.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.009+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c63ad50 0x55e65c63d170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.009+0000 7f80dd42db80 4 client.25922 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:07:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.009+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f80b8022040 con 0x55e65c63ad50 2024-05-20T20:07:58.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.009+0000 7f80cbe00700 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c63ad50 0x55e65c63d170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.010+0000 7f80cbe00700 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c63ad50 0x55e65c63d170 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4a83458b36898e35 server_cookie=82b9c992dbdc8d7b in_seq=0 out_seq=0 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.010+0000 7f80b6000700 10 client.25922 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80b6000700 1 -- 172.21.15.28:0/3577102206 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x7f80b8022040 con 0x55e65c63ad50 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80b6000700 10 client.25922 handle_command_reply: tid=0 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 client.25922 shutdown 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c6277d0 msgr2=0x55e65c62acf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c6277d0 0x55e65c62acf0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f80b8005850 tx=0x7f80b8009510 comp rx=0 tx=0).stop 2024-05-20T20:07:58.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 shutdown_connections 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c63ad50 0x55e65c63d170 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80b6000700 0 client.25922 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c6277d0 0x55e65c62acf0 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 0x55e65c62d410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3577102206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c626eb0 0x55e65c62a7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.011+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 >> 172.21.15.28:0/3577102206 conn(0x55e65c54fbe0 msgr2=0x55e65c620250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.012+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 shutdown_connections 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.012+0000 7f80dd42db80 1 -- 172.21.15.28:0/3577102206 wait complete. 2024-05-20T20:07:58.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.012+0000 7f80dd42db80 20 client.25922 trim_cache size 0 max 16384 2024-05-20T20:07:58.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.055+0000 7f80dd42db80 1 Processor -- start 2024-05-20T20:07:58.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80dd42db80 1 -- start start 2024-05-20T20:07:58.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c6283a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c642a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80dd42db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f80ac00d590 con 0x55e65c6277d0 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80dd42db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f80ac00f0d0 con 0x55e65c621880 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80dd42db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f80ac00dc60 con 0x55e65c626eb0 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.056+0000 7f80cb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58082/0 (socket says 172.21.15.28:58082) 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 -- 172.21.15.28:0/797052822 learned_addr learned my addr 172.21.15.28:0/797052822 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c642a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cbe00700 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c6283a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 -- 172.21.15.28:0/797052822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 msgr2=0x55e65c6283a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c6283a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 -- 172.21.15.28:0/797052822 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 msgr2=0x55e65c621260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.057+0000 7f80cb400700 1 -- 172.21.15.28:0/797052822 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e65c628d40 con 0x55e65c6277d0 2024-05-20T20:07:58.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.058+0000 7f80cb400700 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c642a40 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f80b8003a50 tx=0x7f80b8009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=db9234f6c961261d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:58.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.059+0000 7f80c0c00700 1 -- 172.21.15.28:0/797052822 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80b8031070 con 0x55e65c6277d0 2024-05-20T20:07:58.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.059+0000 7f80c0c00700 1 -- 172.21.15.28:0/797052822 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f80b8029cf0 con 0x55e65c6277d0 2024-05-20T20:07:58.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.059+0000 7f80c0c00700 1 -- 172.21.15.28:0/797052822 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80b8034040 con 0x55e65c6277d0 2024-05-20T20:07:58.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.060+0000 7f80dd42db80 1 -- 172.21.15.28:0/797052822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 msgr2=0x55e65c642a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:58.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.060+0000 7f80dd42db80 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c642a40 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f80b8003a50 tx=0x7f80b8009310 comp rx=0 tx=0).stop 2024-05-20T20:07:58.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.060+0000 7f80dd42db80 1 -- 172.21.15.28:0/797052822 shutdown_connections 2024-05-20T20:07:58.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.060+0000 7f80dd42db80 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c6277d0 0x55e65c642a40 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.060+0000 7f80dd42db80 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c621880 0x55e65c621260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.060+0000 7f80dd42db80 1 --2- 172.21.15.28:0/797052822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c626eb0 0x55e65c6283a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.060+0000 7f80dd42db80 1 -- 172.21.15.28:0/797052822 >> 172.21.15.28:0/797052822 conn(0x55e65c54fbe0 msgr2=0x55e65c627c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:58.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.061+0000 7f80dd42db80 1 -- 172.21.15.28:0/797052822 shutdown_connections 2024-05-20T20:07:58.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.061+0000 7f80dd42db80 1 -- 172.21.15.28:0/797052822 wait complete. 2024-05-20T20:07:58.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.062+0000 7f80dd42db80 1 -- start start 2024-05-20T20:07:58.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.062+0000 7f80dd42db80 1 Processor -- start 2024-05-20T20:07:58.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 0x55e65c633570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c626eb0 0x55e65c633ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80dd42db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c630540 0x55e65c6309c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80dd42db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f80ac00ea00 con 0x55e65c630540 2024-05-20T20:07:58.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80dd42db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f80ac00e330 con 0x55e65c626eb0 2024-05-20T20:07:58.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80dd42db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f80ac00b530 con 0x55e65c621880 2024-05-20T20:07:58.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80dd42db80 10 client.0 _ll_register_callbacks cb 0x55e65c519c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:07:58.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.063+0000 7f80cbe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c626eb0 0x55e65c633ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.064+0000 7f80cbe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c626eb0 0x55e65c633ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58084/0 (socket says 172.21.15.28:58084) 2024-05-20T20:07:58.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.064+0000 7f80cbe00700 1 -- 172.21.15.28:0/3903518073 learned_addr learned my addr 172.21.15.28:0/3903518073 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:07:58.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.064+0000 7f80cbe00700 1 -- 172.21.15.28:0/3903518073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 msgr2=0x55e65c633570 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:07:58.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.064+0000 7f80cbe00700 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 0x55e65c633570 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.064+0000 7f80cbe00700 1 -- 172.21.15.28:0/3903518073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c630540 msgr2=0x55e65c6309c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:07:58.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.064+0000 7f80cb400700 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c630540 0x55e65c6309c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.065+0000 7f80cbe00700 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c630540 0x55e65c6309c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.065+0000 7f80cbe00700 1 -- 172.21.15.28:0/3903518073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e65c634130 con 0x55e65c626eb0 2024-05-20T20:07:58.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.065+0000 7f80cb400700 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c630540 0x55e65c6309c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:07:58.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.066+0000 7f80cbe00700 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c626eb0 0x55e65c633ab0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f80bc004790 tx=0x7f80bc009d70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5d671cdf43961d83 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:07:58.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.066+0000 7f80b6000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:07:58.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.066+0000 7f80b6000700 1 -- 172.21.15.28:0/3903518073 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f80bc003e50 con 0x55e65c626eb0 2024-05-20T20:07:58.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.066+0000 7f80b6000700 1 -- 172.21.15.28:0/3903518073 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f80bc01e480 con 0x55e65c626eb0 2024-05-20T20:07:58.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.066+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f80b8029cf0 con 0x55e65c626eb0 2024-05-20T20:07:58.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.067+0000 7f80b6000700 1 -- 172.21.15.28:0/3903518073 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f80bc01e680 con 0x55e65c626eb0 2024-05-20T20:07:58.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.067+0000 7f80dd42db80 10 client.25928 fetch_fsmap learned FSMap version 29 2024-05-20T20:07:58.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.067+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e65c6a6400 con 0x55e65c626eb0 2024-05-20T20:07:58.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.067+0000 7f80b6000700 1 -- 172.21.15.28:0/3903518073 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f80b8029cf0 con 0x55e65c626eb0 2024-05-20T20:07:58.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.068+0000 7f80b6000700 1 -- 172.21.15.28:0/3903518073 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f80bc025020 con 0x55e65c626eb0 2024-05-20T20:07:58.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.068+0000 7f80dd42db80 10 client.25928 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:07:58.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.068+0000 7f80dd42db80 10 client.25928 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:07:58.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.068+0000 7f80dd42db80 20 client.25928 populate_metadata read hostname 'smithi028' 2024-05-20T20:07:58.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.068+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c6a6aa0 0x55e65c6a8ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:07:58.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.068+0000 7f80dd42db80 4 client.25928 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:07:58.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.068+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55e65c581d90 con 0x55e65c6a6aa0 2024-05-20T20:07:58.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.069+0000 7f80cb400700 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c6a6aa0 0x55e65c6a8ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:07:58.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.069+0000 7f80cb400700 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c6a6aa0 0x55e65c6a8ee0 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4d5600de5d4f71be server_cookie=9fed9577ce2c0c47 in_seq=0 out_seq=0 2024-05-20T20:07:58.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.069+0000 7f80b6000700 10 client.25928 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:58.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80b6000700 1 -- 172.21.15.28:0/3903518073 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x55e65c581d90 con 0x55e65c6a6aa0 2024-05-20T20:07:58.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80b6000700 10 client.25928 handle_command_reply: tid=0 2024-05-20T20:07:58.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 client.25928 shutdown 2024-05-20T20:07:58.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c626eb0 msgr2=0x55e65c633ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c626eb0 0x55e65c633ab0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f80bc004790 tx=0x7f80bc009d70 comp rx=0 tx=0).stop 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 shutdown_connections 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e65c6a6aa0 0x55e65c6a8ee0 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e65c630540 0x55e65c6309c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e65c621880 0x55e65c633570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 --2- 172.21.15.28:0/3903518073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e65c626eb0 0x55e65c633ab0 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80b6000700 0 client.25928 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.070+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 >> 172.21.15.28:0/3903518073 conn(0x55e65c54fbe0 msgr2=0x55e65c62bdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.071+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 shutdown_connections 2024-05-20T20:07:58.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.071+0000 7f80dd42db80 1 -- 172.21.15.28:0/3903518073 wait complete. 2024-05-20T20:07:58.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.071+0000 7f80dd42db80 20 client.25928 trim_cache size 0 max 16384 2024-05-20T20:07:58.073 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:07:58.073 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:07:58.073 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:07:58.073 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:07:58.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.072+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f80b00b2af0 msgr2=0x7f80b00b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:58.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.072+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f80b00b2af0 0x7f80b00b4fb0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f80c40df740 tx=0x7f80c4009f70 comp rx=0 tx=0).stop 2024-05-20T20:07:58.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.072+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f80cc1095b0 msgr2=0x7f80cc10f720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:07:58.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.072+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f80cc1095b0 0x7f80cc10f720 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f80c400efc0 tx=0x7f80c400c3a0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.073+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 shutdown_connections 2024-05-20T20:07:58.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.073+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f80b00b2af0 0x7f80b00b4fb0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.073+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f80cc1095b0 0x7f80cc10f720 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.073+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80cc06b220 0x7f80cc1a08e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.073+0000 7f80d2a00700 1 --2- 172.21.15.28:0/2315308148 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f80cc06bbe0 0x7f80cc1a0e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:07:58.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.073+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 >> 172.21.15.28:0/2315308148 conn(0x7f80cc0793f0 msgr2=0x7f80cc101060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:07:58.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.074+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 shutdown_connections 2024-05-20T20:07:58.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:07:58.074+0000 7f80d2a00700 1 -- 172.21.15.28:0/2315308148 wait complete. 2024-05-20T20:07:59.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:58 smithi028 ceph-mon[31178]: pgmap v73: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:07:59.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:58 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:07:59.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:58 smithi154 ceph-mon[36314]: pgmap v73: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:07:59.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:58 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:07:59.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:58 smithi033 ceph-mon[36179]: pgmap v73: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:07:59.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:58 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:07:59.200 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:07:59.200 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:07:59.958 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:59 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:07:59.959 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:59 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:07:59.959 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:07:59 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:08:00.074 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:00.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:59 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:08:00.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:59 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:08:00.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:07:59 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:08:00.130 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:59 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:08:00.130 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:59 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:08:00.130 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:07:59 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:08:00.786 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:00 smithi028 ceph-mon[31178]: pgmap v74: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 511 B/s wr, 14 op/s 2024-05-20T20:08:01.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.064+0000 7f5537e00700 1 -- 172.21.15.28:0/535741713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538103490 msgr2=0x7f5538103910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:01.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.064+0000 7f5537e00700 1 --2- 172.21.15.28:0/535741713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538103490 0x7f5538103910 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f5524009990 tx=0x7f5524009ca0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.064+0000 7f5537e00700 1 -- 172.21.15.28:0/535741713 shutdown_connections 2024-05-20T20:08:01.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.064+0000 7f5537e00700 1 --2- 172.21.15.28:0/535741713 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5538103e50 0x7f5538110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.064+0000 7f5537e00700 1 --2- 172.21.15.28:0/535741713 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538109540 0x7f5538109920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.064+0000 7f5537e00700 1 --2- 172.21.15.28:0/535741713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538103490 0x7f5538103910 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.064+0000 7f5537e00700 1 -- 172.21.15.28:0/535741713 >> 172.21.15.28:0/535741713 conn(0x7f55380fefc0 msgr2=0x7f55381013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:01.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.065+0000 7f5537e00700 1 -- 172.21.15.28:0/535741713 shutdown_connections 2024-05-20T20:08:01.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.067+0000 7f5537e00700 1 -- 172.21.15.28:0/535741713 wait complete. 2024-05-20T20:08:01.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.069+0000 7f5537e00700 1 Processor -- start 2024-05-20T20:08:01.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.069+0000 7f5537e00700 1 -- start start 2024-05-20T20:08:01.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5537e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538103490 0x7f5538075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:01.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5537e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5538103e50 0x7f5538075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:01.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5537e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538109540 0x7f55380762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:01.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5537e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5538113620 con 0x7f5538103490 2024-05-20T20:08:01.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5537e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f55381134a0 con 0x7f5538103e50 2024-05-20T20:08:01.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5537e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f55381137a0 con 0x7f5538109540 2024-05-20T20:08:01.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5536a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538103490 0x7f5538075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:01.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5536a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538103490 0x7f5538075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54130/0 (socket says 172.21.15.28:54130) 2024-05-20T20:08:01.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.070+0000 7f5536a00700 1 -- 172.21.15.28:0/2477065225 learned_addr learned my addr 172.21.15.28:0/2477065225 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:01.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5537400700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538109540 0x7f55380762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:01.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5536a00700 1 -- 172.21.15.28:0/2477065225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538109540 msgr2=0x7f55380762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:01.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5536a00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538109540 0x7f55380762c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5536a00700 1 -- 172.21.15.28:0/2477065225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5538103e50 msgr2=0x7f5538075d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:01.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5536000700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5538103e50 0x7f5538075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:01.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5536a00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5538103e50 0x7f5538075d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5536a00700 1 -- 172.21.15.28:0/2477065225 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f55381a7f90 con 0x7f5538103490 2024-05-20T20:08:01.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.071+0000 7f5536000700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5538103e50 0x7f5538075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:08:01.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.072+0000 7f5536a00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538103490 0x7f5538075840 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f552800eb30 tx=0x7f552800eef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:01.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.072+0000 7f552f400700 1 -- 172.21.15.28:0/2477065225 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f552800cd40 con 0x7f5538103490 2024-05-20T20:08:01.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.073+0000 7f552f400700 1 -- 172.21.15.28:0/2477065225 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f55280044d0 con 0x7f5538103490 2024-05-20T20:08:01.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.073+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f55381a8280 con 0x7f5538103490 2024-05-20T20:08:01.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.073+0000 7f552f400700 1 -- 172.21.15.28:0/2477065225 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5528010430 con 0x7f5538103490 2024-05-20T20:08:01.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.073+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f55381a87c0 con 0x7f5538103490 2024-05-20T20:08:01.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.074+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f553806a230 con 0x7f5538103490 2024-05-20T20:08:01.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.079+0000 7f552f400700 1 -- 172.21.15.28:0/2477065225 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f5528003fb0 con 0x7f5538103490 2024-05-20T20:08:01.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.081+0000 7f552f400700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f551c0b29f0 0x7f551c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:01.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.081+0000 7f552f400700 1 -- 172.21.15.28:0/2477065225 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f55280e2930 con 0x7f5538103490 2024-05-20T20:08:01.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.082+0000 7f5536000700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f551c0b29f0 0x7f551c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:01.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.083+0000 7f5536000700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f551c0b29f0 0x7f551c0b4eb0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f5524000c00 tx=0x7f5524005fb0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:01.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.090+0000 7f552f400700 1 -- 172.21.15.28:0/2477065225 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f55280aa2c0 con 0x7f5538103490 2024-05-20T20:08:01.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:00 smithi154 ceph-mon[36314]: pgmap v74: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 511 B/s wr, 14 op/s 2024-05-20T20:08:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:00 smithi033 ceph-mon[36179]: pgmap v74: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 511 B/s wr, 14 op/s 2024-05-20T20:08:01.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.536+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f553802e560 con 0x7f5538103490 2024-05-20T20:08:01.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.537+0000 7f552f400700 1 -- 172.21.15.28:0/2477065225 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f55280a9bc0 con 0x7f5538103490 2024-05-20T20:08:01.538 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:08:01.538 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:08:01.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f551c0b29f0 msgr2=0x7f551c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f551c0b29f0 0x7f551c0b4eb0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f5524000c00 tx=0x7f5524005fb0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538103490 msgr2=0x7f5538075840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538103490 0x7f5538075840 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f552800eb30 tx=0x7f552800eef0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 shutdown_connections 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5538109540 0x7f55380762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5538103490 0x7f5538075840 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f551c0b29f0 0x7f551c0b4eb0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 --2- 172.21.15.28:0/2477065225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5538103e50 0x7f5538075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.540+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 >> 172.21.15.28:0/2477065225 conn(0x7f55380fefc0 msgr2=0x7f5538100750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.541+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 shutdown_connections 2024-05-20T20:08:01.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:01.541+0000 7f5537e00700 1 -- 172.21.15.28:0/2477065225 wait complete. 2024-05-20T20:08:01.543 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:08:01.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:01 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2477065225' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:02.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:01 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2477065225' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:02.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:01 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2477065225' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:03.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:02 smithi028 ceph-mon[31178]: pgmap v75: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:03.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:02 smithi154 ceph-mon[36314]: pgmap v75: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:03.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:02 smithi033 ceph-mon[36179]: pgmap v75: 97 pgs: 97 active+clean; 1.4 MiB data, 340 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:03.371 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 146.35 --await-for 45.0' 2024-05-20T20:08:03.372 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 146.35 --await-for 45.0 2024-05-20T20:08:03.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.588+0000 7f749f200700 1 -- 172.21.15.28:0/2171504501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a00758e0 msgr2=0x7f74a0075cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:03.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.588+0000 7f749f200700 1 --2- 172.21.15.28:0/2171504501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a00758e0 0x7f74a0075cc0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f7490009990 tx=0x7f7490009ca0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.588+0000 7f749f200700 1 -- 172.21.15.28:0/2171504501 shutdown_connections 2024-05-20T20:08:03.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.588+0000 7f749f200700 1 --2- 172.21.15.28:0/2171504501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a0114480 0x7f74a0116870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.588+0000 7f749f200700 1 --2- 172.21.15.28:0/2171504501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a00758e0 0x7f74a0075cc0 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.588+0000 7f749f200700 1 --2- 172.21.15.28:0/2171504501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f74a0076200 0x7f74a0113f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.588+0000 7f749f200700 1 -- 172.21.15.28:0/2171504501 >> 172.21.15.28:0/2171504501 conn(0x7f74a00fefc0 msgr2=0x7f74a01013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:03.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.589+0000 7f749f200700 1 -- 172.21.15.28:0/2171504501 shutdown_connections 2024-05-20T20:08:03.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.589+0000 7f749f200700 1 -- 172.21.15.28:0/2171504501 wait complete. 2024-05-20T20:08:03.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.590+0000 7f749f200700 1 Processor -- start 2024-05-20T20:08:03.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.590+0000 7f749f200700 1 -- start start 2024-05-20T20:08:03.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749f200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a00758e0 0x7f74a01a01a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:03.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749f200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a0076200 0x7f74a01a06e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:03.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749f200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f74a0114480 0x7f74a01a4a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:03.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749f200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f74a0119cb0 con 0x7f74a0076200 2024-05-20T20:08:03.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749f200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f74a0119b30 con 0x7f74a0114480 2024-05-20T20:08:03.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749f200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f74a0119e30 con 0x7f74a00758e0 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a00758e0 0x7f74a01a01a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a00758e0 0x7f74a01a01a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38430/0 (socket says 172.21.15.28:38430) 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749de00700 1 -- 172.21.15.28:0/1700785630 learned_addr learned my addr 172.21.15.28:0/1700785630 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749e800700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f74a0114480 0x7f74a01a4a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.591+0000 7f749d400700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a0076200 0x7f74a01a06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749de00700 1 -- 172.21.15.28:0/1700785630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f74a0114480 msgr2=0x7f74a01a4a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749de00700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f74a0114480 0x7f74a01a4a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749de00700 1 -- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a0076200 msgr2=0x7f74a01a06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:03.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749de00700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a0076200 0x7f74a01a06e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749de00700 1 -- 172.21.15.28:0/1700785630 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f74a01a5160 con 0x7f74a00758e0 2024-05-20T20:08:03.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749d400700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a0076200 0x7f74a01a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:03.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749de00700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a00758e0 0x7f74a01a01a0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f7490000c00 tx=0x7f74900057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:03.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f7496a00700 1 -- 172.21.15.28:0/1700785630 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7490027070 con 0x7f74a00758e0 2024-05-20T20:08:03.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f74a01a53f0 con 0x7f74a00758e0 2024-05-20T20:08:03.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f7496a00700 1 -- 172.21.15.28:0/1700785630 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7490022470 con 0x7f74a00758e0 2024-05-20T20:08:03.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.592+0000 7f7496a00700 1 -- 172.21.15.28:0/1700785630 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7490017650 con 0x7f74a00758e0 2024-05-20T20:08:03.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.593+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f74a01accd0 con 0x7f74a00758e0 2024-05-20T20:08:03.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.594+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f74a006a230 con 0x7f74a00758e0 2024-05-20T20:08:03.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.596+0000 7f7496a00700 1 -- 172.21.15.28:0/1700785630 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f749002a070 con 0x7f74a00758e0 2024-05-20T20:08:03.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.598+0000 7f7496a00700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74840b29f0 0x7f74840b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:03.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.598+0000 7f7496a00700 1 -- 172.21.15.28:0/1700785630 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f74900e33e0 con 0x7f74a00758e0 2024-05-20T20:08:03.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.598+0000 7f749d400700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74840b29f0 0x7f74840b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:03.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.599+0000 7f749d400700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74840b29f0 0x7f74840b4eb0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f74a0107270 tx=0x7f748c009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:03.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.602+0000 7f7496a00700 1 -- 172.21.15.28:0/1700785630 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7490013070 con 0x7f74a00758e0 2024-05-20T20:08:03.838 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:03.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.908+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f74a002e560 con 0x7f74a00758e0 2024-05-20T20:08:03.910 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:08:03.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.909+0000 7f7496a00700 1 -- 172.21.15.28:0/1700785630 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f74900a4e20 con 0x7f74a00758e0 2024-05-20T20:08:03.911 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:08:03.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74840b29f0 msgr2=0x7f74840b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:03.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74840b29f0 0x7f74840b4eb0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f74a0107270 tx=0x7f748c009260 comp rx=0 tx=0).stop 2024-05-20T20:08:03.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a00758e0 msgr2=0x7f74a01a01a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a00758e0 0x7f74a01a01a0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f7490000c00 tx=0x7f74900057c0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 shutdown_connections 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74840b29f0 0x7f74840b4eb0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f74a0114480 0x7f74a01a4a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f74a00758e0 0x7f74a01a01a0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 --2- 172.21.15.28:0/1700785630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f74a0076200 0x7f74a01a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 >> 172.21.15.28:0/1700785630 conn(0x7f74a00fefc0 msgr2=0x7f74a00ffcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 shutdown_connections 2024-05-20T20:08:03.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:03.912+0000 7f749f200700 1 -- 172.21.15.28:0/1700785630 wait complete. 2024-05-20T20:08:03.915 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:08:04.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:04 smithi028 ceph-mon[31178]: pgmap v76: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 22 KiB/s wr, 28 op/s 2024-05-20T20:08:04.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:04 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1700785630' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:05.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:04 smithi154 ceph-mon[36314]: pgmap v76: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 22 KiB/s wr, 28 op/s 2024-05-20T20:08:05.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:04 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1700785630' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:04 smithi033 ceph-mon[36179]: pgmap v76: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 22 KiB/s wr, 28 op/s 2024-05-20T20:08:05.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:04 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1700785630' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:05.588 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:08:06.058 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:06.100 INFO:tasks.workunit.client.0.smithi028.stderr:Updating files: 57% (7033/12271) Updating files: 58% (7118/12271) Updating files: 59% (7240/12271) Updating files: 60% (7363/12271) Updating files: 61% (7486/12271) Updating files: 62% (7609/12271) Updating files: 63% (7731/12271) Updating files: 64% (7854/12271) Updating files: 65% (7977/12271) Updating files: 66% (8099/12271) Updating files: 67% (8222/12271) Updating files: 68% (8345/12271) Updating files: 69% (8467/12271) Updating files: 70% (8590/12271) Updating files: 71% (8713/12271) Updating files: 72% (8836/12271) Updating files: 73% (8958/12271) Updating files: 74% (9081/12271) Updating files: 75% (9204/12271) Updating files: 76% (9326/12271) Updating files: 77% (9449/12271) Updating files: 78% (9572/12271) Updating files: 79% (9695/12271) Updating files: 80% (9817/12271) Updating files: 81% (9940/12271) Updating files: 82% (10063/12271) Updating files: 83% (10185/12271) Updating files: 84% (10308/12271) Updating files: 85% (10431/12271) Updating files: 86% (10554/12271) Updating files: 87% (10676/12271) Updating files: 88% (10799/12271) Updating files: 89% (10922/12271) Updating files: 90% (11044/12271) Updating files: 91% (11167/12271) Updating files: 91% (11270/12271) Updating files: 92% (11290/12271) Updating files: 93% (11413/12271) Updating files: 94% (11535/12271) Updating files: 95% (11658/12271) Updating files: 96% (11781/12271) Updating files: 97% (11903/12271) Updating files: 98% (12026/12271) Updating files: 99% (12149/12271) Updating files: 100% (12271/12271) Updating files: 100% (12271/12271), done. 2024-05-20T20:08:06.814 INFO:tasks.workunit.client.0.smithi028.stderr:Note: switching to '4ebd1d5361d457738957dc0455bba90102296634'. 2024-05-20T20:08:06.814 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:06.814 INFO:tasks.workunit.client.0.smithi028.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-05-20T20:08:06.814 INFO:tasks.workunit.client.0.smithi028.stderr:changes and commit them, and you can discard any commits you make in this 2024-05-20T20:08:06.814 INFO:tasks.workunit.client.0.smithi028.stderr:state without impacting any branches by switching back to a branch. 2024-05-20T20:08:06.814 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:06.814 INFO:tasks.workunit.client.0.smithi028.stderr:If you want to create a new branch to retain commits you create, you may 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr:do so (now or later) by using -c with the switch command. Example: 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr: git switch -c 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr:Or undo this operation with: 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr: git switch - 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:06.815 INFO:tasks.workunit.client.0.smithi028.stderr:HEAD is now at 4ebd1d5361d mds/queisce-db: collect acks while bootstrapping 2024-05-20T20:08:06.830 DEBUG:teuthology.orchestra.run.smithi028:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-05-20T20:08:06.924 INFO:tasks.workunit.client.0.smithi028.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-05-20T20:08:06.929 INFO:tasks.workunit.client.0.smithi028.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-05-20T20:08:06.929 INFO:tasks.workunit.client.0.smithi028.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-05-20T20:08:07.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:06 smithi028 ceph-mon[31178]: pgmap v77: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:07.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:06 smithi154 ceph-mon[36314]: pgmap v77: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:07.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:06 smithi033 ceph-mon[36179]: pgmap v77: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:07.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 -- 172.21.15.28:0/1246932102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 msgr2=0x7f5594103c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 --2- 172.21.15.28:0/1246932102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f5594103c30 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f55880099c0 tx=0x7f5588009cd0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 -- 172.21.15.28:0/1246932102 shutdown_connections 2024-05-20T20:08:07.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 --2- 172.21.15.28:0/1246932102 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5594108840 0x7f5594110450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 --2- 172.21.15.28:0/1246932102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f5594103c30 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 --2- 172.21.15.28:0/1246932102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5594104200 0x7f5594108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 -- 172.21.15.28:0/1246932102 >> 172.21.15.28:0/1246932102 conn(0x7f559407a610 msgr2=0x7f559407aa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.902+0000 7f559b600700 1 -- 172.21.15.28:0/1246932102 shutdown_connections 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.903+0000 7f559b600700 1 -- 172.21.15.28:0/1246932102 wait complete. 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.904+0000 7f559b600700 1 Processor -- start 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.904+0000 7f559b600700 1 -- start start 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.904+0000 7f559b600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f559419f7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.904+0000 7f559b600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5594104200 0x7f559419fd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f559b600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5594108840 0x7f5594199990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f559b600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5594113730 con 0x7f5594103850 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f559b600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f55941135b0 con 0x7f5594108840 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f559b600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f55941138b0 con 0x7f5594104200 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f559419f7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f559419f7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60574/0 (socket says 172.21.15.28:60574) 2024-05-20T20:08:07.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 -- 172.21.15.28:0/1903942129 learned_addr learned my addr 172.21.15.28:0/1903942129 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:07.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5593e00700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5594104200 0x7f559419fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5599600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5594108840 0x7f5594199990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 -- 172.21.15.28:0/1903942129 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5594104200 msgr2=0x7f559419fd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5594104200 0x7f559419fd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 -- 172.21.15.28:0/1903942129 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5594108840 msgr2=0x7f5594199990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5594108840 0x7f5594199990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.905+0000 7f5598c00700 1 -- 172.21.15.28:0/1903942129 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f559419a0f0 con 0x7f5594103850 2024-05-20T20:08:07.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.906+0000 7f5598c00700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f559419f7d0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f5588006010 tx=0x7f55880055f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.906+0000 7f5591600700 1 -- 172.21.15.28:0/1903942129 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5588026070 con 0x7f5594103850 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.906+0000 7f5591600700 1 -- 172.21.15.28:0/1903942129 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f558800f460 con 0x7f5594103850 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.906+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f559419a380 con 0x7f5594103850 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.906+0000 7f5591600700 1 -- 172.21.15.28:0/1903942129 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5588009e60 con 0x7f5594103850 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.906+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f55941a66a0 con 0x7f5594103850 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 Processor -- start 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 -- start start 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b908f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b834f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b98260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c867ae0fe0 con 0x55c867b90510 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c867ad0b80 con 0x55c867b83a30 2024-05-20T20:08:07.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.911+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c867ad4bc0 con 0x55c867b83070 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5591600700 1 -- 172.21.15.28:0/1903942129 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f5588029070 con 0x7f5594103850 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b908f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b908f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60578/0 (socket says 172.21.15.28:60578) 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 -- 172.21.15.28:0/2754881969 learned_addr learned my addr 172.21.15.28:0/2754881969 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5593e00700 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b98260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5598c00700 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b834f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 -- 172.21.15.28:0/2754881969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 msgr2=0x55c867b834f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b834f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 -- 172.21.15.28:0/2754881969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 msgr2=0x55c867b98260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b98260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.912+0000 7f5599600700 1 -- 172.21.15.28:0/2754881969 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c867a995c0 con 0x55c867b90510 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.913+0000 7f5599600700 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b908f0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f558c009440 tx=0x7f558c009920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aaa46319723d6e19 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.913+0000 7f5591600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f557c0b2aa0 0x7f557c0b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.913+0000 7f5586000700 1 -- 172.21.15.28:0/2754881969 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f558c013810 con 0x55c867b90510 2024-05-20T20:08:07.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.913+0000 7f5586000700 1 -- 172.21.15.28:0/2754881969 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f558c015440 con 0x55c867b90510 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.913+0000 7f5586000700 1 -- 172.21.15.28:0/2754881969 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f558c013e60 con 0x55c867b90510 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.913+0000 7f5591600700 1 -- 172.21.15.28:0/1903942129 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f55880e2f30 con 0x7f5594103850 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.913+0000 7f5598c00700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f557c0b2aa0 0x7f557c0b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f5598c00700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f557c0b2aa0 0x7f557c0b4f60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x55c867b84230 tx=0x7f558800ff90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2754881969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 msgr2=0x55c867b908f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b908f0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f558c009440 tx=0x7f558c009920 comp rx=0 tx=0).stop 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2754881969 shutdown_connections 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b98260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b908f0 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2754881969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b834f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2754881969 >> 172.21.15.28:0/2754881969 conn(0x55c867ab0d80 msgr2=0x55c867b7e060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.914+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2754881969 shutdown_connections 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.915+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2754881969 wait complete. 2024-05-20T20:08:07.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.915+0000 7f55a5ee3b80 1 -- start start 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 1 Processor -- start 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b97e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b95200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b95740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c8679d7140 con 0x55c867b90510 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c867a71900 con 0x55c867b83a30 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c867b98950 con 0x55c867b83070 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.916+0000 7f55a5ee3b80 10 client.0 _ll_register_callbacks cb 0x55c867a7ae10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5599600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b95740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5599600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b95740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60586/0 (socket says 172.21.15.28:60586) 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5599600700 1 -- 172.21.15.28:0/562427269 learned_addr learned my addr 172.21.15.28:0/562427269 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5593e00700 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b95200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5599600700 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b97e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5593e00700 1 -- 172.21.15.28:0/562427269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 msgr2=0x55c867b97e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5593e00700 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b97e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5593e00700 1 -- 172.21.15.28:0/562427269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 msgr2=0x55c867b95740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5593e00700 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b95740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.917+0000 7f5593e00700 1 -- 172.21.15.28:0/562427269 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c867a20a70 con 0x55c867b83a30 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f5593e00700 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b95200 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f5594109d40 tx=0x7f558000bbf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2628b844de10fbbd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f557aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f557aa00700 1 -- 172.21.15.28:0/562427269 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5580003b50 con 0x55c867b83a30 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f558c015440 con 0x55c867b83a30 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f557aa00700 1 -- 172.21.15.28:0/562427269 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5580003cf0 con 0x55c867b83a30 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f557aa00700 1 -- 172.21.15.28:0/562427269 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5580004830 con 0x55c867b83a30 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f55a5ee3b80 10 client.25949 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.918+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c867aa1200 con 0x55c867b83a30 2024-05-20T20:08:07.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f557aa00700 1 -- 172.21.15.28:0/562427269 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f558c015440 con 0x55c867b83a30 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f557aa00700 1 -- 172.21.15.28:0/562427269 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5580011440 con 0x55c867b83a30 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f55a5ee3b80 10 client.25949 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f55a5ee3b80 10 client.25949 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f55a5ee3b80 20 client.25949 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867ba5490 0x55c867ba7880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f55a5ee3b80 4 client.25949 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c867ae2f30 con 0x55c867ba5490 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.919+0000 7f5599600700 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867ba5490 0x55c867ba7880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.920+0000 7f5599600700 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867ba5490 0x55c867ba7880 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=892380e2cf39812f server_cookie=94f2dd6a5583e49d in_seq=0 out_seq=0 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.920+0000 7f557aa00700 10 client.25949 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f557aa00700 1 -- 172.21.15.28:0/562427269 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55c867ae2f30 con 0x55c867ba5490 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f557aa00700 10 client.25949 handle_command_reply: tid=0 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 client.25949 shutdown 2024-05-20T20:08:07.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 msgr2=0x55c867b95200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b95200 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f5594109d40 tx=0x7f558000bbf0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 shutdown_connections 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867ba5490 0x55c867ba7880 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867b95740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83070 0x55c867b97e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/562427269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83a30 0x55c867b95200 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f557aa00700 0 client.25949 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 >> 172.21.15.28:0/562427269 conn(0x55c867ab0d80 msgr2=0x55c867b7dd60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 shutdown_connections 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/562427269 wait complete. 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.921+0000 7f55a5ee3b80 20 client.25949 trim_cache size 0 max 16384 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 Processor -- start 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 -- start start 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83070 0x55c867b82b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867bad130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867bad670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f55941abae0 con 0x55c867b90510 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f55941ad620 con 0x55c867b83070 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.944+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f55941ac1b0 con 0x55c867b83a30 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867bad670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83070 0x55c867b82b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5593e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867bad130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83070 0x55c867b82b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56160/0 (socket says 172.21.15.28:56160) 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 -- 172.21.15.28:0/2251448283 learned_addr learned my addr 172.21.15.28:0/2251448283 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 -- 172.21.15.28:0/2251448283 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 msgr2=0x55c867bad130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 --2- 172.21.15.28:0/2251448283 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867bad130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 -- 172.21.15.28:0/2251448283 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 msgr2=0x55c867bad670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 --2- 172.21.15.28:0/2251448283 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867bad670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 -- 172.21.15.28:0/2251448283 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c867b7eb30 con 0x55c867b83070 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.945+0000 7f5599600700 1 --2- 172.21.15.28:0/2251448283 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83070 0x55c867b82b30 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f558c015d20 tx=0x7f558c0094d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ff8e57efb62a5bf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.946+0000 7f5585600700 1 -- 172.21.15.28:0/2251448283 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f558c015730 con 0x55c867b83070 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.946+0000 7f5585600700 1 -- 172.21.15.28:0/2251448283 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f558c0158d0 con 0x55c867b83070 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.946+0000 7f5585600700 1 -- 172.21.15.28:0/2251448283 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f558c013dd0 con 0x55c867b83070 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2251448283 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83070 msgr2=0x55c867b82b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2251448283 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83070 0x55c867b82b30 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f558c015d20 tx=0x7f558c0094d0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2251448283 shutdown_connections 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2251448283 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b90510 0x55c867bad670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2251448283 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b83070 0x55c867b82b30 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/2251448283 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867bad130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2251448283 >> 172.21.15.28:0/2251448283 conn(0x55c867ab0d80 msgr2=0x55c867b8f930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:07.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2251448283 shutdown_connections 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/2251448283 wait complete. 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.947+0000 7f55a5ee3b80 1 -- start start 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.948+0000 7f55a5ee3b80 1 Processor -- start 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.948+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b83070 0x55c867b9ab70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.948+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867b9b0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f55a5ee3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b90510 0x55c867b9b5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f55941acf50 con 0x55c867b83070 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f55941ac880 con 0x55c867b90510 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f55a5ee3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f55941a9a80 con 0x55c867b83a30 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f55a5ee3b80 10 client.0 _ll_register_callbacks cb 0x55c867a7ae10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b83070 0x55c867b9ab70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5593e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867b9b0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b83070 0x55c867b9ab70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60606/0 (socket says 172.21.15.28:60606) 2024-05-20T20:08:07.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 -- 172.21.15.28:0/107035189 learned_addr learned my addr 172.21.15.28:0/107035189 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b90510 0x55c867b9b5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 -- 172.21.15.28:0/107035189 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 msgr2=0x55c867b9b0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867b9b0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 -- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b90510 msgr2=0x55c867b9b5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b90510 0x55c867b9b5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.949+0000 7f5599600700 1 -- 172.21.15.28:0/107035189 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c867ba2f10 con 0x55c867b83070 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f5599600700 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b83070 0x55c867b9ab70 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f558c009440 tx=0x7f558c015db0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4a8ee2232b069a31 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f557aa00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f557aa00700 1 -- 172.21.15.28:0/107035189 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f558c013880 con 0x55c867b83070 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f557aa00700 1 -- 172.21.15.28:0/107035189 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f558c004c90 con 0x55c867b83070 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f558c0158d0 con 0x55c867b83070 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f557aa00700 1 -- 172.21.15.28:0/107035189 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f558c03e050 con 0x55c867b83070 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f55a5ee3b80 10 client.15810 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:07.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.950+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c867c11310 con 0x55c867b83070 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f557aa00700 1 -- 172.21.15.28:0/107035189 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f558c0158d0 con 0x55c867b83070 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f557aa00700 1 -- 172.21.15.28:0/107035189 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f558c02e900 con 0x55c867b83070 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f55a5ee3b80 10 client.15810 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f55a5ee3b80 10 client.15810 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f55a5ee3b80 20 client.15810 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867c11780 0x55c867c13b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f55a5ee3b80 4 client.15810 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 146.35, "await_for": 45.0}] 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 146.35, "await_for": 45.0}) -- 0x55c867adc710 con 0x55c867c11780 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.951+0000 7f5593e00700 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867c11780 0x55c867c13b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.952+0000 7f557aa00700 10 client.15810 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.952+0000 7f5593e00700 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867c11780 0x55c867c13b70 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=536157e16a1d27cf server_cookie=28d17cab7f5a793e in_seq=0 out_seq=0 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f557aa00700 1 -- 172.21.15.28:0/107035189 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+589 (crc 0 0 0) 0x55c867adc710 con 0x55c867c11780 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f557aa00700 10 client.15810 handle_command_reply: tid=0 2024-05-20T20:08:07.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 client.15810 shutdown 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b83070 msgr2=0x55c867b9ab70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b83070 0x55c867b9ab70 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f558c009440 tx=0x7f558c015db0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 shutdown_connections 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c867c11780 0x55c867c13b70 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c867b90510 0x55c867b9b5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c867b83070 0x55c867b9ab70 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 --2- 172.21.15.28:0/107035189 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c867b83a30 0x55c867b9b0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:08:07.538 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 6, 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "a49aa3fd": { 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 6, 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 146.3, 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:08:07.539 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:07.540 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 >> 172.21.15.28:0/107035189 conn(0x55c867ab0d80 msgr2=0x55c867b96840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.963+0000 7f557aa00700 0 client.15810 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.964+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 shutdown_connections 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.964+0000 7f55a5ee3b80 1 -- 172.21.15.28:0/107035189 wait complete. 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.964+0000 7f55a5ee3b80 20 client.15810 trim_cache size 0 max 16384 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f557c0b2aa0 msgr2=0x7f557c0b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f557c0b2aa0 0x7f557c0b4f60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x55c867b84230 tx=0x7f558800ff90 comp rx=0 tx=0).stop 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 msgr2=0x7f559419f7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f559419f7d0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f5588006010 tx=0x7f55880055f0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 shutdown_connections 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f557c0b2aa0 0x7f557c0b4f60 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5594108840 0x7f5594199990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5594103850 0x7f559419f7d0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 --2- 172.21.15.28:0/1903942129 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5594104200 0x7f559419fd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:07.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 >> 172.21.15.28:0/1903942129 conn(0x7f559407a610 msgr2=0x7f55941015c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:07.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 shutdown_connections 2024-05-20T20:08:07.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:06.965+0000 7f559b600700 1 -- 172.21.15.28:0/1903942129 wait complete. 2024-05-20T20:08:08.536 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set 'a49aa3fd', quiesce took 11.8 seconds. Will release after: 14.55 2024-05-20T20:08:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:08 smithi028 ceph-mon[31178]: pgmap v78: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:08 smithi154 ceph-mon[36314]: pgmap v78: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:09.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:08 smithi033 ceph-mon[36179]: pgmap v78: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:09.260 INFO:tasks.workunit.client.0.smithi028.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-05-20T20:08:09.363 INFO:tasks.workunit.client.0.smithi028.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-05-20T20:08:09.445 INFO:tasks.workunit.client.0.smithi028.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-05-20T20:08:09.451 INFO:tasks.workunit.client.0.smithi028.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-05-20T20:08:09.451 INFO:tasks.workunit.client.0.smithi028.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-05-20T20:08:09.533 INFO:tasks.workunit.client.0.smithi028.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-05-20T20:08:09.542 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T20:08:09.543 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-05-20T20:08:09.593 INFO:tasks.workunit:Running workunits matching suites/iogen.sh on client.0... 2024-05-20T20:08:09.594 INFO:tasks.workunit:Running workunit suites/iogen.sh... 2024-05-20T20:08:09.594 DEBUG:teuthology.orchestra.run.smithi028:workunit test suites/iogen.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=4ebd1d5361d457738957dc0455bba90102296634 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/iogen.sh 2024-05-20T20:08:09.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 -- 172.21.15.28:0/2954991734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01037f0 msgr2=0x7f89a0103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 --2- 172.21.15.28:0/2954991734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01037f0 0x7f89a0103bd0 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f899000b0d0 tx=0x7f899000b3e0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 -- 172.21.15.28:0/2954991734 shutdown_connections 2024-05-20T20:08:09.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 --2- 172.21.15.28:0/2954991734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 0x7f89a0110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 --2- 172.21.15.28:0/2954991734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01037f0 0x7f89a0103bd0 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 --2- 172.21.15.28:0/2954991734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89a01041a0 0x7f89a01081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 -- 172.21.15.28:0/2954991734 >> 172.21.15.28:0/2954991734 conn(0x7f89a007a6a0 msgr2=0x7f89a007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:09.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 -- 172.21.15.28:0/2954991734 shutdown_connections 2024-05-20T20:08:09.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.934+0000 7f89a7400700 1 -- 172.21.15.28:0/2954991734 wait complete. 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.935+0000 7f89a7400700 1 Processor -- start 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.935+0000 7f89a7400700 1 -- start start 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89a01037f0 0x7f89a019f840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01041a0 0x7f89a019fd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 0x7f89a0199a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a7400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f89a0113730 con 0x7f89a01041a0 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a7400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01135b0 con 0x7f89a01088f0 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a7400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01138b0 con 0x7f89a01037f0 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89a01037f0 0x7f89a019f840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 0x7f89a0199a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f8997e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01041a0 0x7f89a019fd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 0x7f89a0199a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56190/0 (socket says 172.21.15.28:56190) 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89a01037f0 0x7f89a019f840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57050/0 (socket says 172.21.15.28:57050) 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.936+0000 7f89a5400700 1 -- 172.21.15.28:0/4216568397 learned_addr learned my addr 172.21.15.28:0/4216568397 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f89a5400700 1 -- 172.21.15.28:0/4216568397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89a01037f0 msgr2=0x7f89a019f840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f89a5400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89a01037f0 0x7f89a019f840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f89a5400700 1 -- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01041a0 msgr2=0x7f89a019fd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f89a5400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01041a0 0x7f89a019fd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f89a5400700 1 -- 172.21.15.28:0/4216568397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f89a019a1b0 con 0x7f89a01088f0 2024-05-20T20:08:09.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f89a5400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 0x7f89a0199a50 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f899800ea80 tx=0x7f899800ed90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:09.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f899e000700 1 -- 172.21.15.28:0/4216568397 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8998004220 con 0x7f89a01088f0 2024-05-20T20:08:09.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f899e000700 1 -- 172.21.15.28:0/4216568397 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f899800ce00 con 0x7f89a01088f0 2024-05-20T20:08:09.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f899e000700 1 -- 172.21.15.28:0/4216568397 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8998003d30 con 0x7f89a01088f0 2024-05-20T20:08:09.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.937+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f89a019a4a0 con 0x7f89a01088f0 2024-05-20T20:08:09.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.938+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f89a0075770 con 0x7f89a01088f0 2024-05-20T20:08:09.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.941+0000 7f899e000700 1 -- 172.21.15.28:0/4216568397 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f89980043c0 con 0x7f89a01088f0 2024-05-20T20:08:09.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.942+0000 7f89b1e07b80 1 Processor -- start 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.942+0000 7f899e000700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89840b2a70 0x7f89840b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.942+0000 7f89b1e07b80 1 -- start start 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f899e000700 1 -- 172.21.15.28:0/4216568397 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f89980e2870 con 0x7f89a01088f0 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f8997e00700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89840b2a70 0x7f89840b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db489f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db48f30 0x56205db42b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db43140 0x56205db435c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56205daa0e40 con 0x56205db48f30 2024-05-20T20:08:09.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56205da909e0 con 0x56205db43140 2024-05-20T20:08:09.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56205da94a20 con 0x56205db48610 2024-05-20T20:08:09.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f8997e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db48f30 0x56205db42b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db489f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db489f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57064/0 (socket says 172.21.15.28:57064) 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.943+0000 7f89a4a00700 1 -- 172.21.15.28:0/49594948 learned_addr learned my addr 172.21.15.28:0/49594948 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f89a5400700 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db43140 0x56205db435c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f89a4a00700 1 -- 172.21.15.28:0/49594948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db43140 msgr2=0x56205db435c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f89a4a00700 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db43140 0x56205db435c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f89a4a00700 1 -- 172.21.15.28:0/49594948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db48f30 msgr2=0x56205db42b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f89a4a00700 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db48f30 0x56205db42b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f89a4a00700 1 -- 172.21.15.28:0/49594948 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56205da59420 con 0x56205db48610 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f8997e00700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89840b2a70 0x7f89840b4f30 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f89a0103110 tx=0x7f8988013900 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.944+0000 7f89a4a00700 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db489f0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f8990015000 tx=0x7f899000b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=829e61d24dd90e3a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:09.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.945+0000 7f8996000700 1 -- 172.21.15.28:0/49594948 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f899000baa0 con 0x56205db48610 2024-05-20T20:08:09.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.945+0000 7f8996000700 1 -- 172.21.15.28:0/49594948 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f899000bc40 con 0x56205db48610 2024-05-20T20:08:09.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.945+0000 7f8996000700 1 -- 172.21.15.28:0/49594948 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f899001cbc0 con 0x56205db48610 2024-05-20T20:08:09.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 -- 172.21.15.28:0/49594948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 msgr2=0x56205db489f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db489f0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f8990015000 tx=0x7f899000b040 comp rx=0 tx=0).stop 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 -- 172.21.15.28:0/49594948 shutdown_connections 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db43140 0x56205db435c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db489f0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/49594948 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db48f30 0x56205db42b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 -- 172.21.15.28:0/49594948 >> 172.21.15.28:0/49594948 conn(0x56205da70be0 msgr2=0x56205db3d690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 -- 172.21.15.28:0/49594948 shutdown_connections 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.946+0000 7f89b1e07b80 1 -- 172.21.15.28:0/49594948 wait complete. 2024-05-20T20:08:09.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.947+0000 7f89b1e07b80 1 -- start start 2024-05-20T20:08:09.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.947+0000 7f89b1e07b80 1 Processor -- start 2024-05-20T20:08:09.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db4ec10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db4c000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48f30 0x56205db4c540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56205d996fa0 con 0x56205db43140 2024-05-20T20:08:09.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56205da31760 con 0x56205db48f30 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56205db4f920 con 0x56205db48610 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89b1e07b80 10 client.0 _ll_register_callbacks cb 0x56205da3ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f8997e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db4c000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db4ec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f8997e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db4c000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57074/0 (socket says 172.21.15.28:57074) 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db4ec10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60630/0 (socket says 172.21.15.28:60630) 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f8997e00700 1 -- 172.21.15.28:0/1803707454 learned_addr learned my addr 172.21.15.28:0/1803707454 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:09.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f89a5400700 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48f30 0x56205db4c540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.948+0000 7f8997e00700 1 -- 172.21.15.28:0/1803707454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48f30 msgr2=0x56205db4c540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f8997e00700 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48f30 0x56205db4c540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f8997e00700 1 -- 172.21.15.28:0/1803707454 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 msgr2=0x56205db4ec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f8997e00700 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db4ec10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f8997e00700 1 -- 172.21.15.28:0/1803707454 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56205d9e08d0 con 0x56205db48610 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f89a4a00700 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db4ec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f8997e00700 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db4c000 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f8988010710 tx=0x7f89880047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=946689eede3b8fda server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f898ea00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:09.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f898ea00700 1 -- 172.21.15.28:0/1803707454 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8988024070 con 0x56205db48610 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f898ea00700 1 -- 172.21.15.28:0/1803707454 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8988020840 con 0x56205db48610 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.949+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f899000bc40 con 0x56205db48610 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.950+0000 7f898ea00700 1 -- 172.21.15.28:0/1803707454 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8988020a40 con 0x56205db48610 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.950+0000 7f89b1e07b80 10 client.25918 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.950+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56205da61060 con 0x56205db48610 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.950+0000 7f898ea00700 1 -- 172.21.15.28:0/1803707454 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f899000bc40 con 0x56205db48610 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.950+0000 7f898ea00700 1 -- 172.21.15.28:0/1803707454 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f898801f070 con 0x56205db48610 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.951+0000 7f89b1e07b80 10 client.25918 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.951+0000 7f89b1e07b80 10 client.25918 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.951+0000 7f89b1e07b80 20 client.25918 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.951+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205db5c410 0x56205db5e800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.951+0000 7f89b1e07b80 4 client.25918 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:09.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.951+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56205daa2d90 con 0x56205db5c410 2024-05-20T20:08:09.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.951+0000 7f89a4a00700 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205db5c410 0x56205db5e800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.952+0000 7f89a4a00700 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205db5c410 0x56205db5e800 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=2ca6e1dfcdd39bda server_cookie=81b92ad840753b0b in_seq=0 out_seq=0 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.952+0000 7f898ea00700 10 client.25918 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f898ea00700 1 -- 172.21.15.28:0/1803707454 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56205daa2d90 con 0x56205db5c410 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f898ea00700 10 client.25918 handle_command_reply: tid=0 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 client.25918 shutdown 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 msgr2=0x56205db4c000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db4c000 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f8988010710 tx=0x7f89880047c0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 shutdown_connections 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48f30 0x56205db4c540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db4ec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205db5c410 0x56205db5e800 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/1803707454 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48610 0x56205db4c000 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 >> 172.21.15.28:0/1803707454 conn(0x56205da70be0 msgr2=0x56205db3cd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f898ea00700 0 client.25918 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 shutdown_connections 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.953+0000 7f89b1e07b80 1 -- 172.21.15.28:0/1803707454 wait complete. 2024-05-20T20:08:09.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.954+0000 7f89b1e07b80 20 client.25918 trim_cache size 0 max 16384 2024-05-20T20:08:09.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.970+0000 7f89b1e07b80 1 Processor -- start 2024-05-20T20:08:09.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.970+0000 7f89b1e07b80 1 -- start start 2024-05-20T20:08:09.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db42b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db49a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db3d4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01abae0 con 0x56205db43140 2024-05-20T20:08:09.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01ad620 con 0x56205db48610 2024-05-20T20:08:09.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01ac1b0 con 0x56205db48f30 2024-05-20T20:08:09.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db42b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db42b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60632/0 (socket says 172.21.15.28:60632) 2024-05-20T20:08:09.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89a4a00700 1 -- 172.21.15.28:0/3465167879 learned_addr learned my addr 172.21.15.28:0/3465167879 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:09.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f89a5400700 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db3d4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.971+0000 7f8997e00700 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db49a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8997e00700 1 -- 172.21.15.28:0/3465167879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 msgr2=0x56205db3d4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8997e00700 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db3d4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8997e00700 1 -- 172.21.15.28:0/3465167879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 msgr2=0x56205db42b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8997e00700 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db42b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8997e00700 1 -- 172.21.15.28:0/3465167879 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56205db4a570 con 0x56205db48610 2024-05-20T20:08:09.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f89a4a00700 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db42b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:09.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8997e00700 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db49a20 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f898802ad00 tx=0x7f8988009260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bbf7a5510f310a69 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8995600700 1 -- 172.21.15.28:0/3465167879 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8988024040 con 0x56205db48610 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8995600700 1 -- 172.21.15.28:0/3465167879 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f898802a8c0 con 0x56205db48610 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.972+0000 7f8995600700 1 -- 172.21.15.28:0/3465167879 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8988020e40 con 0x56205db48610 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 -- 172.21.15.28:0/3465167879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 msgr2=0x56205db49a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db49a20 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f898802ad00 tx=0x7f8988009260 comp rx=0 tx=0).stop 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 -- 172.21.15.28:0/3465167879 shutdown_connections 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db3d4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db42b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/3465167879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db49a20 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 -- 172.21.15.28:0/3465167879 >> 172.21.15.28:0/3465167879 conn(0x56205da70be0 msgr2=0x56205db49310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 -- 172.21.15.28:0/3465167879 shutdown_connections 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.973+0000 7f89b1e07b80 1 -- 172.21.15.28:0/3465167879 wait complete. 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 -- start start 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 Processor -- start 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db55390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db63aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db640b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01acf50 con 0x56205db43140 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01ac880 con 0x56205db48610 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f89a01a9aa0 con 0x56205db48f30 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.974+0000 7f89b1e07b80 10 client.0 _ll_register_callbacks cb 0x56205da3ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db55390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a4a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db55390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60636/0 (socket says 172.21.15.28:60636) 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a4a00700 1 -- 172.21.15.28:0/2347629991 learned_addr learned my addr 172.21.15.28:0/2347629991 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f8997e00700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db63aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a5400700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db640b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a5400700 1 -- 172.21.15.28:0/2347629991 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 msgr2=0x56205db63aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a5400700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db63aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a5400700 1 -- 172.21.15.28:0/2347629991 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 msgr2=0x56205db55390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a5400700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db55390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a5400700 1 -- 172.21.15.28:0/2347629991 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56205dbc5fc0 con 0x56205db48f30 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.975+0000 7f89a4a00700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db55390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f89a5400700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db640b0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f89980dff70 tx=0x7f8998016560 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ac1a7563c9ba96fc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f898ea00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:09.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f898ea00700 1 -- 172.21.15.28:0/2347629991 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89980b22a0 con 0x56205db48f30 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f898ea00700 1 -- 172.21.15.28:0/2347629991 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f89980b26e0 con 0x56205db48f30 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f898802a8c0 con 0x56205db48f30 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f898ea00700 1 -- 172.21.15.28:0/2347629991 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f89980a9610 con 0x56205db48f30 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f89b1e07b80 10 client.25924 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.976+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56205dbc7160 con 0x56205db48f30 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f898ea00700 1 -- 172.21.15.28:0/2347629991 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f898802a8c0 con 0x56205db48f30 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f898ea00700 1 -- 172.21.15.28:0/2347629991 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f89980a8d60 con 0x56205db48f30 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f89b1e07b80 10 client.25924 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f89b1e07b80 10 client.25924 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f89b1e07b80 20 client.25924 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205dbc7810 0x56205dbc9c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f89b1e07b80 4 client.25924 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:08:09.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56205da9c570 con 0x56205dbc7810 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.977+0000 7f89a4a00700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205dbc7810 0x56205dbc9c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89a4a00700 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205dbc7810 0x56205dbc9c00 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=f00d61ecbcfe1c3b server_cookie=f3ae5415a56f1ace in_seq=0 out_seq=0 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f898ea00700 10 client.25924 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f898ea00700 1 -- 172.21.15.28:0/2347629991 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56205da9c570 con 0x56205dbc7810 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f898ea00700 10 client.25924 handle_command_reply: tid=0 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 client.25924 shutdown 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 msgr2=0x56205db640b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db640b0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f89980dff70 tx=0x7f8998016560 comp rx=0 tx=0).stop 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 shutdown_connections 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56205dbc7810 0x56205dbc9c00 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56205db48f30 0x56205db640b0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f898ea00700 0 client.25924 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56205db43140 0x56205db55390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 --2- 172.21.15.28:0/2347629991 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56205db48610 0x56205db63aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 >> 172.21.15.28:0/2347629991 conn(0x56205da70be0 msgr2=0x56205db470f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:09.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.978+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 shutdown_connections 2024-05-20T20:08:09.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.979+0000 7f89b1e07b80 1 -- 172.21.15.28:0/2347629991 wait complete. 2024-05-20T20:08:09.985 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.979+0000 7f89b1e07b80 20 client.25924 trim_cache size 0 max 16384 2024-05-20T20:08:09.985 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:08:09.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.979+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89840b2a70 msgr2=0x7f89840b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.979+0000 7f89a7400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89840b2a70 0x7f89840b4f30 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f89a0103110 tx=0x7f8988013900 comp rx=0 tx=0).stop 2024-05-20T20:08:09.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.979+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 msgr2=0x7f89a0199a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:09.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.979+0000 7f89a7400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 0x7f89a0199a50 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f899800ea80 tx=0x7f899800ed90 comp rx=0 tx=0).stop 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 shutdown_connections 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89840b2a70 0x7f89840b4f30 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89a01088f0 0x7f89a0199a50 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89a01037f0 0x7f89a019f840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 --2- 172.21.15.28:0/4216568397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89a01041a0 0x7f89a019fd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 >> 172.21.15.28:0/4216568397 conn(0x7f89a007a6a0 msgr2=0x7f89a0109a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 shutdown_connections 2024-05-20T20:08:09.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:09.980+0000 7f89a7400700 1 -- 172.21.15.28:0/4216568397 wait complete. 2024-05-20T20:08:10.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 -- 172.21.15.28:0/1041939470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4108870 msgr2=0x7fb6c4108c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 --2- 172.21.15.28:0/1041939470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4108870 0x7fb6c4108c50 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fb6b8009890 tx=0x7fb6b802a360 comp rx=0 tx=0).stop 2024-05-20T20:08:10.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 -- 172.21.15.28:0/1041939470 shutdown_connections 2024-05-20T20:08:10.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 --2- 172.21.15.28:0/1041939470 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb6c4102860 0x7fb6c4102ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 --2- 172.21.15.28:0/1041939470 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4103220 0x7fb6c410f730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 --2- 172.21.15.28:0/1041939470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4108870 0x7fb6c4108c50 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 -- 172.21.15.28:0/1041939470 >> 172.21.15.28:0/1041939470 conn(0x7fb6c40fe3d0 msgr2=0x7fb6c41007f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:10.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 -- 172.21.15.28:0/1041939470 shutdown_connections 2024-05-20T20:08:10.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.527+0000 7fb6cb200640 1 -- 172.21.15.28:0/1041939470 wait complete. 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.528+0000 7fb6cb200640 1 Processor -- start 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.528+0000 7fb6cb200640 1 -- start start 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6cb200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4102860 0x7fb6c4199100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6cb200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb6c4103220 0x7fb6c4199640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6cb200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4108870 0x7fb6c419d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6cb200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c4112b30 con 0x7fb6c4108870 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6cb200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c41129b0 con 0x7fb6c4103220 2024-05-20T20:08:10.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6cb200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c4112cb0 con 0x7fb6c4102860 2024-05-20T20:08:10.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6ca800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4108870 0x7fb6c419d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6ca800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4108870 0x7fb6c419d9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60656/0 (socket says 172.21.15.28:60656) 2024-05-20T20:08:10.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6c9400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb6c4103220 0x7fb6c4199640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6ca800640 1 -- 172.21.15.28:0/3062014292 learned_addr learned my addr 172.21.15.28:0/3062014292 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:10.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4102860 0x7fb6c4199100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6ca800640 1 -- 172.21.15.28:0/3062014292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4102860 msgr2=0x7fb6c4199100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6ca800640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4102860 0x7fb6c4199100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6ca800640 1 -- 172.21.15.28:0/3062014292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb6c4103220 msgr2=0x7fb6c4199640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.529+0000 7fb6ca800640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb6c4103220 0x7fb6c4199640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.530+0000 7fb6ca800640 1 -- 172.21.15.28:0/3062014292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb6c419e0f0 con 0x7fb6c4108870 2024-05-20T20:08:10.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.530+0000 7fb6ca800640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4108870 0x7fb6c419d9c0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fb6c000efc0 tx=0x7fb6c000c800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:10.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.530+0000 7fb6bea00640 1 -- 172.21.15.28:0/3062014292 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6c0003970 con 0x7fb6c4108870 2024-05-20T20:08:10.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.530+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb6c419e3e0 con 0x7fb6c4108870 2024-05-20T20:08:10.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.530+0000 7fb6bea00640 1 -- 172.21.15.28:0/3062014292 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb6c0004020 con 0x7fb6c4108870 2024-05-20T20:08:10.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.530+0000 7fb6bea00640 1 -- 172.21.15.28:0/3062014292 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6c000b940 con 0x7fb6c4108870 2024-05-20T20:08:10.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.530+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb6c41a5c70 con 0x7fb6c4108870 2024-05-20T20:08:10.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.534+0000 7fb6cf5dc740 1 Processor -- start 2024-05-20T20:08:10.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.534+0000 7fb6bea00640 1 -- 172.21.15.28:0/3062014292 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb6c001d020 con 0x7fb6c4108870 2024-05-20T20:08:10.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.534+0000 7fb6cf5dc740 1 -- start start 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.534+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f0e50 0x5654630f1230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.534+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630fbaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.534+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x5654630fe420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x565463061b90 con 0x5654630f0e50 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x565463051730 con 0x5654630fc030 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x565463055770 con 0x5654630f1770 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f0e50 0x5654630f1230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f0e50 0x5654630f1230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60662/0 (socket says 172.21.15.28:60662) 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9400640 1 -- 172.21.15.28:0/508369782 learned_addr learned my addr 172.21.15.28:0/508369782 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:10.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630fbaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9400640 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x5654630fe420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9e00640 1 -- 172.21.15.28:0/508369782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 msgr2=0x5654630fe420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6bea00640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb6ac0b2950 0x7fb6ac0b4e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x5654630fe420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.535+0000 7fb6c9e00640 1 -- 172.21.15.28:0/508369782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f0e50 msgr2=0x5654630f1230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f0e50 0x5654630f1230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6c9e00640 1 -- 172.21.15.28:0/508369782 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x565462fea8b0 con 0x5654630f1770 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6c9400640 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f0e50 0x5654630f1230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6bea00640 1 -- 172.21.15.28:0/3062014292 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb6c00e24e0 con 0x7fb6c4108870 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6ca800640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb6ac0b2950 0x7fb6ac0b4e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630fbaf0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fb6b802a950 tx=0x7fb6b8009510 comp rx=0 tx=0).ready entity=mon.1 client_cookie=966d674077783ffd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:10.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6b3000640 1 -- 172.21.15.28:0/508369782 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6b8037040 con 0x5654630f1770 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6b3000640 1 -- 172.21.15.28:0/508369782 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb6b802ae20 con 0x5654630f1770 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.536+0000 7fb6b3000640 1 -- 172.21.15.28:0/508369782 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6b8030890 con 0x5654630f1770 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/508369782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 msgr2=0x5654630fbaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6ca800640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb6ac0b2950 0x7fb6ac0b4e10 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fb6c000cf40 tx=0x7fb6c000ceb0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630fbaf0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fb6b802a950 tx=0x7fb6b8009510 comp rx=0 tx=0).stop 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/508369782 shutdown_connections 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x5654630fe420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630fbaf0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/508369782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f0e50 0x5654630f1230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/508369782 >> 172.21.15.28:0/508369782 conn(0x5654630ed360 msgr2=0x5654630ed740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.537+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/508369782 shutdown_connections 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.538+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/508369782 wait complete. 2024-05-20T20:08:10.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.538+0000 7fb6cf5dc740 1 -- start start 2024-05-20T20:08:10.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.538+0000 7fb6cf5dc740 1 Processor -- start 2024-05-20T20:08:10.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x5654630fb990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630f8c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x5654630f91e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x565462d37790 con 0x5654630fc030 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x565462ff9bc0 con 0x5654630f0e50 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x565463103010 con 0x5654630f1770 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6cf5dc740 10 client.0 _ll_register_callbacks cb 0x565462fd7400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630f8c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630f8c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57130/0 (socket says 172.21.15.28:57130) 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 -- 172.21.15.28:0/2736822397 learned_addr learned my addr 172.21.15.28:0/2736822397 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9400640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x5654630f91e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9400640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x5654630fb990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 -- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 msgr2=0x5654630fb990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x5654630fb990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 -- 172.21.15.28:0/2736822397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 msgr2=0x5654630f91e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x5654630f91e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9e00640 1 -- 172.21.15.28:0/2736822397 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x565462d66210 con 0x5654630f1770 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.539+0000 7fb6c9400640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x5654630fb990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.540+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630f8c80 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fb6b802a3a0 tx=0x7fb6b8030980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b199a3b1752fe92c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.540+0000 7fb6a7400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.540+0000 7fb6a7400640 1 -- 172.21.15.28:0/2736822397 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6b8037030 con 0x5654630f1770 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.540+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb6b802ae20 con 0x5654630f1770 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.540+0000 7fb6a7400640 1 -- 172.21.15.28:0/2736822397 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb6b803d480 con 0x5654630f1770 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6a7400640 1 -- 172.21.15.28:0/2736822397 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb6b8058050 con 0x5654630f1770 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 10 client.25936 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:10.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x565462e32620 con 0x5654630f1770 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6a7400640 1 -- 172.21.15.28:0/2736822397 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb6b802ae20 con 0x5654630f1770 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6a7400640 1 -- 172.21.15.28:0/2736822397 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb6b8046690 con 0x5654630f1770 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 10 client.25936 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 10 client.25936 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 20 client.25936 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546310fbb0 0x565463111fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 4 client.25936 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.541+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x565463063ae0 con 0x56546310fbb0 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.542+0000 7fb6c9400640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546310fbb0 0x565463111fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.542+0000 7fb6c9400640 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546310fbb0 0x565463111fa0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=47632d3c62d31dd server_cookie=c77dc45285d45477 in_seq=0 out_seq=0 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.542+0000 7fb6a7400640 10 client.25936 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.543+0000 7fb6a7400640 1 -- 172.21.15.28:0/2736822397 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x565463063ae0 con 0x56546310fbb0 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.543+0000 7fb6a7400640 10 client.25936 handle_command_reply: tid=0 2024-05-20T20:08:10.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.543+0000 7fb6cf5dc740 1 client.25936 shutdown 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.543+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 msgr2=0x5654630f8c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.543+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630f8c80 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fb6b802a3a0 tx=0x7fb6b8030980 comp rx=0 tx=0).stop 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 shutdown_connections 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546310fbb0 0x565463111fa0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x5654630f91e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x5654630f8c80 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/2736822397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x5654630fb990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6a7400640 0 client.25936 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 >> 172.21.15.28:0/2736822397 conn(0x5654630ed360 msgr2=0x5654630f30e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 shutdown_connections 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/2736822397 wait complete. 2024-05-20T20:08:10.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.544+0000 7fb6cf5dc740 20 client.25936 trim_cache size 0 max 16384 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 Processor -- start 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 -- start start 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f0e50 0x56546310c070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f1770 0x5654630fb4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x565463117600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c41ac2a0 con 0x5654630f1770 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c41adde0 con 0x5654630fc030 2024-05-20T20:08:10.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.561+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c41ac970 con 0x5654630f0e50 2024-05-20T20:08:10.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f1770 0x5654630fb4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f1770 0x5654630fb4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60674/0 (socket says 172.21.15.28:60674) 2024-05-20T20:08:10.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 -- 172.21.15.28:0/1640912872 learned_addr learned my addr 172.21.15.28:0/1640912872 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9400640 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f0e50 0x56546310c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 -- 172.21.15.28:0/1640912872 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f0e50 msgr2=0x56546310c070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f0e50 0x56546310c070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9400640 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x565463117600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 -- 172.21.15.28:0/1640912872 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 msgr2=0x565463117600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x565463117600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 -- 172.21.15.28:0/1640912872 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5654630f71d0 con 0x5654630f1770 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9400640 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x565463117600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.562+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f1770 0x5654630fb4c0 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7fb6b8046c50 tx=0x7fb6b80308a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec9493576a5c229a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.563+0000 7fb6b2600640 1 -- 172.21.15.28:0/1640912872 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6b8037050 con 0x5654630f1770 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.563+0000 7fb6b2600640 1 -- 172.21.15.28:0/1640912872 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb6b804a030 con 0x5654630f1770 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.563+0000 7fb6b2600640 1 -- 172.21.15.28:0/1640912872 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6b803d6e0 con 0x5654630f1770 2024-05-20T20:08:10.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1640912872 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f1770 msgr2=0x5654630fb4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f1770 0x5654630fb4c0 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7fb6b8046c50 tx=0x7fb6b80308a0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1640912872 shutdown_connections 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630fc030 0x565463117600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630f1770 0x5654630fb4c0 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1640912872 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f0e50 0x56546310c070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1640912872 >> 172.21.15.28:0/1640912872 conn(0x5654630ed360 msgr2=0x5654630f4320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1640912872 shutdown_connections 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1640912872 wait complete. 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 -- start start 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.564+0000 7fb6cf5dc740 1 Processor -- start 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x565463108560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x565463108aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6cf5dc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x565463103ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c41ad710 con 0x5654630fc030 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c41ad040 con 0x5654630f0e50 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6cf5dc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb6c41aa2a0 con 0x5654630f1770 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6cf5dc740 10 client.0 _ll_register_callbacks cb 0x7fb6b4009850 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6c9400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x565463108560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6c9e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x565463108aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6c9400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x565463108560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56278/0 (socket says 172.21.15.28:56278) 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6c9400640 1 -- 172.21.15.28:0/1638664901 learned_addr learned my addr 172.21.15.28:0/1638664901 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.565+0000 7fb6c9400640 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x565463103ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6c9e00640 1 -- 172.21.15.28:0/1638664901 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 msgr2=0x565463108560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x565463108560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6c9e00640 1 -- 172.21.15.28:0/1638664901 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 msgr2=0x565463103ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x565463103ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6c9e00640 1 -- 172.21.15.28:0/1638664901 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x565463104300 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6c9e00640 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x565463108aa0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x5654630ebbd0 tx=0x7fb6b8009510 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a564a28c4e47574f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6a7400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6a7400640 1 -- 172.21.15.28:0/1638664901 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb6b8037040 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6a7400640 1 -- 172.21.15.28:0/1638664901 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb6b803d480 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.566+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb6b804a030 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6a7400640 1 -- 172.21.15.28:0/1638664901 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb6b8046870 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 10 client.25942 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56546317a050 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6a7400640 1 -- 172.21.15.28:0/1638664901 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb6b804a030 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6a7400640 1 -- 172.21.15.28:0/1638664901 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb6b8051580 con 0x5654630f1770 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 10 client.25942 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 10 client.25942 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:10.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 20 client.25942 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:10.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546317a480 0x56546317c870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:10.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 4 client.25942 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:08:10.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.567+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56546305d2c0 con 0x56546317a480 2024-05-20T20:08:10.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.568+0000 7fb6c9400640 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546317a480 0x56546317c870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:10.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.568+0000 7fb6c9400640 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546317a480 0x56546317c870 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=69062a8e0390ab05 server_cookie=b2400d5e4d5dfcd9 in_seq=0 out_seq=0 2024-05-20T20:08:10.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.568+0000 7fb6a7400640 10 client.25942 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:10.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6a7400640 1 -- 172.21.15.28:0/1638664901 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x56546305d2c0 con 0x56546317a480 2024-05-20T20:08:10.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6a7400640 10 client.25942 handle_command_reply: tid=0 2024-05-20T20:08:10.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 client.25942 shutdown 2024-05-20T20:08:10.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 msgr2=0x565463108aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x565463108aa0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x5654630ebbd0 tx=0x7fb6b8009510 comp rx=0 tx=0).stop 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 shutdown_connections 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56546317a480 0x56546317c870 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5654630fc030 0x565463103ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5654630f1770 0x565463108aa0 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 --2- 172.21.15.28:0/1638664901 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5654630f0e50 0x565463108560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 >> 172.21.15.28:0/1638664901 conn(0x5654630ed360 msgr2=0x5654630ebc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6a7400640 0 client.25942 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 shutdown_connections 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.575+0000 7fb6cf5dc740 1 -- 172.21.15.28:0/1638664901 wait complete. 2024-05-20T20:08:10.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.576+0000 7fb6cf5dc740 20 client.25942 trim_cache size 0 max 16384 2024-05-20T20:08:10.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:08:10.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:08:10.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb6ac0b2950 msgr2=0x7fb6ac0b4e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb6ac0b2950 0x7fb6ac0b4e10 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fb6c000cf40 tx=0x7fb6c000ceb0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4108870 msgr2=0x7fb6c419d9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4108870 0x7fb6c419d9c0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fb6c000efc0 tx=0x7fb6c000c800 comp rx=0 tx=0).stop 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 shutdown_connections 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb6c4103220 0x7fb6c4199640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb6ac0b2950 0x7fb6ac0b4e10 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb6c4108870 0x7fb6c419d9c0 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 --2- 172.21.15.28:0/3062014292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6c4102860 0x7fb6c4199100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 >> 172.21.15.28:0/3062014292 conn(0x7fb6c40fe3d0 msgr2=0x7fb6c40fed80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:10.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 shutdown_connections 2024-05-20T20:08:10.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:10.577+0000 7fb6cb200640 1 -- 172.21.15.28:0/3062014292 wait complete. 2024-05-20T20:08:10.905 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:08:11.029 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:10 smithi028 ceph-mon[31178]: pgmap v79: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 22 KiB/s wr, 28 op/s 2024-05-20T20:08:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:10 smithi154 ceph-mon[36314]: pgmap v79: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 22 KiB/s wr, 28 op/s 2024-05-20T20:08:11.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:10 smithi033 ceph-mon[36179]: pgmap v79: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 22 KiB/s wr, 28 op/s 2024-05-20T20:08:11.430 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:13.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:12 smithi028 ceph-mon[31178]: pgmap v80: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:12 smithi154 ceph-mon[36314]: pgmap v80: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:13.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:12 smithi033 ceph-mon[36179]: pgmap v80: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 22 KiB/s wr, 23 op/s 2024-05-20T20:08:13.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 -- 172.21.15.28:0/2557838038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 msgr2=0x7faa1c075d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 --2- 172.21.15.28:0/2557838038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c075d50 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7faa1000b0a0 tx=0x7faa1000b3b0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 -- 172.21.15.28:0/2557838038 shutdown_connections 2024-05-20T20:08:13.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 --2- 172.21.15.28:0/2557838038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7faa1c10fcb0 0x7faa1c112120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 --2- 172.21.15.28:0/2557838038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c075d50 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 --2- 172.21.15.28:0/2557838038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7faa1c076290 0x7faa1c10f770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 -- 172.21.15.28:0/2557838038 >> 172.21.15.28:0/2557838038 conn(0x7faa1c0fefc0 msgr2=0x7faa1c1013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:13.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 -- 172.21.15.28:0/2557838038 shutdown_connections 2024-05-20T20:08:13.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.730+0000 7faa22200700 1 -- 172.21.15.28:0/2557838038 wait complete. 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.731+0000 7faa22200700 1 Processor -- start 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.731+0000 7faa22200700 1 -- start start 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa22200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c19bd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa22200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7faa1c076290 0x7faa1c19c290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa22200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7faa1c10fcb0 0x7faa1c1a0620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa22200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c115840 con 0x7faa1c075970 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa22200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1156c0 con 0x7faa1c076290 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa22200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1159c0 con 0x7faa1c10fcb0 2024-05-20T20:08:13.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa20e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c19bd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa20e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c19bd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60698/0 (socket says 172.21.15.28:60698) 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa20e00700 1 -- 172.21.15.28:0/1705707168 learned_addr learned my addr 172.21.15.28:0/1705707168 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa1be00700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7faa1c076290 0x7faa1c19c290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa21800700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7faa1c10fcb0 0x7faa1c1a0620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa20e00700 1 -- 172.21.15.28:0/1705707168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7faa1c10fcb0 msgr2=0x7faa1c1a0620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa20e00700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7faa1c10fcb0 0x7faa1c1a0620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa20e00700 1 -- 172.21.15.28:0/1705707168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7faa1c076290 msgr2=0x7faa1c19c290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.732+0000 7faa20e00700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7faa1c076290 0x7faa1c19c290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.733+0000 7faa20e00700 1 -- 172.21.15.28:0/1705707168 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faa1c1a0d60 con 0x7faa1c075970 2024-05-20T20:08:13.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.733+0000 7faa20e00700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c19bd50 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7faa1000b070 tx=0x7faa10004d00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:13.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.733+0000 7faa19600700 1 -- 172.21.15.28:0/1705707168 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa10021070 con 0x7faa1c075970 2024-05-20T20:08:13.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.733+0000 7faa19600700 1 -- 172.21.15.28:0/1705707168 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7faa10007e30 con 0x7faa1c075970 2024-05-20T20:08:13.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.733+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faa1c1a0ff0 con 0x7faa1c075970 2024-05-20T20:08:13.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.733+0000 7faa19600700 1 -- 172.21.15.28:0/1705707168 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa100046c0 con 0x7faa1c075970 2024-05-20T20:08:13.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.734+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faa1c1a88d0 con 0x7faa1c075970 2024-05-20T20:08:13.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.738+0000 7faa19600700 1 -- 172.21.15.28:0/1705707168 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7faa10007980 con 0x7faa1c075970 2024-05-20T20:08:13.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.738+0000 7faa2e1e5b80 1 Processor -- start 2024-05-20T20:08:13.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa2e1e5b80 1 -- start start 2024-05-20T20:08:13.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2c6ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2c0d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2c17f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555daa21ee40 con 0x555daa2c1390 2024-05-20T20:08:13.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555daa20e9e0 con 0x555daa2c70e0 2024-05-20T20:08:13.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555daa212a20 con 0x555daa2c67c0 2024-05-20T20:08:13.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa21800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2c6ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa19600700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faa040b29f0 0x7faa040b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa21800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2c6ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57176/0 (socket says 172.21.15.28:57176) 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.739+0000 7faa21800700 1 -- 172.21.15.28:0/309718112 learned_addr learned my addr 172.21.15.28:0/309718112 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa20e00700 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2c0d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa1be00700 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2c17f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa19600700 1 -- 172.21.15.28:0/1705707168 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7faa1000f020 con 0x7faa1c075970 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa21800700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faa040b29f0 0x7faa040b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa21800700 1 -- 172.21.15.28:0/309718112 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 msgr2=0x555daa2c0d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa21800700 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2c0d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa21800700 1 -- 172.21.15.28:0/309718112 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 msgr2=0x555daa2c17f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa21800700 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2c17f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa21800700 1 -- 172.21.15.28:0/309718112 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555daa1d7420 con 0x555daa2c67c0 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa1be00700 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2c17f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.740+0000 7faa20e00700 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2c0d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.741+0000 7faa21800700 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2c6ba0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7faa140137f0 tx=0x7faa14013b00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6430c7c1ec557eb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.741+0000 7faa21800700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faa040b29f0 0x7faa040b4eb0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7faa1401e000 tx=0x7faa14020000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.741+0000 7faa0e000700 1 -- 172.21.15.28:0/309718112 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa140098c0 con 0x555daa2c67c0 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.741+0000 7faa0e000700 1 -- 172.21.15.28:0/309718112 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7faa14023020 con 0x555daa2c67c0 2024-05-20T20:08:13.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.741+0000 7faa0e000700 1 -- 172.21.15.28:0/309718112 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa1401fdd0 con 0x555daa2c67c0 2024-05-20T20:08:13.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/309718112 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 msgr2=0x555daa2c6ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2c6ba0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7faa140137f0 tx=0x7faa14013b00 comp rx=0 tx=0).stop 2024-05-20T20:08:13.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/309718112 shutdown_connections 2024-05-20T20:08:13.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2c17f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2c6ba0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/309718112 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2c0d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/309718112 >> 172.21.15.28:0/309718112 conn(0x555daa1eebe0 msgr2=0x555daa2bb810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:13.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/309718112 shutdown_connections 2024-05-20T20:08:13.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.742+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/309718112 wait complete. 2024-05-20T20:08:13.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.743+0000 7faa2e1e5b80 1 -- start start 2024-05-20T20:08:13.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.743+0000 7faa2e1e5b80 1 Processor -- start 2024-05-20T20:08:13.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2ccf60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2ca300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2ca840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555daa114fa0 con 0x555daa2c1390 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555daa1af760 con 0x555daa2c70e0 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555daa2cdba0 con 0x555daa2c67c0 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa2e1e5b80 10 client.0 _ll_register_callbacks cb 0x555daa1b8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa21800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2ca840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa20e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2ccf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa21800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2ca840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56294/0 (socket says 172.21.15.28:56294) 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa21800700 1 -- 172.21.15.28:0/2114258217 learned_addr learned my addr 172.21.15.28:0/2114258217 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.744+0000 7faa1be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2ca300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa21800700 1 -- 172.21.15.28:0/2114258217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 msgr2=0x555daa2ca300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa21800700 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2ca300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa21800700 1 -- 172.21.15.28:0/2114258217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 msgr2=0x555daa2ccf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa21800700 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2ccf60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa21800700 1 -- 172.21.15.28:0/2114258217 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555daa15e8d0 con 0x555daa2c70e0 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa20e00700 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2ccf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa1be00700 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2ca300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa21800700 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2ca840 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7faa1401fa10 tx=0x7faa140099d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=674dc6d4a93be922 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa02a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.745+0000 7faa02a00700 1 -- 172.21.15.28:0/2114258217 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa1401f480 con 0x555daa2c70e0 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa02a00700 1 -- 172.21.15.28:0/2114258217 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7faa1401f620 con 0x555daa2c70e0 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7faa14023020 con 0x555daa2c70e0 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa02a00700 1 -- 172.21.15.28:0/2114258217 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7faa140050b0 con 0x555daa2c70e0 2024-05-20T20:08:13.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa2e1e5b80 10 client.26000 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:13.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555daa1df060 con 0x555daa2c70e0 2024-05-20T20:08:13.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa02a00700 1 -- 172.21.15.28:0/2114258217 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7faa14023020 con 0x555daa2c70e0 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa02a00700 1 -- 172.21.15.28:0/2114258217 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7faa14003a90 con 0x555daa2c70e0 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa2e1e5b80 10 client.26000 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.746+0000 7faa2e1e5b80 10 client.26000 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.747+0000 7faa2e1e5b80 20 client.26000 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.747+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa2da7e0 0x555daa2dcbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.747+0000 7faa2e1e5b80 4 client.26000 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.747+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555daa220d90 con 0x555daa2da7e0 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.747+0000 7faa20e00700 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa2da7e0 0x555daa2dcbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.748+0000 7faa20e00700 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa2da7e0 0x555daa2dcbd0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=63fc23d16d37fc37 server_cookie=51edb973549f1086 in_seq=0 out_seq=0 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.748+0000 7faa02a00700 10 client.26000 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa02a00700 1 -- 172.21.15.28:0/2114258217 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x555daa220d90 con 0x555daa2da7e0 2024-05-20T20:08:13.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa02a00700 10 client.26000 handle_command_reply: tid=0 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 client.26000 shutdown 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 msgr2=0x555daa2ca840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2ca840 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7faa1401fa10 tx=0x7faa140099d0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 shutdown_connections 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa2da7e0 0x555daa2dcbd0 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c70e0 0x555daa2ca840 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c1390 0x555daa2ccf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/2114258217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2ca300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa02a00700 0 client.26000 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 >> 172.21.15.28:0/2114258217 conn(0x555daa1eebe0 msgr2=0x555daa2bafc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 shutdown_connections 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.749+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/2114258217 wait complete. 2024-05-20T20:08:13.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.750+0000 7faa2e1e5b80 20 client.26000 trim_cache size 0 max 16384 2024-05-20T20:08:13.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 Processor -- start 2024-05-20T20:08:13.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 -- start start 2024-05-20T20:08:13.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2c0d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2d6c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2c9480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1afee0 con 0x555daa2c70e0 2024-05-20T20:08:13.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1b1a20 con 0x555daa2c1390 2024-05-20T20:08:13.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.767+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1b05b0 con 0x555daa2c67c0 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2c9480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2c9480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60716/0 (socket says 172.21.15.28:60716) 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa1be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2d6c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 -- 172.21.15.28:0/1259776153 learned_addr learned my addr 172.21.15.28:0/1259776153 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa20e00700 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2c0d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 -- 172.21.15.28:0/1259776153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 msgr2=0x555daa2d6c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2d6c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 -- 172.21.15.28:0/1259776153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 msgr2=0x555daa2c0d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2c0d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 -- 172.21.15.28:0/1259776153 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555daa2c6df0 con 0x555daa2c70e0 2024-05-20T20:08:13.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa1be00700 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2d6c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:13.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.768+0000 7faa21800700 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2c9480 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7faa140093d0 tx=0x7faa14013b00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f3bef9a72ef86b6e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:13.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.769+0000 7faa0d600700 1 -- 172.21.15.28:0/1259776153 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa1401f680 con 0x555daa2c70e0 2024-05-20T20:08:13.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.769+0000 7faa0d600700 1 -- 172.21.15.28:0/1259776153 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7faa14023070 con 0x555daa2c70e0 2024-05-20T20:08:13.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.769+0000 7faa0d600700 1 -- 172.21.15.28:0/1259776153 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa14003c30 con 0x555daa2c70e0 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/1259776153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 msgr2=0x555daa2c9480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2c9480 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7faa140093d0 tx=0x7faa14013b00 comp rx=0 tx=0).stop 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/1259776153 shutdown_connections 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2c9480 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2c0d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/1259776153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2d6c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/1259776153 >> 172.21.15.28:0/1259776153 conn(0x555daa1eebe0 msgr2=0x555daa2c5a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/1259776153 shutdown_connections 2024-05-20T20:08:13.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/1259776153 wait complete. 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.770+0000 7faa2e1e5b80 1 -- start start 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 1 Processor -- start 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2d3300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2e19e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2e1f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1b1350 con 0x555daa2c70e0 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1b0c80 con 0x555daa2c1390 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7faa1c1ade80 con 0x555daa2c67c0 2024-05-20T20:08:13.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.771+0000 7faa2e1e5b80 10 client.0 _ll_register_callbacks cb 0x555daa1b8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa21800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2e1f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa20e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2d3300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa1be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2e19e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa20e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2d3300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56318/0 (socket says 172.21.15.28:56318) 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa1be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2e19e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57196/0 (socket says 172.21.15.28:57196) 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa20e00700 1 -- 172.21.15.28:0/714680184 learned_addr learned my addr 172.21.15.28:0/714680184 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa1be00700 1 -- 172.21.15.28:0/714680184 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 msgr2=0x555daa2d3300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa1be00700 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2d3300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa1be00700 1 -- 172.21.15.28:0/714680184 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 msgr2=0x555daa2e1f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa1be00700 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2e1f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.772+0000 7faa1be00700 1 -- 172.21.15.28:0/714680184 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555daa2e24e0 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa1be00700 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2e19e0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x555daa2c7060 tx=0x7faa08010f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ea91d1826c6019f8 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa02a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa02a00700 1 -- 172.21.15.28:0/714680184 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faa08021070 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7faa14023070 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa02a00700 1 -- 172.21.15.28:0/714680184 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7faa0800fc50 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa02a00700 1 -- 172.21.15.28:0/714680184 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7faa0802a050 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa2e1e5b80 10 client.25960 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.773+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555daa34d880 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa02a00700 1 -- 172.21.15.28:0/714680184 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7faa14023070 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa02a00700 1 -- 172.21.15.28:0/714680184 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7faa08024020 con 0x555daa2c67c0 2024-05-20T20:08:13.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa2e1e5b80 10 client.25960 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:13.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa2e1e5b80 10 client.25960 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:08:13.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa2e1e5b80 20 client.25960 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:13.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa34dfe0 0x555daa350400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:13.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa2e1e5b80 4 client.25960 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:08:13.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.774+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x555daa21a570 con 0x555daa34dfe0 2024-05-20T20:08:13.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.775+0000 7faa20e00700 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa34dfe0 0x555daa350400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:13.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.775+0000 7faa20e00700 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa34dfe0 0x555daa350400 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=890005cf1cfcf2e8 server_cookie=cc03822d44a409c3 in_seq=0 out_seq=0 2024-05-20T20:08:13.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.775+0000 7faa02a00700 10 client.25960 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:13.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.775+0000 7faa02a00700 1 -- 172.21.15.28:0/714680184 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x555daa21a570 con 0x555daa34dfe0 2024-05-20T20:08:13.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.775+0000 7faa02a00700 10 client.25960 handle_command_reply: tid=0 2024-05-20T20:08:13.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.775+0000 7faa2e1e5b80 1 client.25960 shutdown 2024-05-20T20:08:13.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 msgr2=0x555daa2e19e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2e19e0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x555daa2c7060 tx=0x7faa08010f90 comp rx=0 tx=0).stop 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 shutdown_connections 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x555daa34dfe0 0x555daa350400 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555daa2c70e0 0x555daa2e1f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555daa2c1390 0x555daa2d3300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 --2- 172.21.15.28:0/714680184 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555daa2c67c0 0x555daa2e19e0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 >> 172.21.15.28:0/714680184 conn(0x555daa1eebe0 msgr2=0x555daa2cb900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa02a00700 0 client.25960 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 shutdown_connections 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 1 -- 172.21.15.28:0/714680184 wait complete. 2024-05-20T20:08:13.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.776+0000 7faa2e1e5b80 20 client.25960 trim_cache size 0 max 16384 2024-05-20T20:08:13.779 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:08:13.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faa040b29f0 msgr2=0x7faa040b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faa040b29f0 0x7faa040b4eb0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7faa1401e000 tx=0x7faa14020000 comp rx=0 tx=0).stop 2024-05-20T20:08:13.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 msgr2=0x7faa1c19bd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c19bd50 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7faa1000b070 tx=0x7faa10004d00 comp rx=0 tx=0).stop 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 shutdown_connections 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7faa1c10fcb0 0x7faa1c1a0620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7faa1c075970 0x7faa1c19bd50 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faa040b29f0 0x7faa040b4eb0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 --2- 172.21.15.28:0/1705707168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7faa1c076290 0x7faa1c19c290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 >> 172.21.15.28:0/1705707168 conn(0x7faa1c0fefc0 msgr2=0x7faa1c0ffd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 shutdown_connections 2024-05-20T20:08:13.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:13.777+0000 7faa22200700 1 -- 172.21.15.28:0/1705707168 wait complete. 2024-05-20T20:08:14.835 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:08:15.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:14 smithi028 ceph-mon[31178]: pgmap v81: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 24 KiB/s wr, 28 op/s 2024-05-20T20:08:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:14 smithi154 ceph-mon[36314]: pgmap v81: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 24 KiB/s wr, 28 op/s 2024-05-20T20:08:15.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:14 smithi033 ceph-mon[36179]: pgmap v81: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 24 KiB/s wr, 28 op/s 2024-05-20T20:08:15.320 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:17.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:16 smithi028 ceph-mon[31178]: pgmap v82: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-05-20T20:08:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:16 smithi154 ceph-mon[36314]: pgmap v82: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-05-20T20:08:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:16 smithi033 ceph-mon[36179]: pgmap v82: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-05-20T20:08:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 -- 172.21.15.28:0/3394138125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f010e950 msgr2=0x7f19f010ed30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 --2- 172.21.15.28:0/3394138125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f010e950 0x7f19f010ed30 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f19e0009990 tx=0x7f19e0009ca0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 -- 172.21.15.28:0/3394138125 shutdown_connections 2024-05-20T20:08:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 --2- 172.21.15.28:0/3394138125 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19f010a440 0x7f19f01168e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 --2- 172.21.15.28:0/3394138125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f010e950 0x7f19f010ed30 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 --2- 172.21.15.28:0/3394138125 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19f0109a80 0x7f19f0109f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 -- 172.21.15.28:0/3394138125 >> 172.21.15.28:0/3394138125 conn(0x7f19f00792d0 msgr2=0x7f19f0108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 -- 172.21.15.28:0/3394138125 shutdown_connections 2024-05-20T20:08:17.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.798+0000 7f19efe00700 1 -- 172.21.15.28:0/3394138125 wait complete. 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.799+0000 7f19efe00700 1 Processor -- start 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.799+0000 7f19efe00700 1 -- start start 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19efe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f0109a80 0x7f19f01a0140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19efe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19f010a440 0x7f19f01a0680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19efe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19f010e950 0x7f19f01a4a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19efe00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f19f0119c30 con 0x7f19f0109a80 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19efe00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f19f0119ab0 con 0x7f19f010a440 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19efe00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f19f0119db0 con 0x7f19f010e950 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f0109a80 0x7f19f01a0140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f0109a80 0x7f19f01a0140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42530/0 (socket says 172.21.15.28:42530) 2024-05-20T20:08:17.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 -- 172.21.15.28:0/533123557 learned_addr learned my addr 172.21.15.28:0/533123557 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:17.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19e6000700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19f010a440 0x7f19f01a0680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19ef400700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19f010e950 0x7f19f01a4a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 -- 172.21.15.28:0/533123557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19f010e950 msgr2=0x7f19f01a4a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19f010e950 0x7f19f01a4a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 -- 172.21.15.28:0/533123557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19f010a440 msgr2=0x7f19f01a0680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19f010a440 0x7f19f01a0680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.800+0000 7f19eea00700 1 -- 172.21.15.28:0/533123557 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f19f01a5150 con 0x7f19f0109a80 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.801+0000 7f19eea00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f0109a80 0x7f19f01a0140 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f19e0000c00 tx=0x7f19e000bfd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.801+0000 7f19f4a00700 1 -- 172.21.15.28:0/533123557 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19e0027070 con 0x7f19f0109a80 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.801+0000 7f19f4a00700 1 -- 172.21.15.28:0/533123557 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f19e0022470 con 0x7f19f0109a80 2024-05-20T20:08:17.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.801+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f19f01a53e0 con 0x7f19f0109a80 2024-05-20T20:08:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.801+0000 7f19f4a00700 1 -- 172.21.15.28:0/533123557 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19e0017650 con 0x7f19f0109a80 2024-05-20T20:08:17.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.801+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f19f01accc0 con 0x7f19f0109a80 2024-05-20T20:08:17.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.805+0000 7f19f4a00700 1 -- 172.21.15.28:0/533123557 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f19e002a070 con 0x7f19f0109a80 2024-05-20T20:08:17.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.806+0000 7f1a006f3b80 1 Processor -- start 2024-05-20T20:08:17.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.806+0000 7f1a006f3b80 1 -- start start 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.806+0000 7f19f4a00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f19d40b2a70 0x7f19d40b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f6fba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f700e0 0x558642f69d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f6a7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558642ec7e40 con 0x558642f6f7c0 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558642eb79e0 con 0x558642f700e0 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558642ebba20 con 0x558642f6a390 2024-05-20T20:08:17.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19f4a00700 1 -- 172.21.15.28:0/533123557 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f19e00e3240 con 0x7f19f0109a80 2024-05-20T20:08:17.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19ef400700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f19d40b2a70 0x7f19d40b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19e6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f700e0 0x558642f69d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19e6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f700e0 0x558642f69d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43054/0 (socket says 172.21.15.28:43054) 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19e6000700 1 -- 172.21.15.28:0/3021484525 learned_addr learned my addr 172.21.15.28:0/3021484525 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19ef400700 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f6a7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19eea00700 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f6fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19e6000700 1 -- 172.21.15.28:0/3021484525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 msgr2=0x558642f6a7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.807+0000 7f19e6000700 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f6a7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19e6000700 1 -- 172.21.15.28:0/3021484525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 msgr2=0x558642f6fba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19e6000700 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f6fba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19e6000700 1 -- 172.21.15.28:0/3021484525 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558642e80420 con 0x558642f700e0 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19ef400700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f19d40b2a70 0x7f19d40b4f30 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f19e80097a0 tx=0x7f19e8009ae0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19e6000700 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f700e0 0x558642f69d70 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f19d8011600 tx=0x7f19d8009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b63c1ca7f00d08ae server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19e4c00700 1 -- 172.21.15.28:0/3021484525 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19d8019030 con 0x558642f700e0 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19e4c00700 1 -- 172.21.15.28:0/3021484525 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f19d80092e0 con 0x558642f700e0 2024-05-20T20:08:17.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.808+0000 7f19e4c00700 1 -- 172.21.15.28:0/3021484525 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19d8007500 con 0x558642f700e0 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3021484525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f700e0 msgr2=0x558642f69d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f700e0 0x558642f69d70 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f19d8011600 tx=0x7f19d8009920 comp rx=0 tx=0).stop 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3021484525 shutdown_connections 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f6a7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f6fba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3021484525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f700e0 0x558642f69d70 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3021484525 >> 172.21.15.28:0/3021484525 conn(0x558642e97be0 msgr2=0x558642f64810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3021484525 shutdown_connections 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.809+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3021484525 wait complete. 2024-05-20T20:08:17.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.810+0000 7f1a006f3b80 1 -- start start 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.810+0000 7f1a006f3b80 1 Processor -- start 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6a390 0x558642f75e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f731b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f736f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558642dbdfa0 con 0x558642f6f7c0 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558642e58760 con 0x558642f6a390 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558642f76c00 con 0x558642f700e0 2024-05-20T20:08:17.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f1a006f3b80 10 client.0 _ll_register_callbacks cb 0x558642e61c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f19ef400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f736f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f19eea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6a390 0x558642f75e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f19ef400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f736f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52342/0 (socket says 172.21.15.28:52342) 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f19ef400700 1 -- 172.21.15.28:0/1569154865 learned_addr learned my addr 172.21.15.28:0/1569154865 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.811+0000 7f19e6000700 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f731b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19ef400700 1 -- 172.21.15.28:0/1569154865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6a390 msgr2=0x558642f75e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19ef400700 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6a390 0x558642f75e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19ef400700 1 -- 172.21.15.28:0/1569154865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 msgr2=0x558642f731b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19ef400700 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f731b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19ef400700 1 -- 172.21.15.28:0/1569154865 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558642e078d0 con 0x558642f700e0 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19eea00700 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6a390 0x558642f75e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19ef400700 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f736f0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f19e8004480 tx=0x7f19e80047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1d6074dfd9b0c889 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:17.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.812+0000 7f19dd600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f19dd600700 1 -- 172.21.15.28:0/1569154865 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19e80044b0 con 0x558642f700e0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f19d80092e0 con 0x558642f700e0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f19dd600700 1 -- 172.21.15.28:0/1569154865 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f19e8003680 con 0x558642f700e0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f19dd600700 1 -- 172.21.15.28:0/1569154865 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f19e8003880 con 0x558642f700e0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f1a006f3b80 10 client.25972 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558642e88060 con 0x558642f700e0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f19dd600700 1 -- 172.21.15.28:0/1569154865 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f19d80092e0 con 0x558642f700e0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.813+0000 7f19dd600700 1 -- 172.21.15.28:0/1569154865 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f19e8020050 con 0x558642f700e0 2024-05-20T20:08:17.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f1a006f3b80 10 client.25972 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f1a006f3b80 10 client.25972 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f1a006f3b80 20 client.25972 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642f83720 0x558642f85b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f1a006f3b80 4 client.25972 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558642ec9d90 con 0x558642f83720 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f19eea00700 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642f83720 0x558642f85b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f19eea00700 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642f83720 0x558642f85b70 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=898a4a5506055432 server_cookie=ff9a061c33aa320f in_seq=0 out_seq=0 2024-05-20T20:08:17.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.814+0000 7f19dd600700 10 client.25972 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.815+0000 7f19dd600700 1 -- 172.21.15.28:0/1569154865 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558642ec9d90 con 0x558642f83720 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.815+0000 7f19dd600700 10 client.25972 handle_command_reply: tid=0 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.815+0000 7f1a006f3b80 1 client.25972 shutdown 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 msgr2=0x558642f736f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f736f0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f19e8004480 tx=0x7f19e80047c0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 shutdown_connections 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f736f0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6a390 0x558642f75e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642f83720 0x558642f85b70 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/1569154865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6f7c0 0x558642f731b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 >> 172.21.15.28:0/1569154865 conn(0x558642e97be0 msgr2=0x558642f63fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f19dd600700 0 client.25972 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 shutdown_connections 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 1 -- 172.21.15.28:0/1569154865 wait complete. 2024-05-20T20:08:17.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.816+0000 7f1a006f3b80 20 client.25972 trim_cache size 0 max 16384 2024-05-20T20:08:17.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.833+0000 7f1a006f3b80 1 Processor -- start 2024-05-20T20:08:17.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.833+0000 7f1a006f3b80 1 -- start start 2024-05-20T20:08:17.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f69d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f6d8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f700e0 0x558642f70d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f19f01b41e0 con 0x558642f700e0 2024-05-20T20:08:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f19f01b5d20 con 0x558642f6f7c0 2024-05-20T20:08:17.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f19f01b48b0 con 0x558642f6a390 2024-05-20T20:08:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19ef400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f700e0 0x558642f70d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f69d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19e6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f6d8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f69d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52350/0 (socket says 172.21.15.28:52350) 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19e6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f6d8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43084/0 (socket says 172.21.15.28:43084) 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 -- 172.21.15.28:0/3359098772 learned_addr learned my addr 172.21.15.28:0/3359098772 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 -- 172.21.15.28:0/3359098772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 msgr2=0x558642f6d8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f6d8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 -- 172.21.15.28:0/3359098772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f700e0 msgr2=0x558642f70d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f700e0 0x558642f70d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19eea00700 1 -- 172.21.15.28:0/3359098772 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558642f861f0 con 0x558642f6a390 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.834+0000 7f19e6000700 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f6d8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.835+0000 7f19eea00700 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f69d70 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f19e00a77d0 tx=0x7f19e0009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6d5c6665651d4243 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.835+0000 7f19dfe00700 1 -- 172.21.15.28:0/3359098772 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19e00aac60 con 0x558642f6a390 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.835+0000 7f19dfe00700 1 -- 172.21.15.28:0/3359098772 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f19e00aa560 con 0x558642f6a390 2024-05-20T20:08:17.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.835+0000 7f19dfe00700 1 -- 172.21.15.28:0/3359098772 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19e0022940 con 0x558642f6a390 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3359098772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 msgr2=0x558642f69d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f69d70 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f19e00a77d0 tx=0x7f19e0009310 comp rx=0 tx=0).stop 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3359098772 shutdown_connections 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f700e0 0x558642f70d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f6a390 0x558642f69d70 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/3359098772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f6d8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3359098772 >> 172.21.15.28:0/3359098772 conn(0x558642e97be0 msgr2=0x558642f71970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3359098772 shutdown_connections 2024-05-20T20:08:17.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 -- 172.21.15.28:0/3359098772 wait complete. 2024-05-20T20:08:17.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 -- start start 2024-05-20T20:08:17.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.836+0000 7f1a006f3b80 1 Processor -- start 2024-05-20T20:08:17.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6a390 0x558642f7c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f7c620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f1a006f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f8a890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f19f01b5650 con 0x558642f6a390 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f19f01b4f80 con 0x558642f6f7c0 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f1a006f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f19f01b21a0 con 0x558642f700e0 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f1a006f3b80 10 client.0 _ll_register_callbacks cb 0x558642e61c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f19e6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f7c620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f19e6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f7c620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43088/0 (socket says 172.21.15.28:43088) 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f19e6000700 1 -- 172.21.15.28:0/791065389 learned_addr learned my addr 172.21.15.28:0/791065389 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f19eea00700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6a390 0x558642f7c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.837+0000 7f19ef400700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f8a890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19e6000700 1 -- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 msgr2=0x558642f8a890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19e6000700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f8a890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19e6000700 1 -- 172.21.15.28:0/791065389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6a390 msgr2=0x558642f7c0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19e6000700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6a390 0x558642f7c0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19e6000700 1 -- 172.21.15.28:0/791065389 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558642f8b0d0 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19eea00700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6a390 0x558642f7c0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19e6000700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f7c620 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f19d8004470 tx=0x7f19d8003f40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=57fcb66440638767 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19dd600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19dd600700 1 -- 172.21.15.28:0/791065389 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f19d8005230 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f19dd600700 1 -- 172.21.15.28:0/791065389 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f19d80053d0 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.838+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f19e00aa560 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f19dd600700 1 -- 172.21.15.28:0/791065389 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f19d8030050 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 10 client.26024 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558642feded0 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f19dd600700 1 -- 172.21.15.28:0/791065389 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f19e00aa560 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f19dd600700 1 -- 172.21.15.28:0/791065389 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f19d8007c10 con 0x558642f6f7c0 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 10 client.26024 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 10 client.26024 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:08:17.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 20 client.26024 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:17.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642fee310 0x558642ff0730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:17.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 4 client.26024 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:08:17.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.839+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x558642ec3570 con 0x558642fee310 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.840+0000 7f19eea00700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642fee310 0x558642ff0730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.840+0000 7f19eea00700 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642fee310 0x558642ff0730 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f1ae8be22e0d211 server_cookie=c91e56c3a92e7dcc in_seq=0 out_seq=0 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.840+0000 7f19dd600700 10 client.26024 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f19dd600700 1 -- 172.21.15.28:0/791065389 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x558642ec3570 con 0x558642fee310 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f19dd600700 10 client.26024 handle_command_reply: tid=0 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 client.26024 shutdown 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 msgr2=0x558642f7c620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f7c620 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f19d8004470 tx=0x7f19d8003f40 comp rx=0 tx=0).stop 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 shutdown_connections 2024-05-20T20:08:17.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558642f700e0 0x558642f8a890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x558642fee310 0x558642ff0730 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558642f6a390 0x558642f7c0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 --2- 172.21.15.28:0/791065389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558642f6f7c0 0x558642f7c620 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 >> 172.21.15.28:0/791065389 conn(0x558642e97be0 msgr2=0x558642f70bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f19dd600700 0 client.26024 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 shutdown_connections 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 1 -- 172.21.15.28:0/791065389 wait complete. 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.841+0000 7f1a006f3b80 20 client.26024 trim_cache size 0 max 16384 2024-05-20T20:08:17.844 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f19d40b2a70 msgr2=0x7f19d40b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f19d40b2a70 0x7f19d40b4f30 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f19e80097a0 tx=0x7f19e8009ae0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f0109a80 msgr2=0x7f19f01a0140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f0109a80 0x7f19f01a0140 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f19e0000c00 tx=0x7f19e000bfd0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 shutdown_connections 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f19f010e950 0x7f19f01a4a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f19f0109a80 0x7f19f01a0140 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f19d40b2a70 0x7f19d40b4f30 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 --2- 172.21.15.28:0/533123557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f19f010a440 0x7f19f01a0680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:17.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 >> 172.21.15.28:0/533123557 conn(0x7f19f00792d0 msgr2=0x7f19f00798f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:17.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 shutdown_connections 2024-05-20T20:08:17.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:17.842+0000 7f19efe00700 1 -- 172.21.15.28:0/533123557 wait complete. 2024-05-20T20:08:18.862 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:18 smithi028 ceph-mon[31178]: pgmap v83: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-05-20T20:08:18.866 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:08:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:18 smithi154 ceph-mon[36314]: pgmap v83: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-05-20T20:08:19.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:18 smithi033 ceph-mon[36179]: pgmap v83: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-05-20T20:08:19.326 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:21.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:20 smithi028 ceph-mon[31178]: pgmap v84: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 1.5 KiB/s wr, 13 op/s 2024-05-20T20:08:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:20 smithi154 ceph-mon[36314]: pgmap v84: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 1.5 KiB/s wr, 13 op/s 2024-05-20T20:08:21.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:20 smithi033 ceph-mon[36179]: pgmap v84: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 1.5 KiB/s wr, 13 op/s 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 -- 172.21.15.28:0/2565267829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8103490 msgr2=0x7face8103910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 --2- 172.21.15.28:0/2565267829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8103490 0x7face8103910 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7facd0009920 tx=0x7facd0009c30 comp rx=0 tx=0).stop 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 -- 172.21.15.28:0/2565267829 shutdown_connections 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 --2- 172.21.15.28:0/2565267829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7face8103e50 0x7face8110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 --2- 172.21.15.28:0/2565267829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7face8109540 0x7face8109920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 --2- 172.21.15.28:0/2565267829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8103490 0x7face8103910 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 -- 172.21.15.28:0/2565267829 >> 172.21.15.28:0/2565267829 conn(0x7face80fefc0 msgr2=0x7face81013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:21.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 -- 172.21.15.28:0/2565267829 shutdown_connections 2024-05-20T20:08:21.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.673+0000 7facece00700 1 -- 172.21.15.28:0/2565267829 wait complete. 2024-05-20T20:08:21.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.675+0000 7facece00700 1 Processor -- start 2024-05-20T20:08:21.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.676+0000 7facece00700 1 -- start start 2024-05-20T20:08:21.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.676+0000 7facece00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7face8103490 0x7face8075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.676+0000 7facece00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7face8103e50 0x7face8075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.676+0000 7facece00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8109540 0x7face80762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.676+0000 7facece00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7face8113670 con 0x7face8103490 2024-05-20T20:08:21.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.676+0000 7facece00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7face81134f0 con 0x7face8109540 2024-05-20T20:08:21.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.676+0000 7facece00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7face81137f0 con 0x7face8103e50 2024-05-20T20:08:21.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7face7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7face8103490 0x7face8075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7face7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7face8103490 0x7face8075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42608/0 (socket says 172.21.15.28:42608) 2024-05-20T20:08:21.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7face7400700 1 -- 172.21.15.28:0/4111285437 learned_addr learned my addr 172.21.15.28:0/4111285437 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:21.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7facdea00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7face8103e50 0x7face8075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7face7e00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8109540 0x7face80762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7facdea00700 1 -- 172.21.15.28:0/4111285437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8109540 msgr2=0x7face80762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7facdea00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8109540 0x7face80762c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7facdea00700 1 -- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7face8103490 msgr2=0x7face8075840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7facdea00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7face8103490 0x7face8075840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.677+0000 7facdea00700 1 -- 172.21.15.28:0/4111285437 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7face81a80d0 con 0x7face8103e50 2024-05-20T20:08:21.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.678+0000 7facdea00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7face8103e50 0x7face8075d80 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7facd0000c00 tx=0x7facd000f950 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:21.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.678+0000 7face4c00700 1 -- 172.21.15.28:0/4111285437 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7facd0027070 con 0x7face8103e50 2024-05-20T20:08:21.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.678+0000 7face4c00700 1 -- 172.21.15.28:0/4111285437 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7facd00054f0 con 0x7face8103e50 2024-05-20T20:08:21.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.678+0000 7face4c00700 1 -- 172.21.15.28:0/4111285437 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7facd0004930 con 0x7face8103e50 2024-05-20T20:08:21.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.678+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7face81a8360 con 0x7face8103e50 2024-05-20T20:08:21.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.678+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7face81a87b0 con 0x7face8103e50 2024-05-20T20:08:21.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.682+0000 7face4c00700 1 -- 172.21.15.28:0/4111285437 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7facd002a070 con 0x7face8103e50 2024-05-20T20:08:21.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.682+0000 7facf8e05b80 1 Processor -- start 2024-05-20T20:08:21.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facf8e05b80 1 -- start start 2024-05-20T20:08:21.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e21400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e1b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e1bf30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facf8e05b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562705d79e40 con 0x562705e1bad0 2024-05-20T20:08:21.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facf8e05b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562705d699e0 con 0x562705e21940 2024-05-20T20:08:21.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facf8e05b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562705d6da20 con 0x562705e21020 2024-05-20T20:08:21.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7face7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e1b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7face7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e1b4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43128/0 (socket says 172.21.15.28:43128) 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.683+0000 7facdea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e1bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7e00700 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e21400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7400700 1 -- 172.21.15.28:0/2032910137 learned_addr learned my addr 172.21.15.28:0/2032910137 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7e00700 1 -- 172.21.15.28:0/2032910137 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 msgr2=0x562705e1b4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face4c00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faccc0b2a70 0x7faccc0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7e00700 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e1b4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7e00700 1 -- 172.21.15.28:0/2032910137 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 msgr2=0x562705e1bf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7e00700 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e1bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7e00700 1 -- 172.21.15.28:0/2032910137 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562705d32420 con 0x562705e21020 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face7400700 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e1b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:21.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.684+0000 7face4c00700 1 -- 172.21.15.28:0/4111285437 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7facd00e2ea0 con 0x7face8103e50 2024-05-20T20:08:21.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.685+0000 7face7e00700 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e21400 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7face0009cd0 tx=0x7face00099e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b34fa28415314b46 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:21.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.685+0000 7face7e00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faccc0b2a70 0x7faccc0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.685+0000 7facdcc00700 1 -- 172.21.15.28:0/2032910137 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7face0017660 con 0x562705e21020 2024-05-20T20:08:21.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.685+0000 7facdcc00700 1 -- 172.21.15.28:0/2032910137 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7face0017cb0 con 0x562705e21020 2024-05-20T20:08:21.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.685+0000 7facdcc00700 1 -- 172.21.15.28:0/2032910137 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7face00203d0 con 0x562705e21020 2024-05-20T20:08:21.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 -- 172.21.15.28:0/2032910137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 msgr2=0x562705e21400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e21400 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7face0009cd0 tx=0x7face00099e0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7face7e00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faccc0b2a70 0x7faccc0b4f30 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7face0017fd0 tx=0x7face0015f20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 -- 172.21.15.28:0/2032910137 shutdown_connections 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e1bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e21400 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 --2- 172.21.15.28:0/2032910137 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e1b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 -- 172.21.15.28:0/2032910137 >> 172.21.15.28:0/2032910137 conn(0x562705d49be0 msgr2=0x562705e16150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 -- 172.21.15.28:0/2032910137 shutdown_connections 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 -- 172.21.15.28:0/2032910137 wait complete. 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.686+0000 7facf8e05b80 1 -- start start 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 1 Processor -- start 2024-05-20T20:08:21.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e249f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e24f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562705c6ffa0 con 0x562705e1bad0 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562705d0a760 con 0x562705e21940 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562705e28460 con 0x562705e21020 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.687+0000 7facf8e05b80 10 client.0 _ll_register_callbacks cb 0x562705d13c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42624/0 (socket says 172.21.15.28:42624) 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7facdea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e249f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 -- 172.21.15.28:0/3232780912 learned_addr learned my addr 172.21.15.28:0/3232780912 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:21.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7e00700 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e24f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 -- 172.21.15.28:0/3232780912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 msgr2=0x562705e249f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e249f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 -- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 msgr2=0x562705e24f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e24f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7face7400700 1 -- 172.21.15.28:0/3232780912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562705cb98d0 con 0x562705e1bad0 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.688+0000 7facdea00700 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e249f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.689+0000 7face7400700 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7facd8011970 tx=0x7facd8011c80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea12f075da89a659 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:21.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.689+0000 7facd5600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.689+0000 7facd5600700 1 -- 172.21.15.28:0/3232780912 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7facd801f060 con 0x562705e1bad0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.689+0000 7facd5600700 1 -- 172.21.15.28:0/3232780912 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7facd8017070 con 0x562705e1bad0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.689+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7face0017cb0 con 0x562705e1bad0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.689+0000 7facd5600700 1 -- 172.21.15.28:0/3232780912 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7facd8004260 con 0x562705e1bad0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.689+0000 7facf8e05b80 10 client.15891 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562705d3a060 con 0x562705e1bad0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facd5600700 1 -- 172.21.15.28:0/3232780912 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7face0017cb0 con 0x562705e1bad0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facd5600700 1 -- 172.21.15.28:0/3232780912 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7facd8003810 con 0x562705e1bad0 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facf8e05b80 10 client.15891 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facf8e05b80 10 client.15891 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facf8e05b80 20 client.15891 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facf8e05b80 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e35020 0x562705e37470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facf8e05b80 4 client.15891 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:21.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.690+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562705d7bd90 con 0x562705e35020 2024-05-20T20:08:21.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.691+0000 7facdea00700 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e35020 0x562705e37470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.691+0000 7facdea00700 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e35020 0x562705e37470 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e94963b47d3ff11b server_cookie=58c497a5c26f29a3 in_seq=0 out_seq=0 2024-05-20T20:08:21.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.691+0000 7facd5600700 10 client.15891 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.692+0000 7facd5600700 1 -- 172.21.15.28:0/3232780912 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562705d7bd90 con 0x562705e35020 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.692+0000 7facd5600700 10 client.15891 handle_command_reply: tid=0 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.692+0000 7facf8e05b80 1 client.15891 shutdown 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.692+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 msgr2=0x562705e27650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.692+0000 7facf8e05b80 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7facd8011970 tx=0x7facd8011c80 comp rx=0 tx=0).stop 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.692+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 shutdown_connections 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e35020 0x562705e37470 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21940 0x562705e24f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 1 --2- 172.21.15.28:0/3232780912 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21020 0x562705e249f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 >> 172.21.15.28:0/3232780912 conn(0x562705d49be0 msgr2=0x562705e1a4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:21.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facd5600700 0 client.15891 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:08:21.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 shutdown_connections 2024-05-20T20:08:21.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 1 -- 172.21.15.28:0/3232780912 wait complete. 2024-05-20T20:08:21.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.693+0000 7facf8e05b80 20 client.15891 trim_cache size 0 max 16384 2024-05-20T20:08:21.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 Processor -- start 2024-05-20T20:08:21.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 -- start start 2024-05-20T20:08:21.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e1eb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21940 0x562705e1f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7face81afdb0 con 0x562705e1bad0 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7face81b18f0 con 0x562705e21020 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.710+0000 7facf8e05b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7face81b0480 con 0x562705e21940 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e1eb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7face7400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e1eb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43152/0 (socket says 172.21.15.28:43152) 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 -- 172.21.15.28:0/1684901342 learned_addr learned my addr 172.21.15.28:0/1684901342 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:21.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7face7e00700 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21940 0x562705e1f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 -- 172.21.15.28:0/1684901342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21940 msgr2=0x562705e1f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21940 0x562705e1f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 -- 172.21.15.28:0/1684901342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 msgr2=0x562705e27650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 -- 172.21.15.28:0/1684901342 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562705e22f60 con 0x562705e21020 2024-05-20T20:08:21.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.711+0000 7facdea00700 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e1eb70 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7facd00a7430 tx=0x7facd00e6950 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e1b2d94842e66075 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:21.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.712+0000 7facd7e00700 1 -- 172.21.15.28:0/1684901342 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7facd00a9b10 con 0x562705e21020 2024-05-20T20:08:21.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.712+0000 7facd7e00700 1 -- 172.21.15.28:0/1684901342 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7facd00056a0 con 0x562705e21020 2024-05-20T20:08:21.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.712+0000 7facd7e00700 1 -- 172.21.15.28:0/1684901342 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7facd00a9930 con 0x562705e21020 2024-05-20T20:08:21.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 -- 172.21.15.28:0/1684901342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 msgr2=0x562705e1eb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e1eb70 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7facd00a7430 tx=0x7facd00e6950 comp rx=0 tx=0).stop 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 -- 172.21.15.28:0/1684901342 shutdown_connections 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e21940 0x562705e1f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e1bad0 0x562705e27650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 --2- 172.21.15.28:0/1684901342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e1eb70 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 -- 172.21.15.28:0/1684901342 >> 172.21.15.28:0/1684901342 conn(0x562705d49be0 msgr2=0x562705e0ffe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 -- 172.21.15.28:0/1684901342 shutdown_connections 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 -- 172.21.15.28:0/1684901342 wait complete. 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.713+0000 7facf8e05b80 1 -- start start 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 1 Processor -- start 2024-05-20T20:08:21.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e1bad0 0x562705e2d8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e2de10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e21940 0x562705e3c170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7face81b1220 con 0x562705e21940 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7face81b0b50 con 0x562705e21020 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7face81add70 con 0x562705e1bad0 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facf8e05b80 10 client.0 _ll_register_callbacks cb 0x562705d13c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facdea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e2de10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facdea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e2de10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43166/0 (socket says 172.21.15.28:43166) 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7facdea00700 1 -- 172.21.15.28:0/117411606 learned_addr learned my addr 172.21.15.28:0/117411606 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.714+0000 7face7400700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e1bad0 0x562705e2d8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7face7e00700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e21940 0x562705e3c170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facdea00700 1 -- 172.21.15.28:0/117411606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e1bad0 msgr2=0x562705e2d8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facdea00700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e1bad0 0x562705e2d8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facdea00700 1 -- 172.21.15.28:0/117411606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e21940 msgr2=0x562705e3c170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:21.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facdea00700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e21940 0x562705e3c170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facdea00700 1 -- 172.21.15.28:0/117411606 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562705e3c930 con 0x562705e21020 2024-05-20T20:08:21.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7face7e00700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e21940 0x562705e3c170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:21.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facdea00700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e2de10 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7facd00aa150 tx=0x7facd00e31d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a401f937eb1c25f6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:21.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facd5600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:21.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facd5600700 1 -- 172.21.15.28:0/117411606 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7facd00a9930 con 0x562705e21020 2024-05-20T20:08:21.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facd5600700 1 -- 172.21.15.28:0/117411606 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7facd000b560 con 0x562705e21020 2024-05-20T20:08:21.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.715+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x562705d75570 con 0x562705e21020 2024-05-20T20:08:21.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facd5600700 1 -- 172.21.15.28:0/117411606 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7facd00e0a30 con 0x562705e21020 2024-05-20T20:08:21.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 10 client.26042 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562705e9f710 con 0x562705e21020 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facd5600700 1 -- 172.21.15.28:0/117411606 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x562705d75570 con 0x562705e21020 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facd5600700 1 -- 172.21.15.28:0/117411606 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7facd00e6040 con 0x562705e21020 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 10 client.26042 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 10 client.26042 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 20 client.26042 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e9fdf0 0x562705ea2210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 4 client.26042 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.716+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x562705bec250 con 0x562705e9fdf0 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.717+0000 7face7400700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e9fdf0 0x562705ea2210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.717+0000 7face7400700 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e9fdf0 0x562705ea2210 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=21e9afb0e274c09a server_cookie=c0b4400ff0e5c922 in_seq=0 out_seq=0 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.717+0000 7facd5600700 10 client.26042 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:08:21.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facd5600700 1 -- 172.21.15.28:0/117411606 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x562705bec250 con 0x562705e9fdf0 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facd5600700 10 client.26042 handle_command_reply: tid=0 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 client.26042 shutdown 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 msgr2=0x562705e2de10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e2de10 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7facd00aa150 tx=0x7facd00e31d0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 shutdown_connections 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x562705e9fdf0 0x562705ea2210 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562705e21940 0x562705e3c170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562705e1bad0 0x562705e2d8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 --2- 172.21.15.28:0/117411606 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562705e21020 0x562705e2de10 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 >> 172.21.15.28:0/117411606 conn(0x562705d49be0 msgr2=0x562705e1eb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facd5600700 0 client.26042 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 shutdown_connections 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 1 -- 172.21.15.28:0/117411606 wait complete. 2024-05-20T20:08:21.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.718+0000 7facf8e05b80 20 client.26042 trim_cache size 0 max 16384 2024-05-20T20:08:21.722 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:08:21.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faccc0b2a70 msgr2=0x7faccc0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faccc0b2a70 0x7faccc0b4f30 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7face0017fd0 tx=0x7face0015f20 comp rx=0 tx=0).stop 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7face8103e50 msgr2=0x7face8075d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7face8103e50 0x7face8075d80 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7facd0000c00 tx=0x7facd000f950 comp rx=0 tx=0).stop 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 shutdown_connections 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7faccc0b2a70 0x7faccc0b4f30 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7face8109540 0x7face80762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7face8103490 0x7face8075840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 --2- 172.21.15.28:0/4111285437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7face8103e50 0x7face8075d80 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 >> 172.21.15.28:0/4111285437 conn(0x7face80fefc0 msgr2=0x7face8100840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:21.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 shutdown_connections 2024-05-20T20:08:21.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:21.719+0000 7facece00700 1 -- 172.21.15.28:0/4111285437 wait complete. 2024-05-20T20:08:22.690 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:08:23.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:22 smithi028 ceph-mon[31178]: pgmap v85: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-05-20T20:08:23.087 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:08:23.205 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:23.205 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:22 smithi033 ceph-mon[36179]: pgmap v85: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-05-20T20:08:23.205 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:22 smithi154 ceph-mon[36314]: pgmap v85: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-05-20T20:08:23.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.727+0000 7f5e24200640 1 -- 172.21.15.28:0/4165783180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c102a60 msgr2=0x7f5e1c102ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.727+0000 7f5e24200640 1 --2- 172.21.15.28:0/4165783180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c102a60 0x7f5e1c102ee0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f5e08009860 tx=0x7f5e0802a340 comp rx=0 tx=0).stop 2024-05-20T20:08:23.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.727+0000 7f5e24200640 1 -- 172.21.15.28:0/4165783180 shutdown_connections 2024-05-20T20:08:23.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.727+0000 7f5e24200640 1 --2- 172.21.15.28:0/4165783180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c102a60 0x7f5e1c102ee0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.727+0000 7f5e24200640 1 --2- 172.21.15.28:0/4165783180 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c103420 0x7f5e1c10f930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.727+0000 7f5e24200640 1 --2- 172.21.15.28:0/4165783180 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5e1c108a70 0x7f5e1c108e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.727+0000 7f5e24200640 1 -- 172.21.15.28:0/4165783180 >> 172.21.15.28:0/4165783180 conn(0x7f5e1c079ee0 msgr2=0x7f5e1c07a2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:23.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.728+0000 7f5e24200640 1 -- 172.21.15.28:0/4165783180 shutdown_connections 2024-05-20T20:08:23.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.728+0000 7f5e24200640 1 -- 172.21.15.28:0/4165783180 wait complete. 2024-05-20T20:08:23.731 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 Processor -- start 2024-05-20T20:08:23.731 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 -- start start 2024-05-20T20:08:23.731 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c102a60 0x7f5e1c19b340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5e1c103420 0x7f5e1c19b880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c108a70 0x7f5e1c19fc00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c112d30 con 0x7f5e1c102a60 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c112bb0 con 0x7f5e1c108a70 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e24200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c112eb0 con 0x7f5e1c103420 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e21800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c102a60 0x7f5e1c19b340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e22200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c108a70 0x7f5e1c19fc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e21800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c102a60 0x7f5e1c19b340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42656/0 (socket says 172.21.15.28:42656) 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.729+0000 7f5e21800640 1 -- 172.21.15.28:0/3547072047 learned_addr learned my addr 172.21.15.28:0/3547072047 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e20e00640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5e1c103420 0x7f5e1c19b880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e22200640 1 -- 172.21.15.28:0/3547072047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5e1c103420 msgr2=0x7f5e1c19b880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e22200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5e1c103420 0x7f5e1c19b880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e22200640 1 -- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c102a60 msgr2=0x7f5e1c19b340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e22200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c102a60 0x7f5e1c19b340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e22200640 1 -- 172.21.15.28:0/3547072047 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5e1c1a0330 con 0x7f5e1c108a70 2024-05-20T20:08:23.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e21800640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c102a60 0x7f5e1c19b340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:23.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.730+0000 7f5e22200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c108a70 0x7f5e1c19fc00 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f5e1400ee70 tx=0x7f5e1400c740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:23.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.731+0000 7f5e1a000640 1 -- 172.21.15.28:0/3547072047 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e140099a0 con 0x7f5e1c108a70 2024-05-20T20:08:23.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.731+0000 7f5e1a000640 1 -- 172.21.15.28:0/3547072047 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5e14003970 con 0x7f5e1c108a70 2024-05-20T20:08:23.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.731+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5e1c1a0620 con 0x7f5e1c108a70 2024-05-20T20:08:23.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.731+0000 7f5e1a000640 1 -- 172.21.15.28:0/3547072047 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e14004750 con 0x7f5e1c108a70 2024-05-20T20:08:23.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.731+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5e1c1a7eb0 con 0x7f5e1c108a70 2024-05-20T20:08:23.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.734+0000 7f5e2702c740 1 Processor -- start 2024-05-20T20:08:23.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.736+0000 7f5e1a000640 1 -- 172.21.15.28:0/3547072047 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f5e1401d020 con 0x7f5e1c108a70 2024-05-20T20:08:23.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.737+0000 7f5e2702c740 1 -- start start 2024-05-20T20:08:23.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.737+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c5c070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.737+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c66930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.737+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c69260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.737+0000 7f5e2702c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55adc2bccb90 con 0x55adc2c66e70 2024-05-20T20:08:23.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.737+0000 7f5e2702c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55adc2bbc730 con 0x55adc2c5c5b0 2024-05-20T20:08:23.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.737+0000 7f5e2702c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55adc2bc0770 con 0x55adc2c5bc90 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c66930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e20e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c69260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c66930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43208/0 (socket says 172.21.15.28:43208) 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 -- 172.21.15.28:0/1368695266 learned_addr learned my addr 172.21.15.28:0/1368695266 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e20e00640 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c5c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 -- 172.21.15.28:0/1368695266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 msgr2=0x55adc2c5c070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c5c070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 -- 172.21.15.28:0/1368695266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 msgr2=0x55adc2c69260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c69260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.738+0000 7f5e21800640 1 -- 172.21.15.28:0/1368695266 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55adc2b558b0 con 0x55adc2c5c5b0 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.739+0000 7f5e1a000640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5e040b2a10 0x7f5e040b4ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.739+0000 7f5e21800640 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c66930 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f5e100115f0 tx=0x7f5e10011ab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c6fd554127de4464 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.739+0000 7f5e1a000640 1 -- 172.21.15.28:0/3547072047 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5e140e2490 con 0x7f5e1c108a70 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.739+0000 7f5e0ea00640 1 -- 172.21.15.28:0/1368695266 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e1001e070 con 0x55adc2c5c5b0 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.739+0000 7f5e0ea00640 1 -- 172.21.15.28:0/1368695266 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5e1000fda0 con 0x55adc2c5c5b0 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.739+0000 7f5e0ea00640 1 -- 172.21.15.28:0/1368695266 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e10019df0 con 0x55adc2c5c5b0 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.739+0000 7f5e20e00640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5e040b2a10 0x7f5e040b4ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e20e00640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5e040b2a10 0x7f5e040b4ed0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f5e0802a930 tx=0x7f5e08000bc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 -- 172.21.15.28:0/1368695266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 msgr2=0x55adc2c66930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c66930 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f5e100115f0 tx=0x7f5e10011ab0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 -- 172.21.15.28:0/1368695266 shutdown_connections 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c66930 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c69260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1368695266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c5c070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 -- 172.21.15.28:0/1368695266 >> 172.21.15.28:0/1368695266 conn(0x55adc2c581a0 msgr2=0x55adc2c58580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 -- 172.21.15.28:0/1368695266 shutdown_connections 2024-05-20T20:08:23.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.740+0000 7f5e2702c740 1 -- 172.21.15.28:0/1368695266 wait complete. 2024-05-20T20:08:23.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.741+0000 7f5e2702c740 1 -- start start 2024-05-20T20:08:23.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.741+0000 7f5e2702c740 1 Processor -- start 2024-05-20T20:08:23.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c66860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5c5b0 0x55adc2c63b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c66e70 0x55adc2c64160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e2702c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55adc28a2790 con 0x55adc2c5c5b0 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e2702c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55adc2b64bc0 con 0x55adc2c66e70 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e2702c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55adc2c69aa0 con 0x55adc2c5bc90 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e2702c740 10 client.0 _ll_register_callbacks cb 0x55adc2b42400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e20e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5c5b0 0x55adc2c63b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e21800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c66860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e20e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5c5b0 0x55adc2c63b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42674/0 (socket says 172.21.15.28:42674) 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e20e00640 1 -- 172.21.15.28:0/1136728979 learned_addr learned my addr 172.21.15.28:0/1136728979 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.742+0000 7f5e22200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c66e70 0x55adc2c64160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.743+0000 7f5e21800640 1 -- 172.21.15.28:0/1136728979 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c66e70 msgr2=0x55adc2c64160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.743+0000 7f5e21800640 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c66e70 0x55adc2c64160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.743+0000 7f5e21800640 1 -- 172.21.15.28:0/1136728979 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5c5b0 msgr2=0x55adc2c63b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.743+0000 7f5e21800640 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5c5b0 0x55adc2c63b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.743+0000 7f5e21800640 1 -- 172.21.15.28:0/1136728979 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55adc28d1210 con 0x55adc2c5bc90 2024-05-20T20:08:23.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.743+0000 7f5e21800640 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c66860 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f5e1001c040 tx=0x7f5e1001adb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2a0e5781feed8004 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:23.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.743+0000 7f5e03400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:23.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.744+0000 7f5e03400640 1 -- 172.21.15.28:0/1136728979 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e1001e050 con 0x55adc2c5bc90 2024-05-20T20:08:23.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.744+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5e1000fda0 con 0x55adc2c5bc90 2024-05-20T20:08:23.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.744+0000 7f5e03400640 1 -- 172.21.15.28:0/1136728979 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5e1000fb00 con 0x55adc2c5bc90 2024-05-20T20:08:23.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.744+0000 7f5e03400640 1 -- 172.21.15.28:0/1136728979 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5e10033050 con 0x55adc2c5bc90 2024-05-20T20:08:23.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.744+0000 7f5e2702c740 10 client.26008 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:23.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.744+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55adc299d620 con 0x55adc2c5bc90 2024-05-20T20:08:23.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.744+0000 7f5e03400640 1 -- 172.21.15.28:0/1136728979 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5e1000fda0 con 0x55adc2c5bc90 2024-05-20T20:08:23.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e03400640 1 -- 172.21.15.28:0/1136728979 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5e10004730 con 0x55adc2c5bc90 2024-05-20T20:08:23.746 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e2702c740 10 client.26008 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e2702c740 10 client.26008 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e2702c740 20 client.26008 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2c76720 0x55adc2c78b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e2702c740 4 client.26008 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55adc2bceae0 con 0x55adc2c76720 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.745+0000 7f5e20e00640 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2c76720 0x55adc2c78b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.746+0000 7f5e20e00640 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2c76720 0x55adc2c78b10 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1b2591dd9280e006 server_cookie=2962d8cf8b85883b in_seq=0 out_seq=0 2024-05-20T20:08:23.747 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.746+0000 7f5e03400640 10 client.26008 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:23.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e03400640 1 -- 172.21.15.28:0/1136728979 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55adc2bceae0 con 0x55adc2c76720 2024-05-20T20:08:23.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e03400640 10 client.26008 handle_command_reply: tid=0 2024-05-20T20:08:23.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 client.26008 shutdown 2024-05-20T20:08:23.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 msgr2=0x55adc2c66860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c66860 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f5e1001c040 tx=0x7f5e1001adb0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 shutdown_connections 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2c76720 0x55adc2c78b10 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c66e70 0x55adc2c64160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5c5b0 0x55adc2c63b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 --2- 172.21.15.28:0/1136728979 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c66860 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e03400640 0 client.26008 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.747+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 >> 172.21.15.28:0/1136728979 conn(0x55adc2c581a0 msgr2=0x55adc2c5ded0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.748+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 shutdown_connections 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.748+0000 7f5e2702c740 1 -- 172.21.15.28:0/1136728979 wait complete. 2024-05-20T20:08:23.749 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.748+0000 7f5e2702c740 20 client.26008 trim_cache size 0 max 16384 2024-05-20T20:08:23.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 Processor -- start 2024-05-20T20:08:23.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 -- start start 2024-05-20T20:08:23.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c72b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c52dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c7e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c1b0800 con 0x55adc2c66e70 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c1b2340 con 0x55adc2c5c5b0 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.773+0000 7f5e2702c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c1b0ed0 con 0x55adc2c5bc90 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c52dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e22200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c7e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c52dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43224/0 (socket says 172.21.15.28:43224) 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 -- 172.21.15.28:0/2488740694 learned_addr learned my addr 172.21.15.28:0/2488740694 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e21800640 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c72b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 -- 172.21.15.28:0/2488740694 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 msgr2=0x55adc2c72b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c72b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 -- 172.21.15.28:0/2488740694 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 msgr2=0x55adc2c7e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c7e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 -- 172.21.15.28:0/2488740694 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55adc2c644d0 con 0x55adc2c5c5b0 2024-05-20T20:08:23.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e21800640 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c72b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:23.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.774+0000 7f5e20e00640 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c52dd0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f5e08035040 tx=0x7f5e08005f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6cb6ad7e679799d3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:23.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.775+0000 7f5e0e000640 1 -- 172.21.15.28:0/2488740694 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e08037040 con 0x55adc2c5c5b0 2024-05-20T20:08:23.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.775+0000 7f5e0e000640 1 -- 172.21.15.28:0/2488740694 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5e08048040 con 0x55adc2c5c5b0 2024-05-20T20:08:23.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.775+0000 7f5e0e000640 1 -- 172.21.15.28:0/2488740694 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e08051960 con 0x55adc2c5c5b0 2024-05-20T20:08:23.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 -- 172.21.15.28:0/2488740694 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 msgr2=0x55adc2c52dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c52dd0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f5e08035040 tx=0x7f5e08005f90 comp rx=0 tx=0).stop 2024-05-20T20:08:23.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 -- 172.21.15.28:0/2488740694 shutdown_connections 2024-05-20T20:08:23.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c52dd0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c66e70 0x55adc2c7e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 --2- 172.21.15.28:0/2488740694 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c5bc90 0x55adc2c72b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 -- 172.21.15.28:0/2488740694 >> 172.21.15.28:0/2488740694 conn(0x55adc2c581a0 msgr2=0x55adc2c57310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 -- 172.21.15.28:0/2488740694 shutdown_connections 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 -- 172.21.15.28:0/2488740694 wait complete. 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 -- start start 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.776+0000 7f5e2702c740 1 Processor -- start 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5bc90 0x55adc2c6bd20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c6c260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e2702c740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c66e70 0x55adc2c6c7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e2702c740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c1b1c70 con 0x55adc2c5bc90 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e2702c740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c1b15a0 con 0x55adc2c5c5b0 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e2702c740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5e1c1ae7e0 con 0x55adc2c66e70 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e2702c740 10 client.0 _ll_register_callbacks cb 0x7f5e1001c070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e21800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5bc90 0x55adc2c6bd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e20e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c6c260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e21800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5bc90 0x55adc2c6bd20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42700/0 (socket says 172.21.15.28:42700) 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e21800640 1 -- 172.21.15.28:0/3565946730 learned_addr learned my addr 172.21.15.28:0/3565946730 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e20e00640 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c6c260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43238/0 (socket says 172.21.15.28:43238) 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.777+0000 7f5e22200640 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c66e70 0x55adc2c6c7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e21800640 1 -- 172.21.15.28:0/3565946730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c66e70 msgr2=0x55adc2c6c7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e21800640 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c66e70 0x55adc2c6c7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e21800640 1 -- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 msgr2=0x55adc2c6c260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e21800640 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c6c260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e21800640 1 -- 172.21.15.28:0/3565946730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55adc2ce7cc0 con 0x55adc2c5bc90 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e21800640 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5bc90 0x55adc2c6bd20 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f5e100059f0 tx=0x7f5e10005f80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9bc442a7f4d9a7e2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e03400640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e03400640 1 -- 172.21.15.28:0/3565946730 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e1001e040 con 0x55adc2c5bc90 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e03400640 1 -- 172.21.15.28:0/3565946730 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f5e10004880 con 0x55adc2c5bc90 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5e08048040 con 0x55adc2c5bc90 2024-05-20T20:08:23.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e03400640 1 -- 172.21.15.28:0/3565946730 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5e10047050 con 0x55adc2c5bc90 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.778+0000 7f5e2702c740 10 client.15915 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55adc2ce8fd0 con 0x55adc2c5bc90 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e03400640 1 -- 172.21.15.28:0/3565946730 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5e08048040 con 0x55adc2c5bc90 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e03400640 1 -- 172.21.15.28:0/3565946730 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5e1000fac0 con 0x55adc2c5bc90 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e2702c740 10 client.15915 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e2702c740 10 client.15915 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e2702c740 20 client.15915 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e2702c740 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2ce9620 0x55adc2ceba40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e2702c740 4 client.15915 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55adc2bc82c0 con 0x55adc2ce9620 2024-05-20T20:08:23.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.779+0000 7f5e20e00640 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2ce9620 0x55adc2ceba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:23.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.780+0000 7f5e20e00640 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2ce9620 0x55adc2ceba40 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=50e956ce4ae01cc9 server_cookie=cac6e4876472c628 in_seq=0 out_seq=0 2024-05-20T20:08:23.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.780+0000 7f5e03400640 10 client.15915 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:23.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e03400640 1 -- 172.21.15.28:0/3565946730 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55adc2bc82c0 con 0x55adc2ce9620 2024-05-20T20:08:23.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e03400640 10 client.15915 handle_command_reply: tid=0 2024-05-20T20:08:23.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 client.15915 shutdown 2024-05-20T20:08:23.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5bc90 msgr2=0x55adc2c6bd20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5bc90 0x55adc2c6bd20 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f5e100059f0 tx=0x7f5e10005f80 comp rx=0 tx=0).stop 2024-05-20T20:08:23.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 shutdown_connections 2024-05-20T20:08:23.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55adc2ce9620 0x55adc2ceba40 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55adc2c5c5b0 0x55adc2c6c260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55adc2c66e70 0x55adc2c6c7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 --2- 172.21.15.28:0/3565946730 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55adc2c5bc90 0x55adc2c6bd20 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e03400640 0 client.15915 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 >> 172.21.15.28:0/3565946730 conn(0x55adc2c581a0 msgr2=0x55adc2c65c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 shutdown_connections 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.784+0000 7f5e2702c740 1 -- 172.21.15.28:0/3565946730 wait complete. 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e2702c740 20 client.15915 trim_cache size 0 max 16384 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:08:23.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:08:23.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5e040b2a10 msgr2=0x7f5e040b4ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5e040b2a10 0x7f5e040b4ed0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f5e0802a930 tx=0x7f5e08000bc0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c108a70 msgr2=0x7f5e1c19fc00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:23.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c108a70 0x7f5e1c19fc00 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f5e1400ee70 tx=0x7f5e1400c740 comp rx=0 tx=0).stop 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 shutdown_connections 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5e1c108a70 0x7f5e1c19fc00 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5e040b2a10 0x7f5e040b4ed0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5e1c103420 0x7f5e1c19b880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 --2- 172.21.15.28:0/3547072047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5e1c102a60 0x7f5e1c19b340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.785+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 >> 172.21.15.28:0/3547072047 conn(0x7f5e1c079ee0 msgr2=0x7f5e1c10dcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.786+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 shutdown_connections 2024-05-20T20:08:23.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:23.786+0000 7f5e24200640 1 -- 172.21.15.28:0/3547072047 wait complete. 2024-05-20T20:08:24.718 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:25.011 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:24 smithi028 ceph-mon[31178]: pgmap v86: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 1.4 KiB/s wr, 13 op/s 2024-05-20T20:08:25.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:24 smithi154 ceph-mon[36314]: pgmap v86: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 1.4 KiB/s wr, 13 op/s 2024-05-20T20:08:25.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:24 smithi033 ceph-mon[36179]: pgmap v86: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 1.4 KiB/s wr, 13 op/s 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.009+0000 7fa943200700 1 -- 172.21.15.28:0/1137476543 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 msgr2=0x7fa93c110310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.009+0000 7fa943200700 1 --2- 172.21.15.28:0/1137476543 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 0x7fa93c110310 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fa93800b0d0 tx=0x7fa93800b3e0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.009+0000 7fa943200700 1 -- 172.21.15.28:0/1137476543 shutdown_connections 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.009+0000 7fa943200700 1 --2- 172.21.15.28:0/1137476543 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 0x7fa93c110310 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.009+0000 7fa943200700 1 --2- 172.21.15.28:0/1137476543 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa93c109540 0x7fa93c109920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.009+0000 7fa943200700 1 --2- 172.21.15.28:0/1137476543 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 0x7fa93c103910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.009+0000 7fa943200700 1 -- 172.21.15.28:0/1137476543 >> 172.21.15.28:0/1137476543 conn(0x7fa93c0fefc0 msgr2=0x7fa93c1013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.010+0000 7fa943200700 1 -- 172.21.15.28:0/1137476543 shutdown_connections 2024-05-20T20:08:26.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.010+0000 7fa943200700 1 -- 172.21.15.28:0/1137476543 wait complete. 2024-05-20T20:08:26.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.011+0000 7fa943200700 1 Processor -- start 2024-05-20T20:08:26.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.011+0000 7fa943200700 1 -- start start 2024-05-20T20:08:26.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa943200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 0x7fa93c19bc80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa943200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 0x7fa93c19c1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa943200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa93c109540 0x7fa93c1a0580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa943200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c113620 con 0x7fa93c103e50 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa943200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1134a0 con 0x7fa93c103490 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa943200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1137a0 con 0x7fa93c109540 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa942800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa93c109540 0x7fa93c1a0580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa941e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 0x7fa93c19bc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa941e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 0x7fa93c19bc80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43260/0 (socket says 172.21.15.28:43260) 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa941e00700 1 -- 172.21.15.28:0/3116318799 learned_addr learned my addr 172.21.15.28:0/3116318799 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.012+0000 7fa941400700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 0x7fa93c19c1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.013+0000 7fa941e00700 1 -- 172.21.15.28:0/3116318799 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa93c109540 msgr2=0x7fa93c1a0580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.013+0000 7fa941e00700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa93c109540 0x7fa93c1a0580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.013+0000 7fa941e00700 1 -- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 msgr2=0x7fa93c19c1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.013+0000 7fa941e00700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 0x7fa93c19c1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.013+0000 7fa941e00700 1 -- 172.21.15.28:0/3116318799 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa93c1a0c80 con 0x7fa93c103490 2024-05-20T20:08:26.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.013+0000 7fa941e00700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 0x7fa93c19bc80 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fa93000e9a0 tx=0x7fa93000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:26.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.014+0000 7fa936a00700 1 -- 172.21.15.28:0/3116318799 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa93000ca10 con 0x7fa93c103490 2024-05-20T20:08:26.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.014+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa93c1a0f70 con 0x7fa93c103490 2024-05-20T20:08:26.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.014+0000 7fa936a00700 1 -- 172.21.15.28:0/3116318799 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa93000cbb0 con 0x7fa93c103490 2024-05-20T20:08:26.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.014+0000 7fa936a00700 1 -- 172.21.15.28:0/3116318799 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa930018680 con 0x7fa93c103490 2024-05-20T20:08:26.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.014+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa93c1a8830 con 0x7fa93c103490 2024-05-20T20:08:26.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.018+0000 7fa936a00700 1 -- 172.21.15.28:0/3116318799 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa930010430 con 0x7fa93c103490 2024-05-20T20:08:26.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.019+0000 7fa94f0fcb80 1 Processor -- start 2024-05-20T20:08:26.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.019+0000 7fa94f0fcb80 1 -- start start 2024-05-20T20:08:26.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.019+0000 7fa936a00700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9240b29f0 0x7fa9240b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa936a00700 1 -- 172.21.15.28:0/3116318799 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa9300e2780 con 0x7fa93c103490 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6a7c0 0x559b33d6aba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d64d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d657f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559b33cc2e40 con 0x559b33d6b0e0 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x559b33cb29e0 con 0x559b33d65390 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559b33cb6a20 con 0x559b33d6a7c0 2024-05-20T20:08:26.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa942800700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9240b29f0 0x7fa9240b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa941e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6a7c0 0x559b33d6aba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa941e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6a7c0 0x559b33d6aba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52522/0 (socket says 172.21.15.28:52522) 2024-05-20T20:08:26.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa941400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d64d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa941e00700 1 -- 172.21.15.28:0/3961452557 learned_addr learned my addr 172.21.15.28:0/3961452557 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:26.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.020+0000 7fa942800700 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d657f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa941e00700 1 -- 172.21.15.28:0/3961452557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 msgr2=0x559b33d657f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa941e00700 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d657f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa941e00700 1 -- 172.21.15.28:0/3961452557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 msgr2=0x559b33d64d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa941e00700 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d64d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa941e00700 1 -- 172.21.15.28:0/3961452557 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559b33c7b420 con 0x559b33d6a7c0 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa941e00700 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6a7c0 0x559b33d6aba0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fa930009230 tx=0x7fa9300a95a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=eb0f5f885057f88f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa942800700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9240b29f0 0x7fa9240b4eb0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fa938005bd0 tx=0x7fa938005a60 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa92b400700 1 -- 172.21.15.28:0/3961452557 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9300a9a10 con 0x559b33d6a7c0 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa92b400700 1 -- 172.21.15.28:0/3961452557 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9300addc0 con 0x559b33d6a7c0 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.021+0000 7fa92b400700 1 -- 172.21.15.28:0/3961452557 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9300cc0b0 con 0x559b33d6a7c0 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/3961452557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6a7c0 msgr2=0x559b33d6aba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6a7c0 0x559b33d6aba0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fa930009230 tx=0x7fa9300a95a0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/3961452557 shutdown_connections 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d657f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6a7c0 0x559b33d6aba0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/3961452557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d64d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/3961452557 >> 172.21.15.28:0/3961452557 conn(0x559b33c92be0 msgr2=0x559b33d5f810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:26.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/3961452557 shutdown_connections 2024-05-20T20:08:26.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.022+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/3961452557 wait complete. 2024-05-20T20:08:26.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.023+0000 7fa94f0fcb80 1 -- start start 2024-05-20T20:08:26.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.023+0000 7fa94f0fcb80 1 Processor -- start 2024-05-20T20:08:26.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d70f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d6e300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6b0e0 0x559b33d6e840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559b33bb8fa0 con 0x559b33d6a7c0 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x559b33c53760 con 0x559b33d6b0e0 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559b33d71c00 con 0x559b33d65390 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa94f0fcb80 10 client.0 _ll_register_callbacks cb 0x559b33c5cc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa941e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d70f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa941400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d6e300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa941e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d70f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52526/0 (socket says 172.21.15.28:52526) 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa941e00700 1 -- 172.21.15.28:0/817471202 learned_addr learned my addr 172.21.15.28:0/817471202 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa941400700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d6e300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42742/0 (socket says 172.21.15.28:42742) 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.024+0000 7fa942800700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6b0e0 0x559b33d6e840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa941400700 1 -- 172.21.15.28:0/817471202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 msgr2=0x559b33d70f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa941400700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d70f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa941400700 1 -- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6b0e0 msgr2=0x559b33d6e840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa941400700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6b0e0 0x559b33d6e840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa941400700 1 -- 172.21.15.28:0/817471202 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559b33c028d0 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa941e00700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d70f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa941400700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d6e300 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fa93c19cbf0 tx=0x7fa92c012960 comp rx=0 tx=0).ready entity=mon.0 client_cookie=126011cc78e11f1d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa91fe00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa91fe00700 1 -- 172.21.15.28:0/817471202 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa92c004830 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa91fe00700 1 -- 172.21.15.28:0/817471202 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa92c0049d0 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.025+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa9300addc0 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa91fe00700 1 -- 172.21.15.28:0/817471202 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa92c004bd0 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 10 client.15927 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559b33c83060 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa91fe00700 1 -- 172.21.15.28:0/817471202 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9300addc0 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa91fe00700 1 -- 172.21.15.28:0/817471202 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa92c01c630 con 0x559b33d6a7c0 2024-05-20T20:08:26.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 10 client.15927 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:26.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 10 client.15927 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:26.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 20 client.15927 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:26.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33d7e850 0x559b33d80c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 4 client.15927 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:26.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.026+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559b33cc4d90 con 0x559b33d7e850 2024-05-20T20:08:26.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.027+0000 7fa941e00700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33d7e850 0x559b33d80c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.027+0000 7fa941e00700 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33d7e850 0x559b33d80c40 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=65c337f0d00feda5 server_cookie=adcdc6ab8a27ef8b in_seq=0 out_seq=0 2024-05-20T20:08:26.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.027+0000 7fa91fe00700 10 client.15927 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:26.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.028+0000 7fa91fe00700 1 -- 172.21.15.28:0/817471202 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x559b33cc4d90 con 0x559b33d7e850 2024-05-20T20:08:26.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.028+0000 7fa91fe00700 10 client.15927 handle_command_reply: tid=0 2024-05-20T20:08:26.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.028+0000 7fa94f0fcb80 1 client.15927 shutdown 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 msgr2=0x559b33d6e300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d6e300 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fa93c19cbf0 tx=0x7fa92c012960 comp rx=0 tx=0).stop 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 shutdown_connections 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33d7e850 0x559b33d80c40 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6b0e0 0x559b33d6e840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d70f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa91fe00700 0 client.15927 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/817471202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d6e300 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 >> 172.21.15.28:0/817471202 conn(0x559b33c92be0 msgr2=0x559b33d5efc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 shutdown_connections 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/817471202 wait complete. 2024-05-20T20:08:26.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.029+0000 7fa94f0fcb80 20 client.15927 trim_cache size 0 max 16384 2024-05-20T20:08:26.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.046+0000 7fa94f0fcb80 1 Processor -- start 2024-05-20T20:08:26.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.046+0000 7fa94f0fcb80 1 -- start start 2024-05-20T20:08:26.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d7aca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6a7c0 0x559b33d69cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d688b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1aff80 con 0x559b33d6b0e0 2024-05-20T20:08:26.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1b1ac0 con 0x559b33d6a7c0 2024-05-20T20:08:26.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1b0650 con 0x559b33d65390 2024-05-20T20:08:26.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d7aca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa942800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d688b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d7aca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52532/0 (socket says 172.21.15.28:52532) 2024-05-20T20:08:26.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941e00700 1 -- 172.21.15.28:0/2206692472 learned_addr learned my addr 172.21.15.28:0/2206692472 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:26.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941400700 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6a7c0 0x559b33d69cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941e00700 1 -- 172.21.15.28:0/2206692472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6a7c0 msgr2=0x559b33d69cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941e00700 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6a7c0 0x559b33d69cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941e00700 1 -- 172.21.15.28:0/2206692472 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 msgr2=0x559b33d688b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.047+0000 7fa941e00700 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d688b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.048+0000 7fa941e00700 1 -- 172.21.15.28:0/2206692472 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559b33d6eb70 con 0x559b33d65390 2024-05-20T20:08:26.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.048+0000 7fa941e00700 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d7aca0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fa93000e970 tx=0x7fa930010c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f822cb240805f511 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:26.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.048+0000 7fa92aa00700 1 -- 172.21.15.28:0/2206692472 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9300bee60 con 0x559b33d65390 2024-05-20T20:08:26.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.048+0000 7fa92aa00700 1 -- 172.21.15.28:0/2206692472 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9300addc0 con 0x559b33d65390 2024-05-20T20:08:26.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.048+0000 7fa92aa00700 1 -- 172.21.15.28:0/2206692472 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa930010720 con 0x559b33d65390 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/2206692472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 msgr2=0x559b33d7aca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d7aca0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fa93000e970 tx=0x7fa930010c10 comp rx=0 tx=0).stop 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/2206692472 shutdown_connections 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6b0e0 0x559b33d688b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d65390 0x559b33d7aca0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/2206692472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d6a7c0 0x559b33d69cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/2206692472 >> 172.21.15.28:0/2206692472 conn(0x559b33c92be0 msgr2=0x559b33d6b4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/2206692472 shutdown_connections 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.049+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/2206692472 wait complete. 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.050+0000 7fa94f0fcb80 1 -- start start 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.050+0000 7fa94f0fcb80 1 Processor -- start 2024-05-20T20:08:26.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.050+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d77270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.050+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d85b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.050+0000 7fa94f0fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6b0e0 0x559b33d86040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1b13f0 con 0x559b33d6a7c0 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1b0d20 con 0x559b33d65390 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa94f0fcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa93c1adf20 con 0x559b33d6b0e0 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa94f0fcb80 10 client.0 _ll_register_callbacks cb 0x559b33c5cc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d85b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d85b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42754/0 (socket says 172.21.15.28:42754) 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941400700 1 -- 172.21.15.28:0/576584442 learned_addr learned my addr 172.21.15.28:0/576584442 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941e00700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d77270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa942800700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6b0e0 0x559b33d86040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941e00700 1 -- 172.21.15.28:0/576584442 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6b0e0 msgr2=0x559b33d86040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941e00700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6b0e0 0x559b33d86040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941e00700 1 -- 172.21.15.28:0/576584442 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 msgr2=0x559b33d85b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941e00700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d85b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941e00700 1 -- 172.21.15.28:0/576584442 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559b33d86580 con 0x559b33d65390 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.051+0000 7fa941400700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d85b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa941e00700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d77270 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fa930012020 tx=0x7fa930009790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7f7df59a4f18d92e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa91fe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa91fe00700 1 -- 172.21.15.28:0/576584442 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9300bee60 con 0x559b33d65390 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa91fe00700 1 -- 172.21.15.28:0/576584442 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa9300a9e70 con 0x559b33d65390 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa9300addc0 con 0x559b33d65390 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa91fe00700 1 -- 172.21.15.28:0/576584442 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa9300cc360 con 0x559b33d65390 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa94f0fcb80 10 client.26078 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559b33de9150 con 0x559b33d65390 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.052+0000 7fa91fe00700 1 -- 172.21.15.28:0/576584442 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9300addc0 con 0x559b33d65390 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa91fe00700 1 -- 172.21.15.28:0/576584442 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa9300a5580 con 0x559b33d65390 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa94f0fcb80 10 client.26078 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa94f0fcb80 10 client.26078 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa94f0fcb80 20 client.26078 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33de9320 0x559b33deb710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa94f0fcb80 4 client.26078 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x559b33cbe570 con 0x559b33de9320 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.053+0000 7fa941400700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33de9320 0x559b33deb710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa941400700 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33de9320 0x559b33deb710 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c27bc0c10b6bd2c5 server_cookie=afb390a82cf089fa in_seq=0 out_seq=0 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa91fe00700 10 client.26078 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa91fe00700 1 -- 172.21.15.28:0/576584442 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x559b33cbe570 con 0x559b33de9320 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa91fe00700 10 client.26078 handle_command_reply: tid=0 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 client.26078 shutdown 2024-05-20T20:08:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 msgr2=0x559b33d77270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d77270 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fa930012020 tx=0x7fa930009790 comp rx=0 tx=0).stop 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 shutdown_connections 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559b33de9320 0x559b33deb710 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559b33d6b0e0 0x559b33d86040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559b33d65390 0x559b33d77270 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 --2- 172.21.15.28:0/576584442 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559b33d6a7c0 0x559b33d85b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 >> 172.21.15.28:0/576584442 conn(0x559b33c92be0 msgr2=0x559b33d6bd50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.054+0000 7fa91fe00700 0 client.26078 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.055+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 shutdown_connections 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.055+0000 7fa94f0fcb80 1 -- 172.21.15.28:0/576584442 wait complete. 2024-05-20T20:08:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.055+0000 7fa94f0fcb80 20 client.26078 trim_cache size 0 max 16384 2024-05-20T20:08:26.058 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:08:26.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.055+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9240b29f0 msgr2=0x7fa9240b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.055+0000 7fa943200700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9240b29f0 0x7fa9240b4eb0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fa938005bd0 tx=0x7fa938005a60 comp rx=0 tx=0).stop 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.055+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 msgr2=0x7fa93c19bc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.055+0000 7fa943200700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 0x7fa93c19bc80 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fa93000e9a0 tx=0x7fa93000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 shutdown_connections 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa93c109540 0x7fa93c1a0580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9240b29f0 0x7fa9240b4eb0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa93c103490 0x7fa93c19bc80 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 --2- 172.21.15.28:0/3116318799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa93c103e50 0x7fa93c19c1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 >> 172.21.15.28:0/3116318799 conn(0x7fa93c0fefc0 msgr2=0x7fa93c100480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:26.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 shutdown_connections 2024-05-20T20:08:26.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:26.056+0000 7fa943200700 1 -- 172.21.15.28:0/3116318799 wait complete. 2024-05-20T20:08:27.059 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:26 smithi028 ceph-mon[31178]: pgmap v87: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:08:27.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:26 smithi154 ceph-mon[36314]: pgmap v87: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:08:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:26 smithi033 ceph-mon[36179]: pgmap v87: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:08:28.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 -- 172.21.15.28:0/955738395 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01037f0 msgr2=0x7f63b0103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:28.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 --2- 172.21.15.28:0/955738395 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01037f0 0x7f63b0103bd0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f63a000b0d0 tx=0x7f63a000b3e0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 -- 172.21.15.28:0/955738395 shutdown_connections 2024-05-20T20:08:28.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 --2- 172.21.15.28:0/955738395 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63b01088f0 0x7f63b0110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 --2- 172.21.15.28:0/955738395 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01037f0 0x7f63b0103bd0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 --2- 172.21.15.28:0/955738395 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01041a0 0x7f63b01081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 -- 172.21.15.28:0/955738395 >> 172.21.15.28:0/955738395 conn(0x7f63b007a6a0 msgr2=0x7f63b007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:28.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 -- 172.21.15.28:0/955738395 shutdown_connections 2024-05-20T20:08:28.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.112+0000 7f63b5c00700 1 -- 172.21.15.28:0/955738395 wait complete. 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.113+0000 7f63b5c00700 1 Processor -- start 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.113+0000 7f63b5c00700 1 -- start start 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63b5c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01037f0 0x7f63b019bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63b5c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63b01041a0 0x7f63b019c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63b5c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01088f0 0x7f63b01a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63b5c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f63b01136e0 con 0x7f63b01088f0 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63b5c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f63b0113560 con 0x7f63b01041a0 2024-05-20T20:08:28.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63b5c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f63b0113860 con 0x7f63b01037f0 2024-05-20T20:08:28.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63aea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01037f0 0x7f63b019bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:28.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63af400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01088f0 0x7f63b01a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:28.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63aea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01037f0 0x7f63b019bdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:48284/0 (socket says 172.21.15.28:48284) 2024-05-20T20:08:28.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63aea00700 1 -- 172.21.15.28:0/2569007078 learned_addr learned my addr 172.21.15.28:0/2569007078 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:28.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.114+0000 7f63a6000700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63b01041a0 0x7f63b019c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63aea00700 1 -- 172.21.15.28:0/2569007078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63b01041a0 msgr2=0x7f63b019c310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63aea00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63b01041a0 0x7f63b019c310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63aea00700 1 -- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01088f0 msgr2=0x7f63b01a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63aea00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01088f0 0x7f63b01a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63aea00700 1 -- 172.21.15.28:0/2569007078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f63b01a0de0 con 0x7f63b01037f0 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63aea00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01037f0 0x7f63b019bdd0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f63a0007a30 tx=0x7f63a0007a60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63a7e00700 1 -- 172.21.15.28:0/2569007078 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f63a001f070 con 0x7f63b01037f0 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f63b01a1070 con 0x7f63b01037f0 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63a7e00700 1 -- 172.21.15.28:0/2569007078 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f63a0004c50 con 0x7f63b01037f0 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.115+0000 7f63a7e00700 1 -- 172.21.15.28:0/2569007078 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f63a001b690 con 0x7f63b01037f0 2024-05-20T20:08:28.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.116+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f63b01a8950 con 0x7f63b01037f0 2024-05-20T20:08:28.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.116+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f63b006a230 con 0x7f63b01037f0 2024-05-20T20:08:28.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.119+0000 7f63a7e00700 1 -- 172.21.15.28:0/2569007078 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f63a0022070 con 0x7f63b01037f0 2024-05-20T20:08:28.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.121+0000 7f63a7e00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63940b2a70 0x7f63940b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:28.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.121+0000 7f63a7e00700 1 -- 172.21.15.28:0/2569007078 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f63a000f020 con 0x7f63b01037f0 2024-05-20T20:08:28.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.121+0000 7f63a6000700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63940b2a70 0x7f63940b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:28.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.122+0000 7f63a6000700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63940b2a70 0x7f63940b4f30 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f63b01108b0 tx=0x7f6398009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:28.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.123+0000 7f63a7e00700 1 -- 172.21.15.28:0/2569007078 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f63a0029020 con 0x7f63b01037f0 2024-05-20T20:08:28.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.424+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f63b01a8b20 con 0x7f63b01037f0 2024-05-20T20:08:28.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.425+0000 7f63a7e00700 1 -- 172.21.15.28:0/2569007078 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f63a00ab450 con 0x7f63b01037f0 2024-05-20T20:08:28.425 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:08:28.425 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.428+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63940b2a70 msgr2=0x7f63940b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.428+0000 7f63b5c00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63940b2a70 0x7f63940b4f30 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f63b01108b0 tx=0x7f6398009260 comp rx=0 tx=0).stop 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.428+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01037f0 msgr2=0x7f63b019bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.428+0000 7f63b5c00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01037f0 0x7f63b019bdd0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f63a0007a30 tx=0x7f63a0007a60 comp rx=0 tx=0).stop 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.428+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 shutdown_connections 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.429+0000 7f63b5c00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f63b01088f0 0x7f63b01a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.429+0000 7f63b5c00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f63b01037f0 0x7f63b019bdd0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.429+0000 7f63b5c00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63940b2a70 0x7f63940b4f30 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.429+0000 7f63b5c00700 1 --2- 172.21.15.28:0/2569007078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f63b01041a0 0x7f63b019c310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.429+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 >> 172.21.15.28:0/2569007078 conn(0x7f63b007a6a0 msgr2=0x7f63b0101630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:28.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.429+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 shutdown_connections 2024-05-20T20:08:28.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:28.429+0000 7f63b5c00700 1 -- 172.21.15.28:0/2569007078 wait complete. 2024-05-20T20:08:28.430 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:08:28.533 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:08:28.534 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:08:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:28 smithi154 ceph-mon[36314]: pgmap v88: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:08:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:28 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2569007078' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:29.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:28 smithi033 ceph-mon[36179]: pgmap v88: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:08:29.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:28 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2569007078' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:29.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:28 smithi028 ceph-mon[31178]: pgmap v88: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:08:29.300 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:28 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2569007078' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:29.375 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id a49aa3fd --release --await' 2024-05-20T20:08:29.375 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id a49aa3fd --release --await 2024-05-20T20:08:29.544 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:31.051 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:30 smithi028 ceph-mon[31178]: pgmap v89: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:08:31.071 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:30 smithi154 ceph-mon[36314]: pgmap v89: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:08:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:30 smithi033 ceph-mon[36179]: pgmap v89: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 255 B/s wr, 12 op/s 2024-05-20T20:08:32.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 -- 172.21.15.28:0/1073287026 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1037f0 msgr2=0x7f2e5c103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 --2- 172.21.15.28:0/1073287026 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1037f0 0x7f2e5c103bd0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f2e440098f0 tx=0x7f2e44009c00 comp rx=0 tx=0).stop 2024-05-20T20:08:32.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 -- 172.21.15.28:0/1073287026 shutdown_connections 2024-05-20T20:08:32.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 --2- 172.21.15.28:0/1073287026 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1088f0 0x7f2e5c110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 --2- 172.21.15.28:0/1073287026 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1037f0 0x7f2e5c103bd0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 --2- 172.21.15.28:0/1073287026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2e5c1041a0 0x7f2e5c1081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 -- 172.21.15.28:0/1073287026 >> 172.21.15.28:0/1073287026 conn(0x7f2e5c07a6a0 msgr2=0x7f2e5c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 -- 172.21.15.28:0/1073287026 shutdown_connections 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.057+0000 7f2e63800700 1 -- 172.21.15.28:0/1073287026 wait complete. 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.059+0000 7f2e63800700 1 Processor -- start 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.059+0000 7f2e63800700 1 -- start start 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e63800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2e5c1037f0 0x7f2e5c19be80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e63800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1041a0 0x7f2e5c19c3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e63800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1088f0 0x7f2e5c1a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e63800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c113730 con 0x7f2e5c1041a0 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e63800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1135b0 con 0x7f2e5c1037f0 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e63800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1138b0 con 0x7f2e5c1088f0 2024-05-20T20:08:32.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e53e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1041a0 0x7f2e5c19c3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e53e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1041a0 0x7f2e5c19c3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35296/0 (socket says 172.21.15.28:35296) 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e53e00700 1 -- 172.21.15.28:0/3339008666 learned_addr learned my addr 172.21.15.28:0/3339008666 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e60e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2e5c1037f0 0x7f2e5c19be80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e61800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1088f0 0x7f2e5c1a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e60e00700 1 -- 172.21.15.28:0/3339008666 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1088f0 msgr2=0x7f2e5c1a0750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.060+0000 7f2e60e00700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1088f0 0x7f2e5c1a0750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e60e00700 1 -- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1041a0 msgr2=0x7f2e5c19c3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e60e00700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1041a0 0x7f2e5c19c3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e60e00700 1 -- 172.21.15.28:0/3339008666 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e5c1a0e90 con 0x7f2e5c1037f0 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e53e00700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1041a0 0x7f2e5c19c3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e60e00700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2e5c1037f0 0x7f2e5c19be80 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f2e4400c010 tx=0x7f2e4400fa20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e5a000700 1 -- 172.21.15.28:0/3339008666 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e440050c0 con 0x7f2e5c1037f0 2024-05-20T20:08:32.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e5a000700 1 -- 172.21.15.28:0/3339008666 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2e4400b560 con 0x7f2e5c1037f0 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e5a000700 1 -- 172.21.15.28:0/3339008666 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e440039a0 con 0x7f2e5c1037f0 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.061+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e5c1a1120 con 0x7f2e5c1037f0 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.062+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e5c1a8a00 con 0x7f2e5c1037f0 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.065+0000 7f2e5a000700 1 -- 172.21.15.28:0/3339008666 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f2e4400b710 con 0x7f2e5c1037f0 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.066+0000 7f2e6e23ab80 1 Processor -- start 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e6e23ab80 1 -- start start 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e5a000700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2e3c0b2a70 0x7f2e3c0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e5a000700 1 -- 172.21.15.28:0/3339008666 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2e440e33f0 con 0x7f2e5c1037f0 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f42a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e61800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2e3c0b2a70 0x7f2e3c0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42fb0 0x55b182f3cba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f3d1c0 0x55b182f3d640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55b182e9aec0 con 0x55b182f3d1c0 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55b182e8aa60 con 0x55b182f42690 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.067+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55b182e8eaa0 con 0x55b182f42fb0 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f42a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e61800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f3d1c0 0x55b182f3d640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e53e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42fb0 0x55b182f3cba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f42a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51800/0 (socket says 172.21.15.28:51800) 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 -- 172.21.15.28:0/99014600 learned_addr learned my addr 172.21.15.28:0/99014600 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 -- 172.21.15.28:0/99014600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42fb0 msgr2=0x55b182f3cba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 --2- 172.21.15.28:0/99014600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42fb0 0x55b182f3cba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 -- 172.21.15.28:0/99014600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f3d1c0 msgr2=0x55b182f3d640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 --2- 172.21.15.28:0/99014600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f3d1c0 0x55b182f3d640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 -- 172.21.15.28:0/99014600 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b182e534a0 con 0x55b182f42690 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e61800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2e3c0b2a70 0x7f2e3c0b4f30 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f2e5400f870 tx=0x7f2e5400fc30 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:32.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.068+0000 7f2e60e00700 1 --2- 172.21.15.28:0/99014600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f42a70 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f2e440a6790 tx=0x7f2e440aa190 comp rx=0 tx=0).ready entity=mon.2 client_cookie=84c5f9d5b014cbf3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.069+0000 7f2e52000700 1 -- 172.21.15.28:0/99014600 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e440aae10 con 0x55b182f42690 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.069+0000 7f2e52000700 1 -- 172.21.15.28:0/99014600 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2e440aa710 con 0x55b182f42690 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.069+0000 7f2e52000700 1 -- 172.21.15.28:0/99014600 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e440920d0 con 0x55b182f42690 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.069+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/99014600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 msgr2=0x55b182f42a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.069+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/99014600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f42a70 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f2e440a6790 tx=0x7f2e440aa190 comp rx=0 tx=0).stop 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/99014600 shutdown_connections 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/99014600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f3d1c0 0x55b182f3d640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/99014600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f42a70 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/99014600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42fb0 0x55b182f3cba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/99014600 >> 172.21.15.28:0/99014600 conn(0x55b182e6ac60 msgr2=0x55b182f37710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/99014600 shutdown_connections 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/99014600 wait complete. 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 -- start start 2024-05-20T20:08:32.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.070+0000 7f2e6e23ab80 1 Processor -- start 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f48ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f46040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f46580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55b182d91020 con 0x55b182f42fb0 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55b182e2b7e0 con 0x55b182f3d1c0 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55b182f49820 con 0x55b182f42690 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e6e23ab80 10 client.0 _ll_register_callbacks cb 0x55b182e34cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e60e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f48ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e53e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f46040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e60e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f48ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51812/0 (socket says 172.21.15.28:51812) 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e60e00700 1 -- 172.21.15.28:0/480610482 learned_addr learned my addr 172.21.15.28:0/480610482 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.071+0000 7f2e61800700 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f46580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e60e00700 1 -- 172.21.15.28:0/480610482 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 msgr2=0x55b182f46040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e60e00700 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f46040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e60e00700 1 -- 172.21.15.28:0/480610482 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 msgr2=0x55b182f46580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e60e00700 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f46580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e60e00700 1 -- 172.21.15.28:0/480610482 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b182dda950 con 0x55b182f3d1c0 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e53e00700 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f46040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e60e00700 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f48ca0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f2e44009f90 tx=0x7f2e440a6c30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c596666193e0d7d9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e4aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e4aa00700 1 -- 172.21.15.28:0/480610482 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e440b34e0 con 0x55b182f3d1c0 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e4aa00700 1 -- 172.21.15.28:0/480610482 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2e440920d0 con 0x55b182f3d1c0 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.072+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2e440aa710 con 0x55b182f3d1c0 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e4aa00700 1 -- 172.21.15.28:0/480610482 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2e440922d0 con 0x55b182f3d1c0 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 10 client.26096 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:32.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55b182e5b0e0 con 0x55b182f3d1c0 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e4aa00700 1 -- 172.21.15.28:0/480610482 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2e440aa710 con 0x55b182f3d1c0 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e4aa00700 1 -- 172.21.15.28:0/480610482 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f2e440b3050 con 0x55b182f3d1c0 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 10 client.26096 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 10 client.26096 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 20 client.26096 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182f56160 0x55b182f58550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 4 client.26096 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.073+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55b182e9ce10 con 0x55b182f56160 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.074+0000 7f2e53e00700 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182f56160 0x55b182f58550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.074+0000 7f2e53e00700 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182f56160 0x55b182f58550 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a5aaa499b590d7fe server_cookie=841ffe45905f63a9 in_seq=0 out_seq=0 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.074+0000 7f2e4aa00700 10 client.26096 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e4aa00700 1 -- 172.21.15.28:0/480610482 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55b182e9ce10 con 0x55b182f56160 2024-05-20T20:08:32.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e4aa00700 10 client.26096 handle_command_reply: tid=0 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 client.26096 shutdown 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 msgr2=0x55b182f48ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f48ca0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f2e44009f90 tx=0x7f2e440a6c30 comp rx=0 tx=0).stop 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 shutdown_connections 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182f56160 0x55b182f58550 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f46580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f48ca0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/480610482 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f46040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 >> 172.21.15.28:0/480610482 conn(0x55b182e6ac60 msgr2=0x55b182f36df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e4aa00700 0 client.26096 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.075+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 shutdown_connections 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.076+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/480610482 wait complete. 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.076+0000 7f2e6e23ab80 20 client.26096 trim_cache size 0 max 16384 2024-05-20T20:08:32.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.098+0000 7f2e6e23ab80 1 Processor -- start 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.098+0000 7f2e6e23ab80 1 -- start start 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f3cba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f37570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f47e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1afee0 con 0x55b182f42fb0 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1b1a20 con 0x55b182f3d1c0 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1b05b0 con 0x55b182f42690 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e60e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f3cba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e60e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f3cba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51816/0 (socket says 172.21.15.28:51816) 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e60e00700 1 -- 172.21.15.28:0/1745223380 learned_addr learned my addr 172.21.15.28:0/1745223380 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e61800700 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f47e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.099+0000 7f2e53e00700 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f37570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e60e00700 1 -- 172.21.15.28:0/1745223380 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 msgr2=0x55b182f37570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e60e00700 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f37570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e60e00700 1 -- 172.21.15.28:0/1745223380 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 msgr2=0x55b182f47e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e60e00700 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f47e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e60e00700 1 -- 172.21.15.28:0/1745223380 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b182f445b0 con 0x55b182f3d1c0 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e53e00700 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f37570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e60e00700 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f3cba0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f2e440a7980 tx=0x7f2e440a9a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=785dc43cc687b458 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e51600700 1 -- 172.21.15.28:0/1745223380 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e440aae10 con 0x55b182f3d1c0 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e51600700 1 -- 172.21.15.28:0/1745223380 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2e440aa710 con 0x55b182f3d1c0 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.100+0000 7f2e51600700 1 -- 172.21.15.28:0/1745223380 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e440aa970 con 0x55b182f3d1c0 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1745223380 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 msgr2=0x55b182f3cba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f3cba0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f2e440a7980 tx=0x7f2e440a9a40 comp rx=0 tx=0).stop 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1745223380 shutdown_connections 2024-05-20T20:08:32.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f47e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f3d1c0 0x55b182f3cba0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1745223380 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f42690 0x55b182f37570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1745223380 >> 172.21.15.28:0/1745223380 conn(0x55b182e6ac60 msgr2=0x55b182f476f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1745223380 shutdown_connections 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.101+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1745223380 wait complete. 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 -- start start 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 Processor -- start 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f3d1c0 0x55b182f5a550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f5aa90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f5b010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1b1350 con 0x55b182f42fb0 2024-05-20T20:08:32.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1b0c80 con 0x55b182f42690 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.102+0000 7f2e6e23ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2e5c1adea0 con 0x55b182f3d1c0 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e6e23ab80 10 client.0 _ll_register_callbacks cb 0x55b182e34cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e61800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f5b010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e53e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f5aa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e61800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f5b010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35344/0 (socket says 172.21.15.28:35344) 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e53e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f5aa90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51820/0 (socket says 172.21.15.28:51820) 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e61800700 1 -- 172.21.15.28:0/1174151745 learned_addr learned my addr 172.21.15.28:0/1174151745 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e60e00700 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f3d1c0 0x55b182f5a550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e53e00700 1 -- 172.21.15.28:0/1174151745 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f3d1c0 msgr2=0x55b182f5a550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e53e00700 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f3d1c0 0x55b182f5a550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e53e00700 1 -- 172.21.15.28:0/1174151745 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 msgr2=0x55b182f5b010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e53e00700 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f5b010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e53e00700 1 -- 172.21.15.28:0/1174151745 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b182fc0cb0 con 0x55b182f42690 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.103+0000 7f2e61800700 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f5b010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e53e00700 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f5aa90 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x55b182e5b250 tx=0x7f2e4c01f6d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=39a0d35af4a567b7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e4aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e4aa00700 1 -- 172.21.15.28:0/1174151745 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e4c01fd20 con 0x55b182f42690 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2e440aa710 con 0x55b182f42690 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e4aa00700 1 -- 172.21.15.28:0/1174151745 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2e4c004830 con 0x55b182f42690 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e4aa00700 1 -- 172.21.15.28:0/1174151745 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2e4c004a30 con 0x55b182f42690 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e6e23ab80 10 client.26102 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.104+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55b182fc2050 con 0x55b182f42690 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e4aa00700 1 -- 172.21.15.28:0/1174151745 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2e440aa710 con 0x55b182f42690 2024-05-20T20:08:32.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e4aa00700 1 -- 172.21.15.28:0/1174151745 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f2e4c02a430 con 0x55b182f42690 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e6e23ab80 10 client.26102 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e6e23ab80 10 client.26102 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e6e23ab80 20 client.26102 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182fc2710 0x55b182fc4b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e6e23ab80 4 client.26102 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x55b182e965f0 con 0x55b182fc2710 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.105+0000 7f2e60e00700 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182fc2710 0x55b182fc4b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.106+0000 7f2e60e00700 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182fc2710 0x55b182fc4b30 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=483759a8b2e274e1 server_cookie=47fc2ebedad24f00 in_seq=0 out_seq=0 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.106+0000 7f2e4aa00700 10 client.26102 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e4aa00700 1 -- 172.21.15.28:0/1174151745 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x55b182e965f0 con 0x55b182fc2710 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e4aa00700 10 client.26102 handle_command_reply: tid=0 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 client.26102 shutdown 2024-05-20T20:08:32.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 msgr2=0x55b182f5aa90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f5aa90 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x55b182e5b250 tx=0x7f2e4c01f6d0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.339 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:08:32.339 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:08:32.339 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "36cdf0ea-3220-42b1-89a0-cf6fa618fb1d", 2024-05-20T20:08:32.339 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:08:32.339 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 shutdown_connections 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b182fc2710 0x55b182fc4b30 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b182f42fb0 0x55b182f5b010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b182f3d1c0 0x55b182f5a550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 --2- 172.21.15.28:0/1174151745 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b182f42690 0x55b182f5aa90 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 >> 172.21.15.28:0/1174151745 conn(0x55b182e6ac60 msgr2=0x55b182f47ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.107+0000 7f2e4aa00700 0 client.26102 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.108+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 shutdown_connections 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.108+0000 7f2e6e23ab80 1 -- 172.21.15.28:0/1174151745 wait complete. 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.108+0000 7f2e6e23ab80 20 client.26102 trim_cache size 0 max 16384 2024-05-20T20:08:32.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.108+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2e3c0b2a70 msgr2=0x7f2e3c0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.108+0000 7f2e63800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2e3c0b2a70 0x7f2e3c0b4f30 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f2e5400f870 tx=0x7f2e5400fc30 comp rx=0 tx=0).stop 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.108+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2e5c1037f0 msgr2=0x7f2e5c19be80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.108+0000 7f2e63800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2e5c1037f0 0x7f2e5c19be80 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f2e4400c010 tx=0x7f2e4400fa20 comp rx=0 tx=0).stop 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 shutdown_connections 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e5c1088f0 0x7f2e5c1a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2e3c0b2a70 0x7f2e3c0b4f30 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2e5c1037f0 0x7f2e5c19be80 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 --2- 172.21.15.28:0/3339008666 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2e5c1041a0 0x7f2e5c19c3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 >> 172.21.15.28:0/3339008666 conn(0x7f2e5c07a6a0 msgr2=0x7f2e5c1019d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 shutdown_connections 2024-05-20T20:08:32.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:32.109+0000 7f2e63800700 1 -- 172.21.15.28:0/3339008666 wait complete. 2024-05-20T20:08:33.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:32 smithi028 ceph-mon[31178]: pgmap v90: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:08:33.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:32 smithi154 ceph-mon[36314]: pgmap v90: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:08:33.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:32 smithi033 ceph-mon[36179]: pgmap v90: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2024-05-20T20:08:33.231 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:08:33.833 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:33.955 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:33 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:08:33.955 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:33 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:08:33.955 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:33 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:08:33.955 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:33 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:08:33.955 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:33 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:08:34.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:33 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:08:34.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:33 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:08:34.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:33 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:08:34.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:33 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:08:34.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:33 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:08:34.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:33 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:08:34.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:33 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:08:34.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:33 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:08:34.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:33 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:08:34.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:33 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:08:34.596 INFO:tasks.workunit.client.0.smithi028.stdout:getting iogen 2024-05-20T20:08:34.597 INFO:tasks.workunit.client.0.smithi028.stderr:+ echo 'getting iogen' 2024-05-20T20:08:34.597 INFO:tasks.workunit.client.0.smithi028.stderr:+ wget http://download.ceph.com/qa/iogen_3.1p0.tar 2024-05-20T20:08:34.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.520+0000 7ffacb000700 1 -- 172.21.15.28:0/1380196683 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 msgr2=0x7ffac41103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.520+0000 7ffacb000700 1 --2- 172.21.15.28:0/1380196683 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 0x7ffac41103d0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7ffabc009f90 tx=0x7ffabc00b330 comp rx=0 tx=0).stop 2024-05-20T20:08:34.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.520+0000 7ffacb000700 1 -- 172.21.15.28:0/1380196683 shutdown_connections 2024-05-20T20:08:34.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.520+0000 7ffacb000700 1 --2- 172.21.15.28:0/1380196683 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 0x7ffac41103d0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.520+0000 7ffacb000700 1 --2- 172.21.15.28:0/1380196683 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffac41095b0 0x7ffac4109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.520+0000 7ffacb000700 1 --2- 172.21.15.28:0/1380196683 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 0x7ffac406b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.520+0000 7ffacb000700 1 -- 172.21.15.28:0/1380196683 >> 172.21.15.28:0/1380196683 conn(0x7ffac40793f0 msgr2=0x7ffac4079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.521+0000 7ffacb000700 1 -- 172.21.15.28:0/1380196683 shutdown_connections 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.521+0000 7ffacb000700 1 -- 172.21.15.28:0/1380196683 wait complete. 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 Processor -- start 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 -- start start 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 0x7ffac419be00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 0x7ffac419c340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffac41095b0 0x7ffac41a06d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffac4113740 con 0x7ffac41095b0 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffac41135c0 con 0x7ffac406b220 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.522+0000 7ffacb000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffac41138c0 con 0x7ffac406bbe0 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 0x7ffac419be00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 0x7ffac419be00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51838/0 (socket says 172.21.15.28:51838) 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 -- 172.21.15.28:0/4212021549 learned_addr learned my addr 172.21.15.28:0/4212021549 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3400700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 0x7ffac419c340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac9000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffac41095b0 0x7ffac41a06d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 -- 172.21.15.28:0/4212021549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 msgr2=0x7ffac419c340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 0x7ffac419c340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 -- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffac41095b0 msgr2=0x7ffac41a06d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffac41095b0 0x7ffac41a06d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 -- 172.21.15.28:0/4212021549 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffac41a0e10 con 0x7ffac406b220 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.523+0000 7ffac3e00700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 0x7ffac419be00 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7ffab400e9a0 tx=0x7ffab400ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.524+0000 7ffac0c00700 1 -- 172.21.15.28:0/4212021549 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab400cbd0 con 0x7ffac406b220 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.524+0000 7ffac0c00700 1 -- 172.21.15.28:0/4212021549 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffab400cd70 con 0x7ffac406b220 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.524+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffac41a1100 con 0x7ffac406b220 2024-05-20T20:08:34.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.524+0000 7ffac0c00700 1 -- 172.21.15.28:0/4212021549 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab4018680 con 0x7ffac406b220 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.524+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffac41a8980 con 0x7ffac406b220 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.528+0000 7ffac0c00700 1 -- 172.21.15.28:0/4212021549 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ffab4018820 con 0x7ffac406b220 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.529+0000 7ffad5928b80 1 Processor -- start 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.529+0000 7ffad5928b80 1 -- start start 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.529+0000 7ffac0c00700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffaa80b2a70 0x7ffaa80b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.529+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f377c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.529+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f31960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.529+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f323c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffad5928b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560e46e8ff40 con 0x560e46f37d00 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffad5928b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x560e46e7fae0 con 0x560e46f31f80 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffad5928b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x560e46e83b20 con 0x560e46f373e0 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffac0c00700 1 -- 172.21.15.28:0/4212021549 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ffab40e26d0 con 0x7ffac406b220 2024-05-20T20:08:34.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffac3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f31960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffac9000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffaa80b2a70 0x7ffaa80b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffac3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f31960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35376/0 (socket says 172.21.15.28:35376) 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffac3400700 1 -- 172.21.15.28:0/1784907471 learned_addr learned my addr 172.21.15.28:0/1784907471 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffac3e00700 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f377c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.530+0000 7ffac9000700 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f323c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac3400700 1 -- 172.21.15.28:0/1784907471 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 msgr2=0x560e46f377c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac3400700 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f377c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac3400700 1 -- 172.21.15.28:0/1784907471 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 msgr2=0x560e46f323c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac3400700 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f323c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac3400700 1 -- 172.21.15.28:0/1784907471 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560e46e48520 con 0x560e46f37d00 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac9000700 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f323c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac3400700 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f31960 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7ffab0009d30 tx=0x7ffab0011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2fde412091cde2d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:34.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffac9000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffaa80b2a70 0x7ffaa80b4f30 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7ffabc0060b0 tx=0x7ffabc006040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.531+0000 7ffab9600700 1 -- 172.21.15.28:0/1784907471 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab0004250 con 0x560e46f37d00 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.532+0000 7ffab9600700 1 -- 172.21.15.28:0/1784907471 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffab00043f0 con 0x560e46f37d00 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.532+0000 7ffab9600700 1 -- 172.21.15.28:0/1784907471 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab0019820 con 0x560e46f37d00 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.532+0000 7ffad5928b80 1 -- 172.21.15.28:0/1784907471 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 msgr2=0x560e46f31960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.532+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f31960 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7ffab0009d30 tx=0x7ffab0011720 comp rx=0 tx=0).stop 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 -- 172.21.15.28:0/1784907471 shutdown_connections 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f323c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f377c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1784907471 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f31960 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 -- 172.21.15.28:0/1784907471 >> 172.21.15.28:0/1784907471 conn(0x560e46e5fce0 msgr2=0x560e46f2c420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 -- 172.21.15.28:0/1784907471 shutdown_connections 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 -- 172.21.15.28:0/1784907471 wait complete. 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.533+0000 7ffad5928b80 1 -- start start 2024-05-20T20:08:34.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 1 Processor -- start 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f3d910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f373e0 0x560e46f3acb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f3b1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560e46d860a0 con 0x560e46f37d00 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x560e46e20860 con 0x560e46f373e0 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x560e46f3e7e0 con 0x560e46f31f80 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffad5928b80 10 client.0 _ll_register_callbacks cb 0x560e46e29d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffac3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f373e0 0x560e46f3acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffac3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f373e0 0x560e46f3acb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51864/0 (socket says 172.21.15.28:51864) 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.534+0000 7ffac3400700 1 -- 172.21.15.28:0/289485989 learned_addr learned my addr 172.21.15.28:0/289485989 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac3e00700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f3d910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac9000700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f3b1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac3400700 1 -- 172.21.15.28:0/289485989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 msgr2=0x560e46f3d910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac3400700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f3d910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac3400700 1 -- 172.21.15.28:0/289485989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 msgr2=0x560e46f3b1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac3400700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f3b1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac3400700 1 -- 172.21.15.28:0/289485989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560e46dcf9d0 con 0x560e46f373e0 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac9000700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f3b1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.535+0000 7ffac3400700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f373e0 0x560e46f3acb0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7ffab0014040 tx=0x7ffab00057f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ed0b63fcd979236b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.536+0000 7ffaae000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.536+0000 7ffaae000700 1 -- 172.21.15.28:0/289485989 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab0027020 con 0x560e46f373e0 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.536+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffab00043f0 con 0x560e46f373e0 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.536+0000 7ffaae000700 1 -- 172.21.15.28:0/289485989 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ffab0003a10 con 0x560e46f373e0 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.536+0000 7ffaae000700 1 -- 172.21.15.28:0/289485989 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffab0032050 con 0x560e46f373e0 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.536+0000 7ffad5928b80 10 client.26114 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:34.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.536+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560e46e50160 con 0x560e46f373e0 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffaae000700 1 -- 172.21.15.28:0/289485989 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffab00043f0 con 0x560e46f373e0 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffaae000700 1 -- 172.21.15.28:0/289485989 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffab0018070 con 0x560e46f373e0 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffad5928b80 10 client.26114 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffad5928b80 10 client.26114 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffad5928b80 20 client.26114 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffad5928b80 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46f4d5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffad5928b80 4 client.26114 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.537+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560e46e91e90 con 0x560e46f4b1b0 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.538+0000 7ffac3e00700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46f4d5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.538+0000 7ffac3e00700 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46f4d5a0 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=848f21c38fa92087 server_cookie=462ce5fac2aaf87d in_seq=0 out_seq=0 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.538+0000 7ffaae000700 10 client.26114 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.539+0000 7ffaae000700 1 -- 172.21.15.28:0/289485989 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x560e46e91e90 con 0x560e46f4b1b0 2024-05-20T20:08:34.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.539+0000 7ffaae000700 10 client.26114 handle_command_reply: tid=0 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.539+0000 7ffad5928b80 1 client.26114 shutdown 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.539+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f373e0 msgr2=0x560e46f3acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.539+0000 7ffad5928b80 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f373e0 0x560e46f3acb0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7ffab0014040 tx=0x7ffab00057f0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 shutdown_connections 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46f4d5a0 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffaae000700 0 client.26114 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f3b1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f3d910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 --2- 172.21.15.28:0/289485989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f373e0 0x560e46f3acb0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 >> 172.21.15.28:0/289485989 conn(0x560e46e5fce0 msgr2=0x560e46f30950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 shutdown_connections 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 1 -- 172.21.15.28:0/289485989 wait complete. 2024-05-20T20:08:34.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.540+0000 7ffad5928b80 20 client.26114 trim_cache size 0 max 16384 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 Processor -- start 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 -- start start 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f31950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f373e0 0x560e46f381a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f37d00 0x560e46f386e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa400d580 con 0x560e46f373e0 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa400f0c0 con 0x560e46f37d00 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.569+0000 7ffad5928b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa400dc50 con 0x560e46f31f80 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f373e0 0x560e46f381a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f31950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f37d00 0x560e46f386e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f373e0 0x560e46f381a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35396/0 (socket says 172.21.15.28:35396) 2024-05-20T20:08:34.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3400700 1 -- 172.21.15.28:0/3126008010 learned_addr learned my addr 172.21.15.28:0/3126008010 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3e00700 1 -- 172.21.15.28:0/3126008010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f37d00 msgr2=0x560e46f386e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3e00700 1 --2- 172.21.15.28:0/3126008010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f37d00 0x560e46f386e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3e00700 1 -- 172.21.15.28:0/3126008010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f373e0 msgr2=0x560e46f381a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3e00700 1 --2- 172.21.15.28:0/3126008010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f373e0 0x560e46f381a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.570+0000 7ffac3e00700 1 -- 172.21.15.28:0/3126008010 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560e46f3b340 con 0x560e46f31f80 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.571+0000 7ffac3e00700 1 --2- 172.21.15.28:0/3126008010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f31950 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7ffab40dfdd0 tx=0x7ffab401a040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7933df8c946548ae server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.571+0000 7ffab8c00700 1 -- 172.21.15.28:0/3126008010 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab40a9470 con 0x560e46f31f80 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.571+0000 7ffab8c00700 1 -- 172.21.15.28:0/3126008010 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffab40aa060 con 0x560e46f31f80 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.571+0000 7ffab8c00700 1 -- 172.21.15.28:0/3126008010 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab40e0250 con 0x560e46f31f80 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 -- 172.21.15.28:0/3126008010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 msgr2=0x560e46f31950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 --2- 172.21.15.28:0/3126008010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f31950 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7ffab40dfdd0 tx=0x7ffab401a040 comp rx=0 tx=0).stop 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 -- 172.21.15.28:0/3126008010 shutdown_connections 2024-05-20T20:08:34.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 --2- 172.21.15.28:0/3126008010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f37d00 0x560e46f386e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 --2- 172.21.15.28:0/3126008010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f31f80 0x560e46f31950 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 --2- 172.21.15.28:0/3126008010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f373e0 0x560e46f381a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 -- 172.21.15.28:0/3126008010 >> 172.21.15.28:0/3126008010 conn(0x560e46e5fce0 msgr2=0x560e46f31240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 -- 172.21.15.28:0/3126008010 shutdown_connections 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.572+0000 7ffad5928b80 1 -- 172.21.15.28:0/3126008010 wait complete. 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 -- start start 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 Processor -- start 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f40b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f41050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f41590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa400e9f0 con 0x560e46f37d00 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa400e320 con 0x560e46f31f80 2024-05-20T20:08:34.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa400b520 con 0x560e46f373e0 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.573+0000 7ffad5928b80 10 client.0 _ll_register_callbacks cb 0x560e46e29d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f40b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac9000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f41590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f40b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51884/0 (socket says 172.21.15.28:51884) 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 -- 172.21.15.28:0/1904049195 learned_addr learned my addr 172.21.15.28:0/1904049195 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f41050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 -- 172.21.15.28:0/1904049195 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 msgr2=0x560e46f41050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f41050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 -- 172.21.15.28:0/1904049195 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 msgr2=0x560e46f41590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f41590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 -- 172.21.15.28:0/1904049195 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560e46fb6000 con 0x560e46f31f80 2024-05-20T20:08:34.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffac3e00700 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f40b10 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7ffab40a6bd0 tx=0x7ffab40e0280 comp rx=0 tx=0).ready entity=mon.2 client_cookie=65fe026076a5c242 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.574+0000 7ffaae000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffaae000700 1 -- 172.21.15.28:0/1904049195 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffab40a9470 con 0x560e46f31f80 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffab40aa060 con 0x560e46f31f80 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffaae000700 1 -- 172.21.15.28:0/1904049195 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7ffab40a9bc0 con 0x560e46f31f80 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffaae000700 1 -- 172.21.15.28:0/1904049195 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffab40cc2c0 con 0x560e46f31f80 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 10 client.26120 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560e46fb7320 con 0x560e46f31f80 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffaae000700 1 -- 172.21.15.28:0/1904049195 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffab40aa060 con 0x560e46f31f80 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffaae000700 1 -- 172.21.15.28:0/1904049195 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffab40c6070 con 0x560e46f31f80 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 10 client.26120 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 10 client.26120 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 20 client.26120 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:34.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46fb99a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 4 client.26120 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "a49aa3fd", "release": true, "await": true}] 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.575+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "a49aa3fd", "release": true, "await": true}) -- 0x560e46e8b670 con 0x560e46f4b1b0 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.576+0000 7ffac3400700 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46fb99a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.576+0000 7ffac3400700 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46fb99a0 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7bd2480065a5697 server_cookie=731a7876d14d75b6 in_seq=0 out_seq=0 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.576+0000 7ffaae000700 10 client.26120 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.580+0000 7ffaae000700 1 -- 172.21.15.28:0/1904049195 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+589 (crc 0 0 0) 0x560e46e8b670 con 0x560e46f4b1b0 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.580+0000 7ffaae000700 10 client.26120 handle_command_reply: tid=0 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.580+0000 7ffad5928b80 1 client.26120 shutdown 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.580+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 msgr2=0x560e46f40b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.580+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f40b10 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7ffab40a6bd0 tx=0x7ffab40e0280 comp rx=0 tx=0).stop 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 shutdown_connections 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x560e46f4b1b0 0x560e46fb99a0 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560e46f37d00 0x560e46f41590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560e46f31f80 0x560e46f40b10 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 8, 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stdout: "a49aa3fd": { 2024-05-20T20:08:34.643 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 8, 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 146.3, 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:08:34.644 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:08:34.645 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:34.645 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:34.645 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:34.645 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:34.645 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:08:34.645 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:08:34.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 --2- 172.21.15.28:0/1904049195 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560e46f373e0 0x560e46f41050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 >> 172.21.15.28:0/1904049195 conn(0x560e46e5fce0 msgr2=0x560e46f380e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffaae000700 0 client.26120 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 shutdown_connections 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 1 -- 172.21.15.28:0/1904049195 wait complete. 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.581+0000 7ffad5928b80 20 client.26120 trim_cache size 0 max 16384 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffaa80b2a70 msgr2=0x7ffaa80b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffaa80b2a70 0x7ffaa80b4f30 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7ffabc0060b0 tx=0x7ffabc006040 comp rx=0 tx=0).stop 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 msgr2=0x7ffac419be00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 0x7ffac419be00 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7ffab400e9a0 tx=0x7ffab400ed60 comp rx=0 tx=0).stop 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 shutdown_connections 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffac41095b0 0x7ffac41a06d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffaa80b2a70 0x7ffaa80b4f30 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffac406b220 0x7ffac419be00 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 --2- 172.21.15.28:0/4212021549 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffac406bbe0 0x7ffac419c340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:34.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 >> 172.21.15.28:0/4212021549 conn(0x7ffac40793f0 msgr2=0x7ffac410e700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:34.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 shutdown_connections 2024-05-20T20:08:34.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:34.582+0000 7ffacb000700 1 -- 172.21.15.28:0/4212021549 wait complete. 2024-05-20T20:08:34.698 INFO:tasks.workunit.client.0.smithi028.stderr:--2024-05-20 20:08:34-- http://download.ceph.com/qa/iogen_3.1p0.tar 2024-05-20T20:08:34.713 INFO:tasks.workunit.client.0.smithi028.stderr:Resolving download.ceph.com (download.ceph.com)... 158.69.68.124, 2607:5300:201:2000::3:58a1 2024-05-20T20:08:34.734 INFO:tasks.workunit.client.0.smithi028.stderr:Connecting to download.ceph.com (download.ceph.com)|158.69.68.124|:80... connected. 2024-05-20T20:08:34.755 INFO:tasks.workunit.client.0.smithi028.stderr:HTTP request sent, awaiting response... 200 OK 2024-05-20T20:08:34.755 INFO:tasks.workunit.client.0.smithi028.stderr:Length: 8071 (7.9K) [application/octet-stream] 2024-05-20T20:08:34.759 INFO:tasks.workunit.client.0.smithi028.stderr:Saving to: ‘iogen_3.1p0.tar’ 2024-05-20T20:08:34.769 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:34.769 INFO:tasks.workunit.client.0.smithi028.stderr: 0K ....... 100% 220M=0s 2024-05-20T20:08:34.769 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:34.769 INFO:tasks.workunit.client.0.smithi028.stderr:2024-05-20 20:08:34 (220 MB/s) - ‘iogen_3.1p0.tar’ saved [8071/8071] 2024-05-20T20:08:34.769 INFO:tasks.workunit.client.0.smithi028.stderr: 2024-05-20T20:08:34.771 INFO:tasks.workunit.client.0.smithi028.stderr:+ tar -xvzf iogen_3.1p0.tar 2024-05-20T20:08:34.832 INFO:tasks.workunit.client.0.smithi028.stdout:iogen_3.1p0 2024-05-20T20:08:34.833 INFO:tasks.workunit.client.0.smithi028.stdout:iogen_3.1p0/src 2024-05-20T20:08:34.834 INFO:tasks.workunit.client.0.smithi028.stdout:iogen_3.1p0/src/iogen.8 2024-05-20T20:08:34.836 INFO:tasks.workunit.client.0.smithi028.stdout:iogen_3.1p0/src/iogen.c 2024-05-20T20:08:34.839 INFO:tasks.workunit.client.0.smithi028.stdout:iogen_3.1p0/README 2024-05-20T20:08:34.840 INFO:tasks.workunit.client.0.smithi028.stdout:iogen_3.1p0/Makefile.bsd 2024-05-20T20:08:34.842 INFO:tasks.workunit.client.0.smithi028.stdout:iogen_3.1p0/Makefile 2024-05-20T20:08:34.846 INFO:tasks.workunit.client.0.smithi028.stderr:+ cd iogen_3.1p0 2024-05-20T20:08:34.846 INFO:tasks.workunit.client.0.smithi028.stdout:making iogen 2024-05-20T20:08:34.846 INFO:tasks.workunit.client.0.smithi028.stderr:+ echo 'making iogen' 2024-05-20T20:08:34.846 INFO:tasks.workunit.client.0.smithi028.stderr:+ make 2024-05-20T20:08:34.860 INFO:tasks.workunit.client.0.smithi028.stdout:gcc -D_FILE_OFFSET_BITS=64 -DVERSION=\"3.1p0\" -Dstrlcpy=strncpy -Darc4random=rand -O2 src/iogen.c -o iogen 2024-05-20T20:08:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:34 smithi154 ceph-mon[36314]: pgmap v91: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:08:35.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:34 smithi033 ceph-mon[36179]: pgmap v91: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:08:35.163 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:34 smithi028 ceph-mon[31178]: pgmap v91: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 255 B/s wr, 13 op/s 2024-05-20T20:08:35.723 INFO:tasks.workunit.client.0.smithi028.stdout:nroff -mandoc -Tascii src/iogen.8 > iogen.cat8 2024-05-20T20:08:36.243 INFO:tasks.workunit.client.0.smithi028.stdout:running iogen 2024-05-20T20:08:36.243 INFO:tasks.workunit.client.0.smithi028.stderr:+ echo 'running iogen' 2024-05-20T20:08:36.243 INFO:tasks.workunit.client.0.smithi028.stderr:+ ./iogen -n 5 -s 2g 2024-05-20T20:08:36.246 INFO:tasks.workunit.client.0.smithi028.stderr:child spawned 100430 2024-05-20T20:08:36.247 INFO:tasks.workunit.client.0.smithi028.stderr:child spawned 100431 2024-05-20T20:08:36.247 INFO:tasks.workunit.client.0.smithi028.stderr:child spawned 100432 2024-05-20T20:08:36.248 INFO:tasks.workunit.client.0.smithi028.stderr:child spawned 100433 2024-05-20T20:08:36.248 INFO:tasks.workunit.client.0.smithi028.stderr:child spawned 100434 2024-05-20T20:08:36.249 INFO:tasks.workunit.client.0.smithi028.stdout:sleep for 10 min 2024-05-20T20:08:36.249 INFO:tasks.workunit.client.0.smithi028.stderr:+ echo 'sleep for 10 min' 2024-05-20T20:08:36.249 INFO:tasks.workunit.client.0.smithi028.stderr:+ sleep 600 2024-05-20T20:08:36.671 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set 'a49aa3fd', total seconds elapsed: 39.9 2024-05-20T20:08:36.895 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:36 smithi028 ceph-mon[31178]: pgmap v92: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:37.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:36.999+0000 7fd6a1600640 1 -- 172.21.15.28:0/2120140570 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 msgr2=0x7fd69c105290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:36.999+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2120140570 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c105290 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fd68c009900 tx=0x7fd68c02a3f0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.000+0000 7fd6a1600640 1 -- 172.21.15.28:0/2120140570 shutdown_connections 2024-05-20T20:08:37.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.000+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2120140570 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd69c105860 0x7fd69c1098a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.000+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2120140570 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd69c109f80 0x7fd69c111b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.000+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2120140570 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c105290 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.000+0000 7fd6a1600640 1 -- 172.21.15.28:0/2120140570 >> 172.21.15.28:0/2120140570 conn(0x7fd69c100750 msgr2=0x7fd69c102b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.001+0000 7fd6a1600640 1 -- 172.21.15.28:0/2120140570 shutdown_connections 2024-05-20T20:08:37.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.001+0000 7fd6a1600640 1 -- 172.21.15.28:0/2120140570 wait complete. 2024-05-20T20:08:37.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.003+0000 7fd6a1600640 1 Processor -- start 2024-05-20T20:08:37.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.003+0000 7fd6a1600640 1 -- start start 2024-05-20T20:08:37.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.003+0000 7fd6a1600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c110620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.003+0000 7fd6a1600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd69c109f80 0x7fd69c110b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd6a1600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd69c1a5310 0x7fd69c1a57c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd6a1600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c078030 con 0x7fd69c109f80 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd6a1600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c077eb0 con 0x7fd69c1a5310 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd6a1600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c0781b0 con 0x7fd69c104eb0 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c110620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c110620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44972/0 (socket says 172.21.15.28:44972) 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd69aa00640 1 -- 172.21.15.28:0/2490075075 learned_addr learned my addr 172.21.15.28:0/2490075075 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.004+0000 7fd69b400640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd69c1a5310 0x7fd69c1a57c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.005+0000 7fd69aa00640 1 -- 172.21.15.28:0/2490075075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd69c1a5310 msgr2=0x7fd69c1a57c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.005+0000 7fd69aa00640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd69c1a5310 0x7fd69c1a57c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.005+0000 7fd69aa00640 1 -- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd69c109f80 msgr2=0x7fd69c110b60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.005+0000 7fd69aa00640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd69c109f80 0x7fd69c110b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.005+0000 7fd69aa00640 1 -- 172.21.15.28:0/2490075075 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd69c1a5e30 con 0x7fd69c104eb0 2024-05-20T20:08:37.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.006+0000 7fd69aa00640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c110620 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fd68c031440 tx=0x7fd68c0045e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.008+0000 7fd693400640 1 -- 172.21.15.28:0/2490075075 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd68c035030 con 0x7fd69c104eb0 2024-05-20T20:08:37.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.008+0000 7fd693400640 1 -- 172.21.15.28:0/2490075075 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd68c003d30 con 0x7fd69c104eb0 2024-05-20T20:08:37.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.008+0000 7fd693400640 1 -- 172.21.15.28:0/2490075075 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd68c0318a0 con 0x7fd69c104eb0 2024-05-20T20:08:37.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.009+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd69c1a60c0 con 0x7fd69c104eb0 2024-05-20T20:08:37.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.009+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd69c113f80 con 0x7fd69c104eb0 2024-05-20T20:08:37.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.015+0000 7fd6a441f740 1 Processor -- start 2024-05-20T20:08:37.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.015+0000 7fd6a441f740 1 -- start start 2024-05-20T20:08:37.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39f3070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fd930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a00260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd6a441f740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bcb3963b90 con 0x55bcb39f2c90 2024-05-20T20:08:37.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd6a441f740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bcb3953730 con 0x55bcb39f35b0 2024-05-20T20:08:37.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd6a441f740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bcb3957770 con 0x55bcb39fde70 2024-05-20T20:08:37.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd69b400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39f3070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd693e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a00260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd693e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a00260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44974/0 (socket says 172.21.15.28:44974) 2024-05-20T20:08:37.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.016+0000 7fd693e00640 1 -- 172.21.15.28:0/918913811 learned_addr learned my addr 172.21.15.28:0/918913811 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.017+0000 7fd693400640 1 -- 172.21.15.28:0/2490075075 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd68c05d620 con 0x7fd69c104eb0 2024-05-20T20:08:37.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.017+0000 7fd693e00640 1 -- 172.21.15.28:0/918913811 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 msgr2=0x55bcb39fd930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:08:37.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.017+0000 7fd693e00640 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fd930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.017+0000 7fd693e00640 1 -- 172.21.15.28:0/918913811 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 msgr2=0x55bcb39f3070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.017+0000 7fd693e00640 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39f3070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.017+0000 7fd693e00640 1 -- 172.21.15.28:0/918913811 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bcb38ec8b0 con 0x55bcb39fde70 2024-05-20T20:08:37.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.017+0000 7fd693e00640 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a00260 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd684009b10 tx=0x7fd68400e350 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bd26a2edf28db202 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.018+0000 7fd69b400640 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39f3070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:37.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.018+0000 7fd68b400640 1 -- 172.21.15.28:0/918913811 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd68400ed50 con 0x55bcb39fde70 2024-05-20T20:08:37.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.018+0000 7fd68b400640 1 -- 172.21.15.28:0/918913811 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd6840035f0 con 0x55bcb39fde70 2024-05-20T20:08:37.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.018+0000 7fd68b400640 1 -- 172.21.15.28:0/918913811 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd684010690 con 0x55bcb39fde70 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.019+0000 7fd6a441f740 1 -- 172.21.15.28:0/918913811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 msgr2=0x55bcb3a00260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.019+0000 7fd6a441f740 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a00260 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd684009b10 tx=0x7fd68400e350 comp rx=0 tx=0).stop 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd693400640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd6800b2b50 0x7fd6800b5010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd693400640 1 -- 172.21.15.28:0/2490075075 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd68c124930 con 0x7fd69c104eb0 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd6a441f740 1 -- 172.21.15.28:0/918913811 shutdown_connections 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd6a441f740 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fd930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd6a441f740 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a00260 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd6a441f740 1 --2- 172.21.15.28:0/918913811 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39f3070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd6a441f740 1 -- 172.21.15.28:0/918913811 >> 172.21.15.28:0/918913811 conn(0x55bcb39ef1a0 msgr2=0x55bcb39ef580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.020+0000 7fd6a441f740 1 -- 172.21.15.28:0/918913811 shutdown_connections 2024-05-20T20:08:37.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.021+0000 7fd6a441f740 1 -- 172.21.15.28:0/918913811 wait complete. 2024-05-20T20:08:37.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.021+0000 7fd6a441f740 1 -- start start 2024-05-20T20:08:37.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.021+0000 7fd69b400640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd6800b2b50 0x7fd6800b5010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.022+0000 7fd6a441f740 1 Processor -- start 2024-05-20T20:08:37.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.023+0000 7fd69b400640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd6800b2b50 0x7fd6800b5010 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fd694005fd0 tx=0x7fd6940096a0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.024 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.023+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39fd8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.023+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fab90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.023+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb39fb2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.023+0000 7fd6a441f740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bcb3639790 con 0x55bcb39f2c90 2024-05-20T20:08:37.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39fd8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd6a441f740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bcb38fbbc0 con 0x55bcb39f35b0 2024-05-20T20:08:37.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd6a441f740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bcb3a00aa0 con 0x55bcb39fde70 2024-05-20T20:08:37.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd6a441f740 10 client.0 _ll_register_callbacks cb 0x55bcb38d9400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39fd8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38624/0 (socket says 172.21.15.28:38624) 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd69aa00640 1 -- 172.21.15.28:0/3595032896 learned_addr learned my addr 172.21.15.28:0/3595032896 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd693e00640 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fab90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd69b400640 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb39fb2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd693e00640 1 -- 172.21.15.28:0/3595032896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 msgr2=0x55bcb39fb2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd693e00640 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb39fb2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd693e00640 1 -- 172.21.15.28:0/3595032896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 msgr2=0x55bcb39fd8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd693e00640 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39fd8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.025+0000 7fd693e00640 1 -- 172.21.15.28:0/3595032896 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bcb3668210 con 0x55bcb39f35b0 2024-05-20T20:08:37.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.026+0000 7fd693e00640 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fab90 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fd684009ae0 tx=0x7fd6840039d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2007f5d15517a591 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.026+0000 7fd67fe00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:37.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.026+0000 7fd67fe00640 1 -- 172.21.15.28:0/3595032896 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd6840043a0 con 0x55bcb39f35b0 2024-05-20T20:08:37.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.026+0000 7fd67fe00640 1 -- 172.21.15.28:0/3595032896 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd684018430 con 0x55bcb39f35b0 2024-05-20T20:08:37.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.026+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd6840035f0 con 0x55bcb39f35b0 2024-05-20T20:08:37.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.027+0000 7fd67fe00640 1 -- 172.21.15.28:0/3595032896 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd68402f050 con 0x55bcb39f35b0 2024-05-20T20:08:37.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.027+0000 7fd6a441f740 10 client.26129 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:37.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.027+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55bcb3734620 con 0x55bcb39f35b0 2024-05-20T20:08:37.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.027+0000 7fd67fe00640 1 -- 172.21.15.28:0/3595032896 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd6840035f0 con 0x55bcb39f35b0 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd67fe00640 1 -- 172.21.15.28:0/3595032896 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd68402a030 con 0x55bcb39f35b0 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd6a441f740 10 client.26129 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd6a441f740 10 client.26129 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd6a441f740 20 client.26129 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd6a441f740 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a0d7e0 0x55bcb3a0fbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd6a441f740 4 client.26129 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55bcb3965ae0 con 0x55bcb3a0d7e0 2024-05-20T20:08:37.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.028+0000 7fd69aa00640 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a0d7e0 0x55bcb3a0fbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.029+0000 7fd69aa00640 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a0d7e0 0x55bcb3a0fbd0 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bad29b31c64e3f47 server_cookie=32244d599213df1a in_seq=0 out_seq=0 2024-05-20T20:08:37.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.029+0000 7fd67fe00640 10 client.26129 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd67fe00640 1 -- 172.21.15.28:0/3595032896 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55bcb3965ae0 con 0x55bcb3a0d7e0 2024-05-20T20:08:37.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd67fe00640 10 client.26129 handle_command_reply: tid=0 2024-05-20T20:08:37.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 client.26129 shutdown 2024-05-20T20:08:37.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 msgr2=0x55bcb39fab90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fab90 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fd684009ae0 tx=0x7fd6840039d0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 shutdown_connections 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a0d7e0 0x55bcb3a0fbd0 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f35b0 0x55bcb39fab90 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb39fb2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 --2- 172.21.15.28:0/3595032896 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f2c90 0x55bcb39fd8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 >> 172.21.15.28:0/3595032896 conn(0x55bcb39ef1a0 msgr2=0x55bcb39f4f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.030+0000 7fd67fe00640 0 client.26129 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.031+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 shutdown_connections 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.031+0000 7fd6a441f740 1 -- 172.21.15.28:0/3595032896 wait complete. 2024-05-20T20:08:37.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.031+0000 7fd6a441f740 20 client.26129 trim_cache size 0 max 16384 2024-05-20T20:08:37.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.064+0000 7fd6a441f740 1 Processor -- start 2024-05-20T20:08:37.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.064+0000 7fd6a441f740 1 -- start start 2024-05-20T20:08:37.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a09b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39f35b0 0x55bcb3a151d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39fde70 0x55bcb3a15710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd6a441f740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c1ac3c0 con 0x55bcb39fde70 2024-05-20T20:08:37.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd6a441f740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c1adf00 con 0x55bcb39f2c90 2024-05-20T20:08:37.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd6a441f740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c1aca90 con 0x55bcb39f35b0 2024-05-20T20:08:37.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a09b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.065+0000 7fd693e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39f35b0 0x55bcb3a151d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69b400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39fde70 0x55bcb3a15710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69b400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39fde70 0x55bcb3a15710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38634/0 (socket says 172.21.15.28:38634) 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a09b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41452/0 (socket says 172.21.15.28:41452) 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69b400640 1 -- 172.21.15.28:0/370683040 learned_addr learned my addr 172.21.15.28:0/370683040 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69aa00640 1 -- 172.21.15.28:0/370683040 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39f35b0 msgr2=0x55bcb3a151d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69aa00640 1 --2- 172.21.15.28:0/370683040 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39f35b0 0x55bcb3a151d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69aa00640 1 -- 172.21.15.28:0/370683040 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39fde70 msgr2=0x55bcb3a15710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69aa00640 1 --2- 172.21.15.28:0/370683040 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39fde70 0x55bcb3a15710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69aa00640 1 -- 172.21.15.28:0/370683040 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bcb39ed6b0 con 0x55bcb39f2c90 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.066+0000 7fd69aa00640 1 --2- 172.21.15.28:0/370683040 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a09b00 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7fd68c0e8f20 tx=0x7fd68c05df40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=80e151518dc085cd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.067+0000 7fd68aa00640 1 -- 172.21.15.28:0/370683040 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd68c104020 con 0x55bcb39f2c90 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.067+0000 7fd68aa00640 1 -- 172.21.15.28:0/370683040 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd68c0f43c0 con 0x55bcb39f2c90 2024-05-20T20:08:37.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.067+0000 7fd68aa00640 1 -- 172.21.15.28:0/370683040 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd68c0f0060 con 0x55bcb39f2c90 2024-05-20T20:08:37.069 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 -- 172.21.15.28:0/370683040 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 msgr2=0x55bcb3a09b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.069 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 --2- 172.21.15.28:0/370683040 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a09b00 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7fd68c0e8f20 tx=0x7fd68c05df40 comp rx=0 tx=0).stop 2024-05-20T20:08:37.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 -- 172.21.15.28:0/370683040 shutdown_connections 2024-05-20T20:08:37.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 --2- 172.21.15.28:0/370683040 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39fde70 0x55bcb3a15710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 --2- 172.21.15.28:0/370683040 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39f35b0 0x55bcb3a151d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 --2- 172.21.15.28:0/370683040 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a09b00 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 -- 172.21.15.28:0/370683040 >> 172.21.15.28:0/370683040 conn(0x55bcb39ef1a0 msgr2=0x55bcb39e9dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 -- 172.21.15.28:0/370683040 shutdown_connections 2024-05-20T20:08:37.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.068+0000 7fd6a441f740 1 -- 172.21.15.28:0/370683040 wait complete. 2024-05-20T20:08:37.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.069+0000 7fd6a441f740 1 -- start start 2024-05-20T20:08:37.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.070+0000 7fd6a441f740 1 Processor -- start 2024-05-20T20:08:37.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.070+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a03280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f35b0 0x55bcb3a037c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd6a441f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a70070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a03280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd69aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a03280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41468/0 (socket says 172.21.15.28:41468) 2024-05-20T20:08:37.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd69aa00640 1 -- 172.21.15.28:0/629126563 learned_addr learned my addr 172.21.15.28:0/629126563 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd693e00640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f35b0 0x55bcb3a037c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd69b400640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a70070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c1ad830 con 0x55bcb39f35b0 2024-05-20T20:08:37.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c1ad160 con 0x55bcb39f2c90 2024-05-20T20:08:37.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.071+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd69c0507e0 con 0x55bcb39fde70 2024-05-20T20:08:37.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.072+0000 7fd6a441f740 10 client.0 _ll_register_callbacks cb 0x7fd694009a00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:37.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.072+0000 7fd693e00640 1 -- 172.21.15.28:0/629126563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 msgr2=0x55bcb3a70070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.072+0000 7fd693e00640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a70070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.072+0000 7fd693e00640 1 -- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 msgr2=0x55bcb3a03280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.072+0000 7fd693e00640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a03280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.072+0000 7fd693e00640 1 -- 172.21.15.28:0/629126563 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bcb3a70660 con 0x55bcb39f35b0 2024-05-20T20:08:37.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.072+0000 7fd69aa00640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a03280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:37.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.073+0000 7fd693e00640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f35b0 0x55bcb3a037c0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fd684018ef0 tx=0x7fd684005ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8f7c54f09969fdbe server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.073+0000 7fd67fe00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:37.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.073+0000 7fd67fe00640 1 -- 172.21.15.28:0/629126563 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd6840107a0 con 0x55bcb39f35b0 2024-05-20T20:08:37.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.073+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55bcb395f2c0 con 0x55bcb39f35b0 2024-05-20T20:08:37.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.073+0000 7fd67fe00640 1 -- 172.21.15.28:0/629126563 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd684005210 con 0x55bcb39f35b0 2024-05-20T20:08:37.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.073+0000 7fd67fe00640 1 -- 172.21.15.28:0/629126563 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd6840035f0 con 0x55bcb39f35b0 2024-05-20T20:08:37.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 10 client.15990 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:37.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55bcb3a779f0 con 0x55bcb39f35b0 2024-05-20T20:08:37.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd67fe00640 1 -- 172.21.15.28:0/629126563 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55bcb395f2c0 con 0x55bcb39f35b0 2024-05-20T20:08:37.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd67fe00640 1 -- 172.21.15.28:0/629126563 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd68402a040 con 0x55bcb39f35b0 2024-05-20T20:08:37.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 10 client.15990 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 10 client.15990 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 20 client.15990 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a77fa0 0x55bcb3a7a3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 4 client.15990 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.074+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55bcb38d94b0 con 0x55bcb3a77fa0 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.075+0000 7fd69aa00640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a77fa0 0x55bcb3a7a3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.076+0000 7fd69aa00640 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a77fa0 0x55bcb3a7a3c0 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=71f1af239c7d87e2 server_cookie=6530d56d3d566a4 in_seq=0 out_seq=0 2024-05-20T20:08:37.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.076+0000 7fd67fe00640 10 client.15990 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd67fe00640 1 -- 172.21.15.28:0/629126563 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55bcb38d94b0 con 0x55bcb3a77fa0 2024-05-20T20:08:37.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd67fe00640 10 client.15990 handle_command_reply: tid=0 2024-05-20T20:08:37.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 client.15990 shutdown 2024-05-20T20:08:37.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f35b0 msgr2=0x55bcb3a037c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f35b0 0x55bcb3a037c0 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fd684018ef0 tx=0x7fd684005ab0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 shutdown_connections 2024-05-20T20:08:37.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bcb3a77fa0 0x55bcb3a7a3c0 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bcb39fde70 0x55bcb3a70070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bcb39f35b0 0x55bcb3a037c0 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 --2- 172.21.15.28:0/629126563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bcb39f2c90 0x55bcb3a03280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.097+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 >> 172.21.15.28:0/629126563 conn(0x55bcb39ef1a0 msgr2=0x55bcb39f9480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.098+0000 7fd67fe00640 0 client.15990 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.098+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 shutdown_connections 2024-05-20T20:08:37.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.098+0000 7fd6a441f740 1 -- 172.21.15.28:0/629126563 wait complete. 2024-05-20T20:08:37.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.098+0000 7fd6a441f740 20 client.15990 trim_cache size 0 max 16384 2024-05-20T20:08:37.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:08:37.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:08:37.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:08:37.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd6800b2b50 msgr2=0x7fd6800b5010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd6800b2b50 0x7fd6800b5010 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fd694005fd0 tx=0x7fd6940096a0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 msgr2=0x7fd69c110620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c110620 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fd68c031440 tx=0x7fd68c0045e0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 shutdown_connections 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd69c1a5310 0x7fd69c1a57c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd6800b2b50 0x7fd6800b5010 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd69c109f80 0x7fd69c110b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 --2- 172.21.15.28:0/2490075075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd69c104eb0 0x7fd69c110620 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 >> 172.21.15.28:0/2490075075 conn(0x7fd69c100750 msgr2=0x7fd69c101aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 shutdown_connections 2024-05-20T20:08:37.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:37.100+0000 7fd6a1600640 1 -- 172.21.15.28:0/2490075075 wait complete. 2024-05-20T20:08:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:36 smithi154 ceph-mon[36314]: pgmap v92: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:37.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:36 smithi033 ceph-mon[36179]: pgmap v92: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:37.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.579+0000 7f8557200700 1 -- 172.21.15.28:0/3999920837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810f9a0 msgr2=0x7f855810fd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.579+0000 7f8557200700 1 --2- 172.21.15.28:0/3999920837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810f9a0 0x7f855810fd80 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f8548009990 tx=0x7f8548009ca0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.580+0000 7f8557200700 1 -- 172.21.15.28:0/3999920837 shutdown_connections 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.580+0000 7f8557200700 1 --2- 172.21.15.28:0/3999920837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f855810a420 0x7f8558116870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.580+0000 7f8557200700 1 --2- 172.21.15.28:0/3999920837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810f9a0 0x7f855810fd80 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.580+0000 7f8557200700 1 --2- 172.21.15.28:0/3999920837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8558109a80 0x7f8558109ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.580+0000 7f8557200700 1 -- 172.21.15.28:0/3999920837 >> 172.21.15.28:0/3999920837 conn(0x7f85580792f0 msgr2=0x7f8558108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.580+0000 7f8557200700 1 -- 172.21.15.28:0/3999920837 shutdown_connections 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.580+0000 7f8557200700 1 -- 172.21.15.28:0/3999920837 wait complete. 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.582+0000 7f8557200700 1 Processor -- start 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.582+0000 7f8557200700 1 -- start start 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.583+0000 7f8557200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8558109a80 0x7f85581a0360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.583+0000 7f8557200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810a420 0x7f85581a08a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.583+0000 7f8557200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f85581a4c30 0x7f85581a50b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.583+0000 7f8557200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8558119c10 con 0x7f855810a420 2024-05-20T20:08:37.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.583+0000 7f8557200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8558119a90 con 0x7f8558109a80 2024-05-20T20:08:37.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.583+0000 7f8557200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8558119d90 con 0x7f85581a4c30 2024-05-20T20:08:37.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.583+0000 7f8555400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810a420 0x7f85581a08a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.584+0000 7f8556800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f85581a4c30 0x7f85581a50b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.584+0000 7f8556800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f85581a4c30 0x7f85581a50b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:45022/0 (socket says 172.21.15.28:45022) 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.584+0000 7f8556800700 1 -- 172.21.15.28:0/515307530 learned_addr learned my addr 172.21.15.28:0/515307530 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.585+0000 7f8555e00700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8558109a80 0x7f85581a0360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.585+0000 7f8555400700 1 -- 172.21.15.28:0/515307530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f85581a4c30 msgr2=0x7f85581a50b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.585+0000 7f8555400700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f85581a4c30 0x7f85581a50b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.585+0000 7f8555400700 1 -- 172.21.15.28:0/515307530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8558109a80 msgr2=0x7f85581a0360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.585+0000 7f8555400700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8558109a80 0x7f85581a0360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.585+0000 7f8555400700 1 -- 172.21.15.28:0/515307530 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f85581a5750 con 0x7f855810a420 2024-05-20T20:08:37.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.585+0000 7f8555e00700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8558109a80 0x7f85581a0360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:08:37.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.587+0000 7f8555400700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810a420 0x7f85581a08a0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f8544006fd0 tx=0x7f854400b730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.588+0000 7f854ea00700 1 -- 172.21.15.28:0/515307530 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8544015020 con 0x7f855810a420 2024-05-20T20:08:37.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.588+0000 7f854ea00700 1 -- 172.21.15.28:0/515307530 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8544011ce0 con 0x7f855810a420 2024-05-20T20:08:37.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.588+0000 7f854ea00700 1 -- 172.21.15.28:0/515307530 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8544010ce0 con 0x7f855810a420 2024-05-20T20:08:37.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.589+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f85581a5a40 con 0x7f855810a420 2024-05-20T20:08:37.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.589+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f855807c080 con 0x7f855810a420 2024-05-20T20:08:37.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.597+0000 7f854ea00700 1 -- 172.21.15.28:0/515307530 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f8544018020 con 0x7f855810a420 2024-05-20T20:08:37.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.600+0000 7f854ea00700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f853c0b2bb0 0x7f853c0b5070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.600+0000 7f854ea00700 1 -- 172.21.15.28:0/515307530 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f85440e1ac0 con 0x7f855810a420 2024-05-20T20:08:37.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.601+0000 7f85674f5b80 1 Processor -- start 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f85674f5b80 1 -- start start 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d7220 0x5572f81d7600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d14b0 0x5572f81d1910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81ddf70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f85674f5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5572f812fe40 con 0x5572f81d1e50 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f85674f5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5572f811f9e0 con 0x5572f81d7220 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f85674f5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5572f8123a20 con 0x5572f81d14b0 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.602+0000 7f8555e00700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f853c0b2bb0 0x7f853c0b5070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.603+0000 7f8556800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81ddf70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.604+0000 7f8555e00700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f853c0b2bb0 0x7f853c0b5070 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f8548000c00 tx=0x7f854800b540 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.604+0000 7f8556800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81ddf70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38666/0 (socket says 172.21.15.28:38666) 2024-05-20T20:08:37.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.604+0000 7f8556800700 1 -- 172.21.15.28:0/781012909 learned_addr learned my addr 172.21.15.28:0/781012909 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.605+0000 7f8555e00700 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d7220 0x5572f81d7600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.605+0000 7f8555400700 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d14b0 0x5572f81d1910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.605+0000 7f8556800700 1 -- 172.21.15.28:0/781012909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d14b0 msgr2=0x5572f81d1910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.605+0000 7f8556800700 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d14b0 0x5572f81d1910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.605+0000 7f8556800700 1 -- 172.21.15.28:0/781012909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d7220 msgr2=0x5572f81d7600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.605+0000 7f8556800700 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d7220 0x5572f81d7600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.605+0000 7f8556800700 1 -- 172.21.15.28:0/781012909 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5572f80e8420 con 0x5572f81d1e50 2024-05-20T20:08:37.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.606+0000 7f8556800700 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81ddf70 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f855000a730 tx=0x7f8550007990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=945b1196fc55be5e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.607+0000 7f8543400700 1 -- 172.21.15.28:0/781012909 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8550003a00 con 0x5572f81d1e50 2024-05-20T20:08:37.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.607+0000 7f8543400700 1 -- 172.21.15.28:0/781012909 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8550003ba0 con 0x5572f81d1e50 2024-05-20T20:08:37.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.607+0000 7f8543400700 1 -- 172.21.15.28:0/781012909 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8550014660 con 0x5572f81d1e50 2024-05-20T20:08:37.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.609+0000 7f85674f5b80 1 -- 172.21.15.28:0/781012909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 msgr2=0x5572f81ddf70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.609+0000 7f85674f5b80 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81ddf70 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f855000a730 tx=0x7f8550007990 comp rx=0 tx=0).stop 2024-05-20T20:08:37.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.609+0000 7f85674f5b80 1 -- 172.21.15.28:0/781012909 shutdown_connections 2024-05-20T20:08:37.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.609+0000 7f85674f5b80 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81ddf70 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.609+0000 7f85674f5b80 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d7220 0x5572f81d7600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.609+0000 7f85674f5b80 1 --2- 172.21.15.28:0/781012909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d14b0 0x5572f81d1910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.609+0000 7f85674f5b80 1 -- 172.21.15.28:0/781012909 >> 172.21.15.28:0/781012909 conn(0x5572f80ffbe0 msgr2=0x5572f81cc1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.610+0000 7f85674f5b80 1 -- 172.21.15.28:0/781012909 shutdown_connections 2024-05-20T20:08:37.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.610+0000 7f85674f5b80 1 -- 172.21.15.28:0/781012909 wait complete. 2024-05-20T20:08:37.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.613+0000 7f85674f5b80 1 -- start start 2024-05-20T20:08:37.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.613+0000 7f85674f5b80 1 Processor -- start 2024-05-20T20:08:37.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.615+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81ddaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.615+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d1e50 0x5572f81dae90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.615+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d7220 0x5572f81db3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f8555e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81ddaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f8555400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d1e50 0x5572f81dae90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f8555400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d1e50 0x5572f81dae90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:45032/0 (socket says 172.21.15.28:45032) 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f8556800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d7220 0x5572f81db3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f8556800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d7220 0x5572f81db3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38682/0 (socket says 172.21.15.28:38682) 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f85674f5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5572f8025fa0 con 0x5572f81d7220 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f85674f5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5572f80c0760 con 0x5572f81d14b0 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f85674f5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5572f81de4b0 con 0x5572f81d1e50 2024-05-20T20:08:37.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f85674f5b80 10 client.0 _ll_register_callbacks cb 0x5572f80c9c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.616+0000 7f8555400700 1 -- 172.21.15.28:0/4124362562 learned_addr learned my addr 172.21.15.28:0/4124362562 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.617+0000 7f8556800700 1 -- 172.21.15.28:0/4124362562 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d1e50 msgr2=0x5572f81dae90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.617+0000 7f8556800700 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d1e50 0x5572f81dae90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.617+0000 7f8556800700 1 -- 172.21.15.28:0/4124362562 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 msgr2=0x5572f81ddaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.617+0000 7f8556800700 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81ddaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.617+0000 7f8556800700 1 -- 172.21.15.28:0/4124362562 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5572f806f8d0 con 0x5572f81d7220 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.618+0000 7f8556800700 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d7220 0x5572f81db3d0 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f855000afd0 tx=0x7f8550007f00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f0e50c1a576ad865 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.618+0000 7f8537e00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.619+0000 7f8537e00700 1 -- 172.21.15.28:0/4124362562 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8550022070 con 0x5572f81d7220 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.619+0000 7f8537e00700 1 -- 172.21.15.28:0/4124362562 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8550004850 con 0x5572f81d7220 2024-05-20T20:08:37.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.619+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8550003ba0 con 0x5572f81d7220 2024-05-20T20:08:37.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.620+0000 7f8537e00700 1 -- 172.21.15.28:0/4124362562 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8550003a00 con 0x5572f81d7220 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.620+0000 7f85674f5b80 10 client.16002 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.620+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5572f80f0060 con 0x5572f81d7220 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.620+0000 7f8537e00700 1 -- 172.21.15.28:0/4124362562 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8550003ba0 con 0x5572f81d7220 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f8537e00700 1 -- 172.21.15.28:0/4124362562 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8550014400 con 0x5572f81d7220 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f85674f5b80 10 client.16002 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f85674f5b80 10 client.16002 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f85674f5b80 20 client.16002 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f85674f5b80 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f81eae10 0x5572f81ed200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f85674f5b80 4 client.16002 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:37.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5572f8131d90 con 0x5572f81eae10 2024-05-20T20:08:37.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.621+0000 7f8555e00700 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f81eae10 0x5572f81ed200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.623+0000 7f8555e00700 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f81eae10 0x5572f81ed200 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3dbf2b012c44f592 server_cookie=97552002cdff30a4 in_seq=0 out_seq=0 2024-05-20T20:08:37.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.623+0000 7f8537e00700 10 client.16002 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.624+0000 7f8537e00700 1 -- 172.21.15.28:0/4124362562 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5572f8131d90 con 0x5572f81eae10 2024-05-20T20:08:37.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.624+0000 7f8537e00700 10 client.16002 handle_command_reply: tid=0 2024-05-20T20:08:37.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.624+0000 7f85674f5b80 1 client.16002 shutdown 2024-05-20T20:08:37.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.625+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d7220 msgr2=0x5572f81db3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.625+0000 7f85674f5b80 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d7220 0x5572f81db3d0 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f855000afd0 tx=0x7f8550007f00 comp rx=0 tx=0).stop 2024-05-20T20:08:37.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.625+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 shutdown_connections 2024-05-20T20:08:37.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.625+0000 7f85674f5b80 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f81eae10 0x5572f81ed200 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.625+0000 7f85674f5b80 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d7220 0x5572f81db3d0 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.625+0000 7f85674f5b80 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81ddaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.626+0000 7f85674f5b80 1 --2- 172.21.15.28:0/4124362562 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d1e50 0x5572f81dae90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.626+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 >> 172.21.15.28:0/4124362562 conn(0x5572f80ffbe0 msgr2=0x5572f81d04a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.626+0000 7f8537e00700 0 client.16002 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.626+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 shutdown_connections 2024-05-20T20:08:37.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.627+0000 7f85674f5b80 1 -- 172.21.15.28:0/4124362562 wait complete. 2024-05-20T20:08:37.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.627+0000 7f85674f5b80 20 client.16002 trim_cache size 0 max 16384 2024-05-20T20:08:37.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.674+0000 7f85674f5b80 1 Processor -- start 2024-05-20T20:08:37.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.674+0000 7f85674f5b80 1 -- start start 2024-05-20T20:08:37.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.674+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81d9ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.675+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81d3af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.675+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81d4030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.675+0000 7f85674f5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8558083850 con 0x5572f81d1e50 2024-05-20T20:08:37.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.675+0000 7f85674f5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f85581bece0 con 0x5572f81d14b0 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.675+0000 7f85674f5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8558083f20 con 0x5572f81d7220 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.676+0000 7f8556800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81d4030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.676+0000 7f8556800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81d4030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:45040/0 (socket says 172.21.15.28:45040) 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.676+0000 7f8555e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81d9ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.676+0000 7f8555400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81d3af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.676+0000 7f8555e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81d9ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41508/0 (socket says 172.21.15.28:41508) 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.676+0000 7f8555e00700 1 -- 172.21.15.28:0/2619727010 learned_addr learned my addr 172.21.15.28:0/2619727010 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.677+0000 7f8555e00700 1 -- 172.21.15.28:0/2619727010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 msgr2=0x5572f81d4030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.677+0000 7f8555e00700 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81d4030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.677+0000 7f8555e00700 1 -- 172.21.15.28:0/2619727010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 msgr2=0x5572f81d3af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.677+0000 7f8555e00700 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81d3af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.677+0000 7f8555e00700 1 -- 172.21.15.28:0/2619727010 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5572f81e7710 con 0x5572f81d14b0 2024-05-20T20:08:37.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.677+0000 7f8555400700 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81d3af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:37.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.678+0000 7f8555e00700 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81d9ed0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f855810aa90 tx=0x7f8548020000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1d8ddadba2003790 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.679+0000 7f8542a00700 1 -- 172.21.15.28:0/2619727010 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8548037030 con 0x5572f81d14b0 2024-05-20T20:08:37.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.679+0000 7f8542a00700 1 -- 172.21.15.28:0/2619727010 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8548005690 con 0x5572f81d14b0 2024-05-20T20:08:37.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.679+0000 7f8542a00700 1 -- 172.21.15.28:0/2619727010 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f854800f550 con 0x5572f81d14b0 2024-05-20T20:08:37.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.680+0000 7f85674f5b80 1 -- 172.21.15.28:0/2619727010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 msgr2=0x5572f81d9ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.680+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81d9ed0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f855810aa90 tx=0x7f8548020000 comp rx=0 tx=0).stop 2024-05-20T20:08:37.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.680+0000 7f85674f5b80 1 -- 172.21.15.28:0/2619727010 shutdown_connections 2024-05-20T20:08:37.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.680+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81d4030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.680+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d14b0 0x5572f81d9ed0 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.680+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2619727010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81d1e50 0x5572f81d3af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.680+0000 7f85674f5b80 1 -- 172.21.15.28:0/2619727010 >> 172.21.15.28:0/2619727010 conn(0x5572f80ffbe0 msgr2=0x5572f81d97c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.681+0000 7f85674f5b80 1 -- 172.21.15.28:0/2619727010 shutdown_connections 2024-05-20T20:08:37.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.681+0000 7f85674f5b80 1 -- 172.21.15.28:0/2619727010 wait complete. 2024-05-20T20:08:37.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.682+0000 7f85674f5b80 1 -- start start 2024-05-20T20:08:37.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.682+0000 7f85674f5b80 1 Processor -- start 2024-05-20T20:08:37.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.683+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d1e50 0x5572f81f0410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.683+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81f0950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.683+0000 7f85674f5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81ed200 0x5572f81ed680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.683+0000 7f85674f5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f85581be610 con 0x5572f81ed200 2024-05-20T20:08:37.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.683+0000 7f85674f5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f85581bdf40 con 0x5572f81d1e50 2024-05-20T20:08:37.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.683+0000 7f85674f5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f85580817f0 con 0x5572f81d7220 2024-05-20T20:08:37.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.683+0000 7f85674f5b80 10 client.0 _ll_register_callbacks cb 0x5572f80c9c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d1e50 0x5572f81f0410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d1e50 0x5572f81f0410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41518/0 (socket says 172.21.15.28:41518) 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 -- 172.21.15.28:0/2496173894 learned_addr learned my addr 172.21.15.28:0/2496173894 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8556800700 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81ed200 0x5572f81ed680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 -- 172.21.15.28:0/2496173894 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 msgr2=0x5572f81f0950 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81f0950 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 -- 172.21.15.28:0/2496173894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81ed200 msgr2=0x5572f81ed680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81ed200 0x5572f81ed680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.684+0000 7f8555e00700 1 -- 172.21.15.28:0/2496173894 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5572f81edd50 con 0x5572f81d1e50 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.685+0000 7f8555e00700 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d1e50 0x5572f81f0410 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f8548037000 tx=0x7f8548003980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eb780682fc0aae89 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:37.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.685+0000 7f8537e00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.686+0000 7f8537e00700 1 -- 172.21.15.28:0/2496173894 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8548040070 con 0x5572f81d1e50 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.686+0000 7f8537e00700 1 -- 172.21.15.28:0/2496173894 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8548003dd0 con 0x5572f81d1e50 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.686+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8548005690 con 0x5572f81d1e50 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.687+0000 7f8537e00700 1 -- 172.21.15.28:0/2496173894 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f854800f550 con 0x5572f81d1e50 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.687+0000 7f85674f5b80 10 client.26147 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.687+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5572f8256420 con 0x5572f81d1e50 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.687+0000 7f8537e00700 1 -- 172.21.15.28:0/2496173894 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8548005690 con 0x5572f81d1e50 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.688+0000 7f8537e00700 1 -- 172.21.15.28:0/2496173894 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f854800fba0 con 0x5572f81d1e50 2024-05-20T20:08:37.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.688+0000 7f85674f5b80 10 client.26147 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:37.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.688+0000 7f85674f5b80 10 client.26147 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:37.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.688+0000 7f85674f5b80 20 client.26147 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:37.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.688+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f8256a50 0x5572f8258e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:37.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.688+0000 7f85674f5b80 4 client.26147 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:08:37.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.688+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x5572f812b570 con 0x5572f8256a50 2024-05-20T20:08:37.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.689+0000 7f8555400700 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f8256a50 0x5572f8258e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:37.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.690+0000 7f8555400700 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f8256a50 0x5572f8258e70 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=59fb6aec2a6550c3 server_cookie=3fab03956a454e52 in_seq=0 out_seq=0 2024-05-20T20:08:37.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.690+0000 7f8537e00700 10 client.26147 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.691+0000 7f8537e00700 1 -- 172.21.15.28:0/2496173894 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x5572f812b570 con 0x5572f8256a50 2024-05-20T20:08:37.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.691+0000 7f8537e00700 10 client.26147 handle_command_reply: tid=0 2024-05-20T20:08:37.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.691+0000 7f85674f5b80 1 client.26147 shutdown 2024-05-20T20:08:37.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.692+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d1e50 msgr2=0x5572f81f0410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.692+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d1e50 0x5572f81f0410 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f8548037000 tx=0x7f8548003980 comp rx=0 tx=0).stop 2024-05-20T20:08:37.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.693+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 shutdown_connections 2024-05-20T20:08:37.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.693+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5572f8256a50 0x5572f8258e70 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.693+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5572f81ed200 0x5572f81ed680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.693+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5572f81d1e50 0x5572f81f0410 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.693+0000 7f85674f5b80 1 --2- 172.21.15.28:0/2496173894 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5572f81d7220 0x5572f81f0950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.693+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 >> 172.21.15.28:0/2496173894 conn(0x5572f80ffbe0 msgr2=0x5572f81d04a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.693+0000 7f8537e00700 0 client.26147 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:37.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.701+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 shutdown_connections 2024-05-20T20:08:37.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.701+0000 7f85674f5b80 1 -- 172.21.15.28:0/2496173894 wait complete. 2024-05-20T20:08:37.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.704+0000 7f85674f5b80 20 client.26147 trim_cache size 0 max 16384 2024-05-20T20:08:37.707 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:08:37.707 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:08:37.707 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:08:37.707 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.707+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f853c0b2bb0 msgr2=0x7f853c0b5070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.707+0000 7f8557200700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f853c0b2bb0 0x7f853c0b5070 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f8548000c00 tx=0x7f854800b540 comp rx=0 tx=0).stop 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.707+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810a420 msgr2=0x7f85581a08a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.707+0000 7f8557200700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810a420 0x7f85581a08a0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f8544006fd0 tx=0x7f854400b730 comp rx=0 tx=0).stop 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.709+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 shutdown_connections 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.709+0000 7f8557200700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f85581a4c30 0x7f85581a50b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.709+0000 7f8557200700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f853c0b2bb0 0x7f853c0b5070 secure :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f8548000c00 tx=0x7f854800b540 comp rx=0 tx=0).stop 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.709+0000 7f8557200700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8558109a80 0x7f85581a0360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.709+0000 7f8557200700 1 --2- 172.21.15.28:0/515307530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f855810a420 0x7f85581a08a0 secure :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f8544006fd0 tx=0x7f854400b730 comp rx=0 tx=0).stop 2024-05-20T20:08:37.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.709+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 >> 172.21.15.28:0/515307530 conn(0x7f85580792f0 msgr2=0x7f8558079a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:37.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.711+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 shutdown_connections 2024-05-20T20:08:37.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:37.711+0000 7f8557200700 1 -- 172.21.15.28:0/515307530 wait complete. 2024-05-20T20:08:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:38 smithi154 ceph-mon[36314]: pgmap v93: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:39.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:38 smithi033 ceph-mon[36179]: pgmap v93: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:39.327 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:38 smithi028 ceph-mon[31178]: pgmap v93: 97 pgs: 97 active+clean; 1.4 MiB data, 344 MiB used, 1.0 TiB / 1.0 TiB avail; 5.4 KiB/s rd, 255 B/s wr, 9 op/s 2024-05-20T20:08:41.029 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:08:41.029 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:08:41.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:40 smithi154 ceph-mon[36314]: pgmap v94: 97 pgs: 97 active+clean; 218 MiB data, 361 MiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 18 MiB/s wr, 31 op/s 2024-05-20T20:08:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:40 smithi033 ceph-mon[36179]: pgmap v94: 97 pgs: 97 active+clean; 218 MiB data, 361 MiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 18 MiB/s wr, 31 op/s 2024-05-20T20:08:41.130 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:40 smithi028 ceph-mon[31178]: pgmap v94: 97 pgs: 97 active+clean; 218 MiB data, 361 MiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 18 MiB/s wr, 31 op/s 2024-05-20T20:08:41.944 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:43.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:42 smithi028 ceph-mon[31178]: pgmap v95: 97 pgs: 97 active+clean; 218 MiB data, 361 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 18 MiB/s wr, 27 op/s 2024-05-20T20:08:43.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:42 smithi154 ceph-mon[36314]: pgmap v95: 97 pgs: 97 active+clean; 218 MiB data, 361 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 18 MiB/s wr, 27 op/s 2024-05-20T20:08:43.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:42 smithi033 ceph-mon[36179]: pgmap v95: 97 pgs: 97 active+clean; 218 MiB data, 361 MiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 18 MiB/s wr, 27 op/s 2024-05-20T20:08:44.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:43 smithi028 ceph-mon[31178]: pgmap v96: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 151 MiB/s wr, 67 op/s 2024-05-20T20:08:44.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:43 smithi154 ceph-mon[36314]: pgmap v96: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 151 MiB/s wr, 67 op/s 2024-05-20T20:08:44.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:43 smithi033 ceph-mon[36179]: pgmap v96: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 151 MiB/s wr, 67 op/s 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.461+0000 7f9037400700 1 -- 172.21.15.28:0/3575956917 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90301035f0 msgr2=0x7f9030103a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.461+0000 7f9037400700 1 --2- 172.21.15.28:0/3575956917 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90301035f0 0x7f9030103a50 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f901c0098f0 tx=0x7f901c009c00 comp rx=0 tx=0).stop 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.461+0000 7f9037400700 1 -- 172.21.15.28:0/3575956917 shutdown_connections 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.461+0000 7f9037400700 1 --2- 172.21.15.28:0/3575956917 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9030103f90 0x7f9030110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.461+0000 7f9037400700 1 --2- 172.21.15.28:0/3575956917 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030108500 0x7f90301088e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.461+0000 7f9037400700 1 --2- 172.21.15.28:0/3575956917 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90301035f0 0x7f9030103a50 secure :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f901c0098f0 tx=0x7f901c009c00 comp rx=0 tx=0).stop 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.461+0000 7f9037400700 1 -- 172.21.15.28:0/3575956917 >> 172.21.15.28:0/3575956917 conn(0x7f903007a6a0 msgr2=0x7f903007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:46.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.462+0000 7f9037400700 1 -- 172.21.15.28:0/3575956917 shutdown_connections 2024-05-20T20:08:46.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.462+0000 7f9037400700 1 -- 172.21.15.28:0/3575956917 wait complete. 2024-05-20T20:08:46.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.463+0000 7f9037400700 1 Processor -- start 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9037400700 1 -- start start 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9037400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030103f90 0x7f903010d650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9037400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9030108500 0x7f903010db90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9037400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f903010e0d0 0x7f90301ac6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9037400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f90300771d0 con 0x7f9030108500 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9037400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9030077050 con 0x7f9030103f90 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9037400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9030077350 con 0x7f903010e0d0 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9034a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030103f90 0x7f903010d650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:46.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.464+0000 7f9035400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f903010e0d0 0x7f90301ac6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9035400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f903010e0d0 0x7f90301ac6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:45050/0 (socket says 172.21.15.28:45050) 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9035400700 1 -- 172.21.15.28:0/720819073 learned_addr learned my addr 172.21.15.28:0/720819073 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9034a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030103f90 0x7f903010d650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41536/0 (socket says 172.21.15.28:41536) 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9034a00700 1 -- 172.21.15.28:0/720819073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f903010e0d0 msgr2=0x7f90301ac6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9034a00700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f903010e0d0 0x7f90301ac6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9034a00700 1 -- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9030108500 msgr2=0x7f903010db90 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9034a00700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9030108500 0x7f903010db90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.465+0000 7f9034a00700 1 -- 172.21.15.28:0/720819073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f903006cf60 con 0x7f9030103f90 2024-05-20T20:08:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.466+0000 7f9034a00700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030103f90 0x7f903010d650 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f902400d740 tx=0x7f902400da50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.466+0000 7f902d600700 1 -- 172.21.15.28:0/720819073 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9024017070 con 0x7f9030103f90 2024-05-20T20:08:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.466+0000 7f902d600700 1 -- 172.21.15.28:0/720819073 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f902400f690 con 0x7f9030103f90 2024-05-20T20:08:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.466+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f903006d250 con 0x7f9030103f90 2024-05-20T20:08:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.466+0000 7f902d600700 1 -- 172.21.15.28:0/720819073 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9024003b10 con 0x7f9030103f90 2024-05-20T20:08:46.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.467+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f903006d6d0 con 0x7f9030103f90 2024-05-20T20:08:46.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.469+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9030104cb0 con 0x7f9030103f90 2024-05-20T20:08:46.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.470+0000 7f902d600700 1 -- 172.21.15.28:0/720819073 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f9024003cb0 con 0x7f9030103f90 2024-05-20T20:08:46.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.471+0000 7f902d600700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90180b29f0 0x7f90180b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:46.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.472+0000 7f902d600700 1 -- 172.21.15.28:0/720819073 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f90240e1d20 con 0x7f9030103f90 2024-05-20T20:08:46.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.472+0000 7f902fe00700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90180b29f0 0x7f90180b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:46.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.473+0000 7f902fe00700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90180b29f0 0x7f90180b4eb0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f903010ee10 tx=0x7f901c0058e0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:46.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.476+0000 7f902d600700 1 -- 172.21.15.28:0/720819073 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f90240a9710 con 0x7f9030103f90 2024-05-20T20:08:46.775 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:46 smithi028 ceph-mon[31178]: pgmap v97: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 151 MiB/s wr, 61 op/s 2024-05-20T20:08:46.965 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.964+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f903010eb80 con 0x7f9030103f90 2024-05-20T20:08:46.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.965+0000 7f902d600700 1 -- 172.21.15.28:0/720819073 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f90240a9010 con 0x7f9030103f90 2024-05-20T20:08:46.966 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:08:46.966 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:08:46.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90180b29f0 msgr2=0x7f90180b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:46.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90180b29f0 0x7f90180b4eb0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f903010ee10 tx=0x7f901c0058e0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030103f90 msgr2=0x7f903010d650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:46.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030103f90 0x7f903010d650 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f902400d740 tx=0x7f902400da50 comp rx=0 tx=0).stop 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 shutdown_connections 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f903010e0d0 0x7f90301ac6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f90180b29f0 0x7f90180b4eb0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9030103f90 0x7f903010d650 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 --2- 172.21.15.28:0/720819073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9030108500 0x7f903010db90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 >> 172.21.15.28:0/720819073 conn(0x7f903007a6a0 msgr2=0x7f9030101800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.968+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 shutdown_connections 2024-05-20T20:08:46.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:46.969+0000 7f9037400700 1 -- 172.21.15.28:0/720819073 wait complete. 2024-05-20T20:08:46.972 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:08:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:46 smithi154 ceph-mon[36314]: pgmap v97: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 151 MiB/s wr, 61 op/s 2024-05-20T20:08:47.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:46 smithi033 ceph-mon[36179]: pgmap v97: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 151 MiB/s wr, 61 op/s 2024-05-20T20:08:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:47 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/720819073' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:48.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:47 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/720819073' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:48.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:47 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/720819073' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:08:48.566 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:08:48.795 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:48 smithi028 ceph-mon[31178]: pgmap v98: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 151 MiB/s wr, 61 op/s 2024-05-20T20:08:49.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:48 smithi154 ceph-mon[36314]: pgmap v98: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 151 MiB/s wr, 61 op/s 2024-05-20T20:08:49.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:48 smithi033 ceph-mon[36179]: pgmap v98: 97 pgs: 97 active+clean; 1.8 GiB data, 673 MiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 151 MiB/s wr, 61 op/s 2024-05-20T20:08:49.441 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:50.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.304+0000 7eff69800640 1 -- 172.21.15.28:0/4157157026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff6410a370 msgr2=0x7eff64115660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.304+0000 7eff69800640 1 --2- 172.21.15.28:0/4157157026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff6410a370 0x7eff64115660 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7eff4c009930 tx=0x7eff4c02a4f0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.306+0000 7eff69800640 1 -- 172.21.15.28:0/4157157026 shutdown_connections 2024-05-20T20:08:50.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.306+0000 7eff69800640 1 --2- 172.21.15.28:0/4157157026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff6410a370 0x7eff64115660 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.306+0000 7eff69800640 1 --2- 172.21.15.28:0/4157157026 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7eff64115ba0 0x7eff64117f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.306+0000 7eff69800640 1 --2- 172.21.15.28:0/4157157026 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 0x7eff64109e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.306+0000 7eff69800640 1 -- 172.21.15.28:0/4157157026 >> 172.21.15.28:0/4157157026 conn(0x7eff6406db20 msgr2=0x7eff6406df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:50.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.307+0000 7eff69800640 1 -- 172.21.15.28:0/4157157026 shutdown_connections 2024-05-20T20:08:50.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.307+0000 7eff69800640 1 -- 172.21.15.28:0/4157157026 wait complete. 2024-05-20T20:08:50.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.309+0000 7eff69800640 1 Processor -- start 2024-05-20T20:08:50.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.310+0000 7eff69800640 1 -- start start 2024-05-20T20:08:50.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.310+0000 7eff69800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 0x7eff641a9930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.311+0000 7eff69800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7eff6410a370 0x7eff641a9e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.311+0000 7eff62a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 0x7eff641a9930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.311+0000 7eff62a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 0x7eff641a9930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54316/0 (socket says 172.21.15.28:54316) 2024-05-20T20:08:50.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.312+0000 7eff69800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff64115ba0 0x7eff641a3ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.312+0000 7eff69800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7eff6411b580 con 0x7eff64109a50 2024-05-20T20:08:50.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.312+0000 7eff62a00640 1 -- 172.21.15.28:0/2198282022 learned_addr learned my addr 172.21.15.28:0/2198282022 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:50.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.312+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7eff6411b400 con 0x7eff64115ba0 2024-05-20T20:08:50.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.312+0000 7eff5a000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7eff6410a370 0x7eff641a9e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.312+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7eff6411b700 con 0x7eff6410a370 2024-05-20T20:08:50.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.312+0000 7eff63400640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff64115ba0 0x7eff641a3ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.313+0000 7eff5a000640 1 -- 172.21.15.28:0/2198282022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff64115ba0 msgr2=0x7eff641a3ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.313+0000 7eff5a000640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff64115ba0 0x7eff641a3ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.313+0000 7eff5a000640 1 -- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 msgr2=0x7eff641a9930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.313+0000 7eff5a000640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 0x7eff641a9930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.313+0000 7eff5a000640 1 -- 172.21.15.28:0/2198282022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7eff641a4260 con 0x7eff6410a370 2024-05-20T20:08:50.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.313+0000 7eff63400640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff64115ba0 0x7eff641a3ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:50.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.313+0000 7eff62a00640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 0x7eff641a9930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:50.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.314+0000 7eff5a000640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7eff6410a370 0x7eff641a9e70 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7eff4c009850 tx=0x7eff4c003df0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:50.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.315+0000 7eff5be00640 1 -- 172.21.15.28:0/2198282022 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff4c03f070 con 0x7eff6410a370 2024-05-20T20:08:50.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.315+0000 7eff5be00640 1 -- 172.21.15.28:0/2198282022 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7eff4c031d60 con 0x7eff6410a370 2024-05-20T20:08:50.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.315+0000 7eff5be00640 1 -- 172.21.15.28:0/2198282022 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff4c035030 con 0x7eff6410a370 2024-05-20T20:08:50.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.315+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7eff641a44f0 con 0x7eff6410a370 2024-05-20T20:08:50.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.315+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7eff641b07f0 con 0x7eff6410a370 2024-05-20T20:08:50.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.320+0000 7eff5be00640 1 -- 172.21.15.28:0/2198282022 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7eff4c042070 con 0x7eff6410a370 2024-05-20T20:08:50.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.322+0000 7eff5be00640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7eff480b2c20 0x7eff480b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.322+0000 7eff5be00640 1 -- 172.21.15.28:0/2198282022 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7eff4c102d60 con 0x7eff6410a370 2024-05-20T20:08:50.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.323+0000 7eff62a00640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7eff480b2c20 0x7eff480b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.324+0000 7eff62a00640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7eff480b2c20 0x7eff480b50e0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7eff5400f4d0 tx=0x7eff5400a380 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:50.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 Processor -- start 2024-05-20T20:08:50.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 -- start start 2024-05-20T20:08:50.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6e9070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e95b0 0x564d6e6f3930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6f6260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564d6e659b90 con 0x564d6e6f3e70 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564d6e649730 con 0x564d6e6e95b0 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.326+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564d6e64d770 con 0x564d6e6e8c90 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6f6260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6f6260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54320/0 (socket says 172.21.15.28:54320) 2024-05-20T20:08:50.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 -- 172.21.15.28:0/1442193295 learned_addr learned my addr 172.21.15.28:0/1442193295 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:50.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 -- 172.21.15.28:0/1442193295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 msgr2=0x564d6e6e9070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:50.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 --2- 172.21.15.28:0/1442193295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6e9070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 -- 172.21.15.28:0/1442193295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e95b0 msgr2=0x564d6e6f3930 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:08:50.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 --2- 172.21.15.28:0/1442193295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e95b0 0x564d6e6f3930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.327+0000 7eff63400640 1 -- 172.21.15.28:0/1442193295 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564d6e5e28b0 con 0x564d6e6f3e70 2024-05-20T20:08:50.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.328+0000 7eff63400640 1 --2- 172.21.15.28:0/1442193295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6f6260 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7eff5c00a870 tx=0x7eff5c009650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4ab968c4fa366b31 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:50.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.329+0000 7eff53e00640 1 -- 172.21.15.28:0/1442193295 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff5c018020 con 0x564d6e6f3e70 2024-05-20T20:08:50.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.329+0000 7eff53e00640 1 -- 172.21.15.28:0/1442193295 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7eff5c014400 con 0x564d6e6f3e70 2024-05-20T20:08:50.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.329+0000 7eff53e00640 1 -- 172.21.15.28:0/1442193295 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff5c0135c0 con 0x564d6e6f3e70 2024-05-20T20:08:50.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.330+0000 7eff6c7ec740 1 -- 172.21.15.28:0/1442193295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 msgr2=0x564d6e6f6260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.330+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/1442193295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6f6260 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7eff5c00a870 tx=0x7eff5c009650 comp rx=0 tx=0).stop 2024-05-20T20:08:50.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.331+0000 7eff6c7ec740 1 -- 172.21.15.28:0/1442193295 shutdown_connections 2024-05-20T20:08:50.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.331+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/1442193295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e95b0 0x564d6e6f3930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.331+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/1442193295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6f6260 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.331+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/1442193295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6e9070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.331+0000 7eff6c7ec740 1 -- 172.21.15.28:0/1442193295 >> 172.21.15.28:0/1442193295 conn(0x564d6e6e51a0 msgr2=0x564d6e6e5580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:50.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.331+0000 7eff6c7ec740 1 -- 172.21.15.28:0/1442193295 shutdown_connections 2024-05-20T20:08:50.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.331+0000 7eff6c7ec740 1 -- 172.21.15.28:0/1442193295 wait complete. 2024-05-20T20:08:50.332 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.332+0000 7eff6c7ec740 1 -- start start 2024-05-20T20:08:50.332 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.332+0000 7eff6c7ec740 1 Processor -- start 2024-05-20T20:08:50.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.333+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.333+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 0x564d6e6f11a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.333+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e6f16e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.333+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564d6e32f790 con 0x564d6e6e8c90 2024-05-20T20:08:50.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.333+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564d6e5f1bc0 con 0x564d6e6f3e70 2024-05-20T20:08:50.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.333+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564d6e6f6aa0 con 0x564d6e6e95b0 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.333+0000 7eff6c7ec740 10 client.0 _ll_register_callbacks cb 0x564d6e5cf400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54332/0 (socket says 172.21.15.28:54332) 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 -- 172.21.15.28:0/3870537924 learned_addr learned my addr 172.21.15.28:0/3870537924 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff63400640 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e6f16e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 -- 172.21.15.28:0/3870537924 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 msgr2=0x564d6e6f11a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 0x564d6e6f11a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 -- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 msgr2=0x564d6e6f16e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:50.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e6f16e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.334+0000 7eff62a00640 1 -- 172.21.15.28:0/3870537924 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564d6e35e210 con 0x564d6e6e8c90 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.335+0000 7eff62a00640 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0c60 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7eff54010f00 tx=0x7eff540046c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c35e262fa6cdaad server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.335+0000 7eff50c00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.335+0000 7eff50c00640 1 -- 172.21.15.28:0/3870537924 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff54016070 con 0x564d6e6e8c90 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.335+0000 7eff50c00640 1 -- 172.21.15.28:0/3870537924 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7eff54027680 con 0x564d6e6e8c90 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.335+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7eff5c014400 con 0x564d6e6e8c90 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.336+0000 7eff50c00640 1 -- 172.21.15.28:0/3870537924 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7eff54027880 con 0x564d6e6e8c90 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.336+0000 7eff6c7ec740 10 client.16020 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.336+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564d6e42a620 con 0x564d6e6e8c90 2024-05-20T20:08:50.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.336+0000 7eff50c00640 1 -- 172.21.15.28:0/3870537924 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7eff5c014400 con 0x564d6e6e8c90 2024-05-20T20:08:50.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.336+0000 7eff50c00640 1 -- 172.21.15.28:0/3870537924 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7eff5401c420 con 0x564d6e6e8c90 2024-05-20T20:08:50.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.337+0000 7eff6c7ec740 10 client.16020 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:50.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.337+0000 7eff6c7ec740 10 client.16020 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:50.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.337+0000 7eff6c7ec740 20 client.16020 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:50.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.337+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e703950 0x564d6e705d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.337+0000 7eff6c7ec740 4 client.16020 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:50.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.337+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564d6e65bae0 con 0x564d6e703950 2024-05-20T20:08:50.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.338+0000 7eff5a000640 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e703950 0x564d6e705d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.339+0000 7eff5a000640 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e703950 0x564d6e705d40 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fef91b272ceae798 server_cookie=8f1047b82f6a417a in_seq=0 out_seq=0 2024-05-20T20:08:50.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.339+0000 7eff50c00640 10 client.16020 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:50.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.340+0000 7eff50c00640 1 -- 172.21.15.28:0/3870537924 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x564d6e65bae0 con 0x564d6e703950 2024-05-20T20:08:50.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.340+0000 7eff50c00640 10 client.16020 handle_command_reply: tid=0 2024-05-20T20:08:50.341 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 client.16020 shutdown 2024-05-20T20:08:50.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e8c90 msgr2=0x564d6e6f0c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0c60 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7eff54010f00 tx=0x7eff540046c0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 shutdown_connections 2024-05-20T20:08:50.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e703950 0x564d6e705d40 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e6f16e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 0x564d6e6f11a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3870537924 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0c60 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 >> 172.21.15.28:0/3870537924 conn(0x564d6e6e51a0 msgr2=0x564d6e6eaed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:50.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.342+0000 7eff50c00640 0 client.16020 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:50.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.344+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 shutdown_connections 2024-05-20T20:08:50.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.344+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3870537924 wait complete. 2024-05-20T20:08:50.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.344+0000 7eff6c7ec740 20 client.16020 trim_cache size 0 max 16384 2024-05-20T20:08:50.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.371+0000 7eff6c7ec740 1 Processor -- start 2024-05-20T20:08:50.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.372+0000 7eff6c7ec740 1 -- start start 2024-05-20T20:08:50.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.372+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.372+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e95b0 0x564d6e6ea640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.372+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e70b610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.372+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7eff641b9030 con 0x564d6e6e95b0 2024-05-20T20:08:50.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.372+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7eff641bab70 con 0x564d6e6f3e70 2024-05-20T20:08:50.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.372+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7eff641b9700 con 0x564d6e6e8c90 2024-05-20T20:08:50.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.373+0000 7eff63400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e70b610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.373+0000 7eff63400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e70b610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52290/0 (socket says 172.21.15.28:52290) 2024-05-20T20:08:50.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.373+0000 7eff63400640 1 -- 172.21.15.28:0/187416174 learned_addr learned my addr 172.21.15.28:0/187416174 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:50.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.373+0000 7eff62a00640 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.374+0000 7eff62a00640 1 -- 172.21.15.28:0/187416174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 msgr2=0x564d6e70b610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:50.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.374+0000 7eff62a00640 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e70b610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.374+0000 7eff62a00640 1 -- 172.21.15.28:0/187416174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e95b0 msgr2=0x564d6e6ea640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:50.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.374+0000 7eff62a00640 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e95b0 0x564d6e6ea640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.374+0000 7eff62a00640 1 -- 172.21.15.28:0/187416174 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564d6e7060b0 con 0x564d6e6e8c90 2024-05-20T20:08:50.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.374+0000 7eff62a00640 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0540 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7eff5401c9b0 tx=0x7eff5401bc90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b86909110e031c43 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:50.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.374+0000 7eff53400640 1 -- 172.21.15.28:0/187416174 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff54016040 con 0x564d6e6e8c90 2024-05-20T20:08:50.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.375+0000 7eff53400640 1 -- 172.21.15.28:0/187416174 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7eff54027680 con 0x564d6e6e8c90 2024-05-20T20:08:50.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.375+0000 7eff53400640 1 -- 172.21.15.28:0/187416174 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff54019600 con 0x564d6e6e8c90 2024-05-20T20:08:50.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.375+0000 7eff6c7ec740 1 -- 172.21.15.28:0/187416174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 msgr2=0x564d6e6f0540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.375+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0540 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7eff5401c9b0 tx=0x7eff5401bc90 comp rx=0 tx=0).stop 2024-05-20T20:08:50.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.376+0000 7eff6c7ec740 1 -- 172.21.15.28:0/187416174 shutdown_connections 2024-05-20T20:08:50.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.376+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6f3e70 0x564d6e70b610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.376+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6e95b0 0x564d6e6ea640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.376+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/187416174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e8c90 0x564d6e6f0540 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.376+0000 7eff6c7ec740 1 -- 172.21.15.28:0/187416174 >> 172.21.15.28:0/187416174 conn(0x564d6e6e51a0 msgr2=0x564d6e6ec160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:50.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.377+0000 7eff6c7ec740 1 -- 172.21.15.28:0/187416174 shutdown_connections 2024-05-20T20:08:50.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.377+0000 7eff6c7ec740 1 -- 172.21.15.28:0/187416174 wait complete. 2024-05-20T20:08:50.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.378+0000 7eff6c7ec740 1 -- start start 2024-05-20T20:08:50.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.378+0000 7eff6c7ec740 1 Processor -- start 2024-05-20T20:08:50.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.379+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e8c90 0x564d6e6fc3f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.379+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 0x564d6e6ed880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.379+0000 7eff6c7ec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6eddc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.379+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7eff641ba4a0 con 0x564d6e6f3e70 2024-05-20T20:08:50.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.379+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7eff641b9dd0 con 0x564d6e6e8c90 2024-05-20T20:08:50.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.379+0000 7eff6c7ec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7eff641b7010 con 0x564d6e6e95b0 2024-05-20T20:08:50.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.379+0000 7eff6c7ec740 10 client.0 _ll_register_callbacks cb 0x7eff5400acb0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:50.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.380+0000 7eff63400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6eddc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.380+0000 7eff63400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6eddc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54352/0 (socket says 172.21.15.28:54352) 2024-05-20T20:08:50.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.380+0000 7eff63400640 1 -- 172.21.15.28:0/3735295506 learned_addr learned my addr 172.21.15.28:0/3735295506 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:50.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.381+0000 7eff5a000640 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 0x564d6e6ed880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.381+0000 7eff63400640 1 -- 172.21.15.28:0/3735295506 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 msgr2=0x564d6e6ed880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:50.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.381+0000 7eff63400640 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 0x564d6e6ed880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.381+0000 7eff63400640 1 -- 172.21.15.28:0/3735295506 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e8c90 msgr2=0x564d6e6fc3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:50.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.381+0000 7eff63400640 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e8c90 0x564d6e6fc3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.381+0000 7eff63400640 1 -- 172.21.15.28:0/3735295506 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564d6e6ee5f0 con 0x564d6e6f3e70 2024-05-20T20:08:50.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.383+0000 7eff63400640 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6eddc0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7eff5c00a870 tx=0x7eff5c009f00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da6a7f0821cd827d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:50.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.383+0000 7eff50c00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:08:50.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.386+0000 7eff50c00640 1 -- 172.21.15.28:0/3735295506 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7eff5c018040 con 0x564d6e6f3e70 2024-05-20T20:08:50.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.386+0000 7eff50c00640 1 -- 172.21.15.28:0/3735295506 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7eff5c014610 con 0x564d6e6f3e70 2024-05-20T20:08:50.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.386+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7eff54027680 con 0x564d6e6f3e70 2024-05-20T20:08:50.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.387+0000 7eff50c00640 1 -- 172.21.15.28:0/3735295506 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7eff5c014810 con 0x564d6e6f3e70 2024-05-20T20:08:50.387 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.387+0000 7eff6c7ec740 10 client.16023 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:50.387 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.387+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564d6e776c90 con 0x564d6e6f3e70 2024-05-20T20:08:50.387 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.388+0000 7eff50c00640 1 -- 172.21.15.28:0/3735295506 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7eff54027680 con 0x564d6e6f3e70 2024-05-20T20:08:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.389+0000 7eff50c00640 1 -- 172.21.15.28:0/3735295506 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7eff5c026070 con 0x564d6e6f3e70 2024-05-20T20:08:50.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.389+0000 7eff6c7ec740 10 client.16023 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.389+0000 7eff6c7ec740 10 client.16023 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:08:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.389+0000 7eff6c7ec740 20 client.16023 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.389+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e7770d0 0x564d6e7794c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.389+0000 7eff6c7ec740 4 client.16023 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:08:50.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.389+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564d6e6552c0 con 0x564d6e7770d0 2024-05-20T20:08:50.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.390+0000 7eff62a00640 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e7770d0 0x564d6e7794c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:50.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.393+0000 7eff62a00640 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e7770d0 0x564d6e7794c0 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cc7d34c82e36fc0e server_cookie=b820c82e7fec7300 in_seq=0 out_seq=0 2024-05-20T20:08:50.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.393+0000 7eff50c00640 10 client.16023 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:50.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.412+0000 7eff50c00640 1 -- 172.21.15.28:0/3735295506 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x564d6e6552c0 con 0x564d6e7770d0 2024-05-20T20:08:50.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.412+0000 7eff50c00640 10 client.16023 handle_command_reply: tid=0 2024-05-20T20:08:50.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.412+0000 7eff6c7ec740 1 client.16023 shutdown 2024-05-20T20:08:50.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.412+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 msgr2=0x564d6e6eddc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.413+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6eddc0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7eff5c00a870 tx=0x7eff5c009f00 comp rx=0 tx=0).stop 2024-05-20T20:08:50.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 shutdown_connections 2024-05-20T20:08:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d6e7770d0 0x564d6e7794c0 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d6e6f3e70 0x564d6e6eddc0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d6e6e95b0 0x564d6e6ed880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 --2- 172.21.15.28:0/3735295506 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d6e6e8c90 0x564d6e6fc3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 >> 172.21.15.28:0/3735295506 conn(0x564d6e6e51a0 msgr2=0x564d6e6f0ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff50c00640 0 client.16023 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:08:50.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 shutdown_connections 2024-05-20T20:08:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 1 -- 172.21.15.28:0/3735295506 wait complete. 2024-05-20T20:08:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.416+0000 7eff6c7ec740 20 client.16023 trim_cache size 0 max 16384 2024-05-20T20:08:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:08:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:08:50.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:08:50.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.420+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7eff480b2c20 msgr2=0x7eff480b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.420+0000 7eff69800640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7eff480b2c20 0x7eff480b50e0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7eff5400f4d0 tx=0x7eff5400a380 comp rx=0 tx=0).stop 2024-05-20T20:08:50.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.420+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7eff6410a370 msgr2=0x7eff641a9e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:50.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.420+0000 7eff69800640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7eff6410a370 0x7eff641a9e70 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7eff4c009850 tx=0x7eff4c003df0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.421+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 shutdown_connections 2024-05-20T20:08:50.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.421+0000 7eff69800640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7eff64115ba0 0x7eff641a3ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.421+0000 7eff69800640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7eff480b2c20 0x7eff480b50e0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.421+0000 7eff69800640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7eff6410a370 0x7eff641a9e70 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.421+0000 7eff69800640 1 --2- 172.21.15.28:0/2198282022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7eff64109a50 0x7eff641a9930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:50.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.421+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 >> 172.21.15.28:0/2198282022 conn(0x7eff6406db20 msgr2=0x7eff6410b450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:50.424 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.421+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 shutdown_connections 2024-05-20T20:08:50.424 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:08:50.422+0000 7eff69800640 1 -- 172.21.15.28:0/2198282022 wait complete. 2024-05-20T20:08:50.792 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:50 smithi028 ceph-mon[31178]: pgmap v99: 97 pgs: 97 active+clean; 3.4 GiB data, 979 MiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 286 MiB/s wr, 101 op/s 2024-05-20T20:08:51.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:50 smithi154 ceph-mon[36314]: pgmap v99: 97 pgs: 97 active+clean; 3.4 GiB data, 979 MiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 286 MiB/s wr, 101 op/s 2024-05-20T20:08:51.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:50 smithi033 ceph-mon[36179]: pgmap v99: 97 pgs: 97 active+clean; 3.4 GiB data, 979 MiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 286 MiB/s wr, 101 op/s 2024-05-20T20:08:53.061 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:52 smithi028 ceph-mon[31178]: pgmap v100: 97 pgs: 97 active+clean; 3.4 GiB data, 979 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 268 MiB/s wr, 79 op/s 2024-05-20T20:08:53.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:52 smithi154 ceph-mon[36314]: pgmap v100: 97 pgs: 97 active+clean; 3.4 GiB data, 979 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 268 MiB/s wr, 79 op/s 2024-05-20T20:08:53.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:52 smithi033 ceph-mon[36179]: pgmap v100: 97 pgs: 97 active+clean; 3.4 GiB data, 979 MiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 268 MiB/s wr, 79 op/s 2024-05-20T20:08:53.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.757+0000 7f78a4e00700 1 -- 172.21.15.28:0/1434966197 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a01046d0 msgr2=0x7f78a0104ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.759+0000 7f78a4e00700 1 --2- 172.21.15.28:0/1434966197 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a01046d0 0x7f78a0104ab0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f7888009990 tx=0x7f7888009ca0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.763+0000 7f78a4e00700 1 -- 172.21.15.28:0/1434966197 shutdown_connections 2024-05-20T20:08:53.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.763+0000 7f78a4e00700 1 --2- 172.21.15.28:0/1434966197 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f78a010dab0 0x7f78a010ff20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.763+0000 7f78a4e00700 1 --2- 172.21.15.28:0/1434966197 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a01046d0 0x7f78a0104ab0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.763+0000 7f78a4e00700 1 --2- 172.21.15.28:0/1434966197 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f78a0104ff0 0x7f78a010d4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.763+0000 7f78a4e00700 1 -- 172.21.15.28:0/1434966197 >> 172.21.15.28:0/1434966197 conn(0x7f78a00fde60 msgr2=0x7f78a0100280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:53.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.763+0000 7f78a4e00700 1 -- 172.21.15.28:0/1434966197 shutdown_connections 2024-05-20T20:08:53.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.763+0000 7f78a4e00700 1 -- 172.21.15.28:0/1434966197 wait complete. 2024-05-20T20:08:53.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.765+0000 7f78a4e00700 1 Processor -- start 2024-05-20T20:08:53.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.765+0000 7f78a4e00700 1 -- start start 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f78a4e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f78a01046d0 0x7f78a019e8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f78a4e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f78a0104ff0 0x7f78a0197960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f78a4e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a010dab0 0x7f78a0197ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f78a4e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f78a01115c0 con 0x7f78a01046d0 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f78a4e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f78a0111440 con 0x7f78a010dab0 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f78a4e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f78a0111740 con 0x7f78a0104ff0 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f789fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a010dab0 0x7f78a0197ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f789fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a010dab0 0x7f78a0197ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52330/0 (socket says 172.21.15.28:52330) 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.766+0000 7f789fe00700 1 -- 172.21.15.28:0/192274076 learned_addr learned my addr 172.21.15.28:0/192274076 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.767+0000 7f789f400700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f78a01046d0 0x7f78a019e8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.767+0000 7f789fe00700 1 -- 172.21.15.28:0/192274076 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f78a0104ff0 msgr2=0x7f78a0197960 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:08:53.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.767+0000 7f789fe00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f78a0104ff0 0x7f78a0197960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.767+0000 7f789fe00700 1 -- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f78a01046d0 msgr2=0x7f78a019e8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.767+0000 7f789fe00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f78a01046d0 0x7f78a019e8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.767+0000 7f789fe00700 1 -- 172.21.15.28:0/192274076 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f78a0198510 con 0x7f78a010dab0 2024-05-20T20:08:53.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.767+0000 7f789fe00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a010dab0 0x7f78a0197ea0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f789800b4a0 tx=0x7f789800b860 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:53.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.768+0000 7f789cc00700 1 -- 172.21.15.28:0/192274076 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7898015020 con 0x7f78a010dab0 2024-05-20T20:08:53.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.768+0000 7f789cc00700 1 -- 172.21.15.28:0/192274076 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7898011ce0 con 0x7f78a010dab0 2024-05-20T20:08:53.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.768+0000 7f789cc00700 1 -- 172.21.15.28:0/192274076 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7898010cd0 con 0x7f78a010dab0 2024-05-20T20:08:53.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.773+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f78a019c400 con 0x7f78a010dab0 2024-05-20T20:08:53.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.774+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f78a019c8c0 con 0x7f78a010dab0 2024-05-20T20:08:53.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.782+0000 7f789cc00700 1 -- 172.21.15.28:0/192274076 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7898018020 con 0x7f78a010dab0 2024-05-20T20:08:53.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.785+0000 7f789cc00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f78840b2a50 0x7f78840b4f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.786+0000 7f789cc00700 1 -- 172.21.15.28:0/192274076 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f78980e1b20 con 0x7f78a010dab0 2024-05-20T20:08:53.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.787+0000 7f789f400700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f78840b2a50 0x7f78840b4f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.792+0000 7f78b0ca6b80 1 Processor -- start 2024-05-20T20:08:53.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.792+0000 7f789f400700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f78840b2a50 0x7f78840b4f10 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f7888003820 tx=0x7f788800b540 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:53.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.793+0000 7f78b0ca6b80 1 -- start start 2024-05-20T20:08:53.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.793+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01afba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.793+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01a9d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.793+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01aa390 0x558ba01aa7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.793+0000 7f78b0ca6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558ba0107e40 con 0x558ba01aa390 2024-05-20T20:08:53.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.793+0000 7f78b0ca6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558ba00f79e0 con 0x558ba01af7c0 2024-05-20T20:08:53.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.793+0000 7f78b0ca6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558ba00fba20 con 0x558ba01b00e0 2024-05-20T20:08:53.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01afba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01afba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52334/0 (socket says 172.21.15.28:52334) 2024-05-20T20:08:53.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 -- 172.21.15.28:0/1602181537 learned_addr learned my addr 172.21.15.28:0/1602181537 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:53.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 -- 172.21.15.28:0/1602181537 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 msgr2=0x558ba01a9d70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:08:53.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 --2- 172.21.15.28:0/1602181537 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01a9d70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 -- 172.21.15.28:0/1602181537 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01aa390 msgr2=0x558ba01aa7f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:08:53.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 --2- 172.21.15.28:0/1602181537 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01aa390 0x558ba01aa7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.794+0000 7f789f400700 1 -- 172.21.15.28:0/1602181537 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558ba00c0420 con 0x558ba01af7c0 2024-05-20T20:08:53.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.795+0000 7f789f400700 1 --2- 172.21.15.28:0/1602181537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01afba0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f7888015000 tx=0x7f7888030000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aec90ae166b14a29 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:53.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.795+0000 7f7894c00700 1 -- 172.21.15.28:0/1602181537 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f788800f6e0 con 0x558ba01af7c0 2024-05-20T20:08:53.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.795+0000 7f7894c00700 1 -- 172.21.15.28:0/1602181537 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f788800fd30 con 0x558ba01af7c0 2024-05-20T20:08:53.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.795+0000 7f7894c00700 1 -- 172.21.15.28:0/1602181537 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7888036420 con 0x558ba01af7c0 2024-05-20T20:08:53.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.803+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1602181537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 msgr2=0x558ba01afba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.803+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1602181537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01afba0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f7888015000 tx=0x7f7888030000 comp rx=0 tx=0).stop 2024-05-20T20:08:53.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.804+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1602181537 shutdown_connections 2024-05-20T20:08:53.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.804+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1602181537 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01aa390 0x558ba01aa7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.804+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1602181537 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01afba0 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.804+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1602181537 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01a9d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.804+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1602181537 >> 172.21.15.28:0/1602181537 conn(0x558ba00d7be0 msgr2=0x558ba01a4810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:53.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.809+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1602181537 shutdown_connections 2024-05-20T20:08:53.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.809+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1602181537 wait complete. 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.810+0000 7f78b0ca6b80 1 -- start start 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.810+0000 7f78b0ca6b80 1 Processor -- start 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.812+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01b5ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.813+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01b3270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.813+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b37b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.814+0000 7f789fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b37b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.814+0000 7f789fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b37b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54386/0 (socket says 172.21.15.28:54386) 2024-05-20T20:08:53.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.814+0000 7f789fe00700 1 -- 172.21.15.28:0/409270218 learned_addr learned my addr 172.21.15.28:0/409270218 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:53.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.815+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558b9fffdfa0 con 0x558ba01b00e0 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.815+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558ba0098760 con 0x558ba01af7c0 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.815+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558ba01b6ba0 con 0x558ba01aa390 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.817+0000 7f7896a00700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01b3270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.817+0000 7f7896a00700 1 -- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 msgr2=0x558ba01b5ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.814+0000 7f789f400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01b5ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.820+0000 7f7896a00700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01b5ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.820+0000 7f7896a00700 1 -- 172.21.15.28:0/409270218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 msgr2=0x558ba01b37b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.820+0000 7f7896a00700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b37b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.820+0000 7f7896a00700 1 -- 172.21.15.28:0/409270218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558ba00478d0 con 0x558ba01af7c0 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.820+0000 7f789fe00700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b37b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.823+0000 7f789f400700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01b5ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:08:53.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.824+0000 7f78b0ca6b80 10 client.0 _ll_register_callbacks cb 0x558ba00a1c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:53.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.824+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f788800fd30 con 0x558ba01af7c0 2024-05-20T20:08:53.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.824+0000 7f7896a00700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01b3270 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f78a01991b0 tx=0x7f789000bbe0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2e6dd1d81e20e35c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:53.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.824+0000 7f788d600700 10 client.26177 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:08:53.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.825+0000 7f788d600700 1 -- 172.21.15.28:0/409270218 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f789001e050 con 0x558ba01af7c0 2024-05-20T20:08:53.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.825+0000 7f788d600700 1 -- 172.21.15.28:0/409270218 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7890010860 con 0x558ba01af7c0 2024-05-20T20:08:53.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.835+0000 7f788d600700 1 -- 172.21.15.28:0/409270218 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f7890014070 con 0x558ba01af7c0 2024-05-20T20:08:53.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.836+0000 7f78b0ca6b80 10 client.26177 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:53.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.836+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x558ba00c8060 con 0x558ba01af7c0 2024-05-20T20:08:53.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.837+0000 7f788d600700 1 -- 172.21.15.28:0/409270218 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7890010a60 con 0x558ba01af7c0 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.837+0000 7f78b0ca6b80 10 client.26177 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.837+0000 7f78b0ca6b80 10 client.26177 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.837+0000 7f78b0ca6b80 20 client.26177 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.837+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba01c35a0 0x558ba01c5990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.837+0000 7f78b0ca6b80 4 client.26177 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.837+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558ba0109d90 con 0x558ba01c35a0 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.839+0000 7f789f400700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba01c35a0 0x558ba01c5990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.840+0000 7f789f400700 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba01c35a0 0x558ba01c5990 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=99dc8625d266a25a server_cookie=26b4da433bf80b5e in_seq=0 out_seq=0 2024-05-20T20:08:53.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.840+0000 7f788d600700 10 client.26177 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:53.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f788d600700 1 -- 172.21.15.28:0/409270218 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558ba0109d90 con 0x558ba01c35a0 2024-05-20T20:08:53.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f788d600700 10 client.26177 handle_command_reply: tid=0 2024-05-20T20:08:53.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 client.26177 shutdown 2024-05-20T20:08:53.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 msgr2=0x558ba01b3270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01b3270 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f78a01991b0 tx=0x7f789000bbe0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 shutdown_connections 2024-05-20T20:08:53.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b37b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba01c35a0 0x558ba01c5990 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01b5ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/409270218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01af7c0 0x558ba01b3270 secure :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f78a01991b0 tx=0x7f789000bbe0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.841+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 >> 172.21.15.28:0/409270218 conn(0x558ba00d7be0 msgr2=0x558ba01a3fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:53.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.842+0000 7f788d600700 0 client.26177 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:53.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.844+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 shutdown_connections 2024-05-20T20:08:53.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.844+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/409270218 wait complete. 2024-05-20T20:08:53.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.845+0000 7f78b0ca6b80 20 client.26177 trim_cache size 0 max 16384 2024-05-20T20:08:53.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.874+0000 7f78b0ca6b80 1 Processor -- start 2024-05-20T20:08:53.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.874+0000 7f78b0ca6b80 1 -- start start 2024-05-20T20:08:53.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.875+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01a9d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.876+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b38a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.876+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01bfc40 0x558ba01b0e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.876+0000 7f78b0ca6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f78a0073970 con 0x558ba01b00e0 2024-05-20T20:08:53.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.876+0000 7f78b0ca6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f78a00754b0 con 0x558ba01bfc40 2024-05-20T20:08:53.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.876+0000 7f78b0ca6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f78a0074040 con 0x558ba01aa390 2024-05-20T20:08:53.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.877+0000 7f789fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01bfc40 0x558ba01b0e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f789fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01bfc40 0x558ba01b0e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52350/0 (socket says 172.21.15.28:52350) 2024-05-20T20:08:53.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f789fe00700 1 -- 172.21.15.28:0/340794662 learned_addr learned my addr 172.21.15.28:0/340794662 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:53.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f789f400700 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01a9d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f7896a00700 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f789fe00700 1 -- 172.21.15.28:0/340794662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 msgr2=0x558ba01a9d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f789fe00700 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01a9d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f789fe00700 1 -- 172.21.15.28:0/340794662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 msgr2=0x558ba01b38a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.878+0000 7f789fe00700 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b38a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.879+0000 7f789fe00700 1 -- 172.21.15.28:0/340794662 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558ba01b5b10 con 0x558ba01bfc40 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.879+0000 7f789f400700 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01a9d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.879+0000 7f7896a00700 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b38a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.879+0000 7f789fe00700 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01bfc40 0x558ba01b0e30 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f789800bc60 tx=0x7f7898009320 comp rx=0 tx=0).ready entity=mon.2 client_cookie=63264f25c394bf49 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.880+0000 7f788fe00700 1 -- 172.21.15.28:0/340794662 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f78980a8f20 con 0x558ba01bfc40 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.880+0000 7f788fe00700 1 -- 172.21.15.28:0/340794662 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f78980ad2d0 con 0x558ba01bfc40 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.880+0000 7f788fe00700 1 -- 172.21.15.28:0/340794662 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f78980b1b50 con 0x558ba01bfc40 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.881+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/340794662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01bfc40 msgr2=0x558ba01b0e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.881+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01bfc40 0x558ba01b0e30 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f789800bc60 tx=0x7f7898009320 comp rx=0 tx=0).stop 2024-05-20T20:08:53.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.882+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/340794662 shutdown_connections 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.882+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01bfc40 0x558ba01b0e30 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.882+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01aa390 0x558ba01a9d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.882+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/340794662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01b00e0 0x558ba01b38a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.882+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/340794662 >> 172.21.15.28:0/340794662 conn(0x558ba00d7be0 msgr2=0x558ba01b1b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.883+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/340794662 shutdown_connections 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.883+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/340794662 wait complete. 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.884+0000 7f78b0ca6b80 1 -- start start 2024-05-20T20:08:53.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.885+0000 7f78b0ca6b80 1 Processor -- start 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.887+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01aa390 0x558ba01cafe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.890+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01cb520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.890+0000 7f78b0ca6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01bfc40 0x558ba01cba60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.890+0000 7f789fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01bfc40 0x558ba01cba60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.891+0000 7f789fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01bfc40 0x558ba01cba60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:54414/0 (socket says 172.21.15.28:54414) 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.891+0000 7f789fe00700 1 -- 172.21.15.28:0/1775548563 learned_addr learned my addr 172.21.15.28:0/1775548563 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.891+0000 7f7896a00700 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01cb520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.891+0000 7f789f400700 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01aa390 0x558ba01cafe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.891+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f78a0074de0 con 0x558ba01bfc40 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.891+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f78a0074710 con 0x558ba01aa390 2024-05-20T20:08:53.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.891+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f78a0071910 con 0x558ba01b00e0 2024-05-20T20:08:53.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.892+0000 7f78b0ca6b80 10 client.0 _ll_register_callbacks cb 0x558ba00a1c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:53.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.893+0000 7f7896a00700 1 -- 172.21.15.28:0/1775548563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01aa390 msgr2=0x558ba01cafe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:53.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.893+0000 7f7896a00700 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01aa390 0x558ba01cafe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.893+0000 7f7896a00700 1 -- 172.21.15.28:0/1775548563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01bfc40 msgr2=0x558ba01cba60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:53.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.893+0000 7f7896a00700 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01bfc40 0x558ba01cba60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.894+0000 7f7896a00700 1 -- 172.21.15.28:0/1775548563 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558ba022d3d0 con 0x558ba01b00e0 2024-05-20T20:08:53.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.895+0000 7f7896a00700 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01cb520 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x558ba01b4300 tx=0x7f7890004be0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=29f8b363e5b0b542 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:53.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.895+0000 7f788d600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:53.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.896+0000 7f788d600700 1 -- 172.21.15.28:0/1775548563 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f789001e040 con 0x558ba01b00e0 2024-05-20T20:08:53.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.897+0000 7f788d600700 1 -- 172.21.15.28:0/1775548563 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7890018a70 con 0x558ba01b00e0 2024-05-20T20:08:53.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.897+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f78980ad2d0 con 0x558ba01b00e0 2024-05-20T20:08:53.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.898+0000 7f788d600700 1 -- 172.21.15.28:0/1775548563 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f7890033050 con 0x558ba01b00e0 2024-05-20T20:08:53.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.898+0000 7f78b0ca6b80 10 client.26122 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:53.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.898+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558ba022e600 con 0x558ba01b00e0 2024-05-20T20:08:53.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.899+0000 7f788d600700 1 -- 172.21.15.28:0/1775548563 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f78980ad2d0 con 0x558ba01b00e0 2024-05-20T20:08:53.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.899+0000 7f788d600700 1 -- 172.21.15.28:0/1775548563 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7890019a00 con 0x558ba01b00e0 2024-05-20T20:08:53.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.900+0000 7f78b0ca6b80 10 client.26122 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:53.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.900+0000 7f78b0ca6b80 10 client.26122 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:08:53.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.900+0000 7f78b0ca6b80 20 client.26122 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:53.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.901+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba022eca0 0x558ba0231090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:53.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.901+0000 7f78b0ca6b80 4 client.26122 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:08:53.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.901+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x558ba0103570 con 0x558ba022eca0 2024-05-20T20:08:53.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.902+0000 7f789f400700 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba022eca0 0x558ba0231090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:53.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.902+0000 7f789f400700 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba022eca0 0x558ba0231090 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=f31a216ca4af0e3 server_cookie=b5cde226683fda4e in_seq=0 out_seq=0 2024-05-20T20:08:53.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.903+0000 7f788d600700 10 client.26122 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:53.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.903+0000 7f788d600700 1 -- 172.21.15.28:0/1775548563 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x558ba0103570 con 0x558ba022eca0 2024-05-20T20:08:53.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.903+0000 7f788d600700 10 client.26122 handle_command_reply: tid=0 2024-05-20T20:08:53.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.903+0000 7f78b0ca6b80 1 client.26122 shutdown 2024-05-20T20:08:53.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.911+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 msgr2=0x558ba01cb520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.911+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01cb520 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x558ba01b4300 tx=0x7f7890004be0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.912+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 shutdown_connections 2024-05-20T20:08:53.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.912+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558ba01bfc40 0x558ba01cba60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.912+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558ba01aa390 0x558ba01cafe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.912+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x558ba022eca0 0x558ba0231090 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.912+0000 7f78b0ca6b80 1 --2- 172.21.15.28:0/1775548563 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558ba01b00e0 0x558ba01cb520 secure :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0x558ba01b4300 tx=0x7f7890004be0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.912+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 >> 172.21.15.28:0/1775548563 conn(0x558ba00d7be0 msgr2=0x558ba01b23f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:53.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.912+0000 7f788d600700 0 client.26122 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:08:53.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.913+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 shutdown_connections 2024-05-20T20:08:53.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.914+0000 7f78b0ca6b80 1 -- 172.21.15.28:0/1775548563 wait complete. 2024-05-20T20:08:53.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.914+0000 7f78b0ca6b80 20 client.26122 trim_cache size 0 max 16384 2024-05-20T20:08:53.914 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:08:53.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f78840b2a50 msgr2=0x7f78840b4f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f78840b2a50 0x7f78840b4f10 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f7888003820 tx=0x7f788800b540 comp rx=0 tx=0).stop 2024-05-20T20:08:53.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a010dab0 msgr2=0x7f78a0197ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:53.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a010dab0 0x7f78a0197ea0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f789800b4a0 tx=0x7f789800b860 comp rx=0 tx=0).stop 2024-05-20T20:08:53.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 shutdown_connections 2024-05-20T20:08:53.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f78840b2a50 0x7f78840b4f10 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f78a010dab0 0x7f78a0197ea0 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f78a01046d0 0x7f78a019e8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 --2- 172.21.15.28:0/192274076 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f78a0104ff0 0x7f78a0197960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:53.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.923+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 >> 172.21.15.28:0/192274076 conn(0x7f78a00fde60 msgr2=0x7f78a00febd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:53.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.924+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 shutdown_connections 2024-05-20T20:08:53.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:53.925+0000 7f78a4e00700 1 -- 172.21.15.28:0/192274076 wait complete. 2024-05-20T20:08:55.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:54 smithi154 ceph-mon[36314]: pgmap v101: 97 pgs: 97 active+clean; 4.6 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 372 MiB/s wr, 113 op/s 2024-05-20T20:08:55.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:54 smithi033 ceph-mon[36179]: pgmap v101: 97 pgs: 97 active+clean; 4.6 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 372 MiB/s wr, 113 op/s 2024-05-20T20:08:55.209 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:54 smithi028 ceph-mon[31178]: pgmap v101: 97 pgs: 97 active+clean; 4.6 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 372 MiB/s wr, 113 op/s 2024-05-20T20:08:55.296 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:08:56.019 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:08:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:56 smithi028 ceph-mon[31178]: pgmap v102: 97 pgs: 97 active+clean; 5.0 GiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 272 MiB/s wr, 82 op/s 2024-05-20T20:08:57.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:56 smithi154 ceph-mon[36314]: pgmap v102: 97 pgs: 97 active+clean; 5.0 GiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 272 MiB/s wr, 82 op/s 2024-05-20T20:08:57.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:56 smithi033 ceph-mon[36179]: pgmap v102: 97 pgs: 97 active+clean; 5.0 GiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 272 MiB/s wr, 82 op/s 2024-05-20T20:08:58.991 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:58 smithi028 ceph-mon[31178]: pgmap v103: 97 pgs: 97 active+clean; 5.0 GiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 272 MiB/s wr, 82 op/s 2024-05-20T20:08:59.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:58 smithi154 ceph-mon[36314]: pgmap v103: 97 pgs: 97 active+clean; 5.0 GiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 272 MiB/s wr, 82 op/s 2024-05-20T20:08:59.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:58 smithi033 ceph-mon[36179]: pgmap v103: 97 pgs: 97 active+clean; 5.0 GiB data, 1.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 272 MiB/s wr, 82 op/s 2024-05-20T20:08:59.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.344+0000 7f5429c00700 1 -- 172.21.15.28:0/2543977408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5424106010 msgr2=0x7f5424112520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.344+0000 7f5429c00700 1 --2- 172.21.15.28:0/2543977408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5424106010 0x7f5424112520 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f5410009990 tx=0x7f5410009ca0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.344+0000 7f5429c00700 1 -- 172.21.15.28:0/2543977408 shutdown_connections 2024-05-20T20:08:59.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.344+0000 7f5429c00700 1 --2- 172.21.15.28:0/2543977408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5424106010 0x7f5424112520 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.344+0000 7f5429c00700 1 --2- 172.21.15.28:0/2543977408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f542410b650 0x7f542410ba30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.344+0000 7f5429c00700 1 --2- 172.21.15.28:0/2543977408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424105650 0x7f5424105ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.344+0000 7f5429c00700 1 -- 172.21.15.28:0/2543977408 >> 172.21.15.28:0/2543977408 conn(0x7f5424101120 msgr2=0x7f5424103540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:59.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.349+0000 7f5429c00700 1 -- 172.21.15.28:0/2543977408 shutdown_connections 2024-05-20T20:08:59.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.349+0000 7f5429c00700 1 -- 172.21.15.28:0/2543977408 wait complete. 2024-05-20T20:08:59.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.354+0000 7f5429c00700 1 Processor -- start 2024-05-20T20:08:59.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.355+0000 7f5429c00700 1 -- start start 2024-05-20T20:08:59.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.367+0000 7f5429c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5424105650 0x7f54241a17e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.368+0000 7f5429c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424106010 0x7f54241a1d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.368+0000 7f5429c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f542410b650 0x7f542419b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.369+0000 7f5429c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f54241158e0 con 0x7f5424105650 2024-05-20T20:08:59.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.369+0000 7f5429c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5424115760 con 0x7f5424106010 2024-05-20T20:08:59.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.370+0000 7f5429c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5424115a60 con 0x7f542410b650 2024-05-20T20:08:59.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.370+0000 7f5423400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424106010 0x7f54241a1d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.371+0000 7f5429200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f542410b650 0x7f542419b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.371+0000 7f5423400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424106010 0x7f54241a1d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34324/0 (socket says 172.21.15.28:34324) 2024-05-20T20:08:59.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.372+0000 7f5423400700 1 -- 172.21.15.28:0/3162539649 learned_addr learned my addr 172.21.15.28:0/3162539649 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:59.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.373+0000 7f5423e00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5424105650 0x7f54241a17e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.374+0000 7f5423400700 1 -- 172.21.15.28:0/3162539649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f542410b650 msgr2=0x7f542419b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.374+0000 7f5423400700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f542410b650 0x7f542419b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.374+0000 7f5423400700 1 -- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5424105650 msgr2=0x7f54241a17e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.374+0000 7f5423400700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5424105650 0x7f54241a17e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.374+0000 7f5423400700 1 -- 172.21.15.28:0/3162539649 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f542419c100 con 0x7f5424106010 2024-05-20T20:08:59.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.376+0000 7f5423400700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424106010 0x7f54241a1d20 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f541400b550 tx=0x7f541400b860 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:59.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.378+0000 7f5420c00700 1 -- 172.21.15.28:0/3162539649 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5414015020 con 0x7f5424106010 2024-05-20T20:08:59.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.379+0000 7f5420c00700 1 -- 172.21.15.28:0/3162539649 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5414011ce0 con 0x7f5424106010 2024-05-20T20:08:59.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.379+0000 7f5420c00700 1 -- 172.21.15.28:0/3162539649 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5414010d30 con 0x7f5424106010 2024-05-20T20:08:59.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.379+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f542419c3f0 con 0x7f5424106010 2024-05-20T20:08:59.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.379+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f54241a86b0 con 0x7f5424106010 2024-05-20T20:08:59.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.387+0000 7f5420c00700 1 -- 172.21.15.28:0/3162539649 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f5414018020 con 0x7f5424106010 2024-05-20T20:08:59.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.389+0000 7f5420c00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f54080b2b40 0x7f54080b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.392+0000 7f5435c76b80 1 Processor -- start 2024-05-20T20:08:59.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.392+0000 7f5435c76b80 1 -- start start 2024-05-20T20:08:59.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.393+0000 7f5420c00700 1 -- 172.21.15.28:0/3162539649 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f54140e1ba0 con 0x7f5424106010 2024-05-20T20:08:59.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.394+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b9320d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.394+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b931ea70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5423400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b931ea70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5423400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b931ea70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41428/0 (socket says 172.21.15.28:41428) 2024-05-20T20:08:59.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5423400700 1 -- 172.21.15.28:0/405009756 learned_addr learned my addr 172.21.15.28:0/405009756 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:59.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5435c76b80 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b9318d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5581b9279e40 con 0x5581b931efb0 2024-05-20T20:08:59.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5581b92699e0 con 0x5581b9320950 2024-05-20T20:08:59.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5581b926da20 con 0x5581b931e5f0 2024-05-20T20:08:59.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.396+0000 7f5423e00700 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b9320d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.397+0000 7f5423e00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f54080b2b40 0x7f54080b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.398+0000 7f5423e00700 1 -- 172.21.15.28:0/405009756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 msgr2=0x5581b931ea70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.398+0000 7f5423e00700 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b931ea70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.398+0000 7f5423e00700 1 -- 172.21.15.28:0/405009756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 msgr2=0x5581b9318d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.398+0000 7f5423e00700 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b9318d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.398+0000 7f5423e00700 1 -- 172.21.15.28:0/405009756 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5581b9232420 con 0x5581b9320950 2024-05-20T20:08:59.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.398+0000 7f5423e00700 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b9320d30 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f541c007810 tx=0x7f541c007b20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=70a80e881005871d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:59.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.399+0000 7f5419600700 1 -- 172.21.15.28:0/405009756 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f541c01e070 con 0x5581b9320950 2024-05-20T20:08:59.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.399+0000 7f5419600700 1 -- 172.21.15.28:0/405009756 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f541c0047e0 con 0x5581b9320950 2024-05-20T20:08:59.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.399+0000 7f5419600700 1 -- 172.21.15.28:0/405009756 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f541c0178b0 con 0x5581b9320950 2024-05-20T20:08:59.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.400+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 msgr2=0x5581b9320d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.400+0000 7f5435c76b80 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b9320d30 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f541c007810 tx=0x7f541c007b20 comp rx=0 tx=0).stop 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.400+0000 7f5423e00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f54080b2b40 0x7f54080b5000 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f541c021000 tx=0x7f541c01e000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.401+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 shutdown_connections 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.401+0000 7f5435c76b80 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b9318d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.401+0000 7f5435c76b80 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b9320d30 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.401+0000 7f5435c76b80 1 --2- 172.21.15.28:0/405009756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b931ea70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.401+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 >> 172.21.15.28:0/405009756 conn(0x5581b9249be0 msgr2=0x5581b9316810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:59.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.402+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 shutdown_connections 2024-05-20T20:08:59.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.402+0000 7f5435c76b80 1 -- 172.21.15.28:0/405009756 wait complete. 2024-05-20T20:08:59.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.404+0000 7f5435c76b80 1 -- start start 2024-05-20T20:08:59.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.404+0000 7f5435c76b80 1 Processor -- start 2024-05-20T20:08:59.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.406+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931e5f0 0x5581b933c580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.406+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931efb0 0x5581b933cac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.407+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 0x5581b93369a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.407+0000 7f5435c76b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5581b916ffa0 con 0x5581b9320950 2024-05-20T20:08:59.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.407+0000 7f5423400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931efb0 0x5581b933cac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.407+0000 7f5423400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931efb0 0x5581b933cac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41432/0 (socket says 172.21.15.28:41432) 2024-05-20T20:08:59.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.407+0000 7f5423400700 1 -- 172.21.15.28:0/1076800606 learned_addr learned my addr 172.21.15.28:0/1076800606 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:59.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.408+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5581b920a760 con 0x5581b931e5f0 2024-05-20T20:08:59.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.408+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5581b93193b0 con 0x5581b931efb0 2024-05-20T20:08:59.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.408+0000 7f5435c76b80 10 client.0 _ll_register_callbacks cb 0x5581b9213c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:59.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.409+0000 7f5423400700 1 -- 172.21.15.28:0/1076800606 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931e5f0 msgr2=0x5581b933c580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.409+0000 7f5423400700 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931e5f0 0x5581b933c580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.409+0000 7f5423400700 1 -- 172.21.15.28:0/1076800606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 msgr2=0x5581b93369a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.409+0000 7f5423400700 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 0x5581b93369a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.409+0000 7f5423400700 1 -- 172.21.15.28:0/1076800606 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5581b91b98d0 con 0x5581b931efb0 2024-05-20T20:08:59.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.411+0000 7f5423400700 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931efb0 0x5581b933cac0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f541400b520 tx=0x7f54140a8760 comp rx=0 tx=0).ready entity=mon.1 client_cookie=df0aa7e96d047b45 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:59.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.411+0000 7f540e000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:59.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.411+0000 7f540e000700 1 -- 172.21.15.28:0/1076800606 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f54140c3020 con 0x5581b931efb0 2024-05-20T20:08:59.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.411+0000 7f540e000700 1 -- 172.21.15.28:0/1076800606 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f54140a93b0 con 0x5581b931efb0 2024-05-20T20:08:59.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.412+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f541c0047e0 con 0x5581b931efb0 2024-05-20T20:08:59.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.413+0000 7f540e000700 1 -- 172.21.15.28:0/1076800606 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f54140a8f40 con 0x5581b931efb0 2024-05-20T20:08:59.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.414+0000 7f5435c76b80 10 client.26134 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:59.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.414+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5581b923a060 con 0x5581b931efb0 2024-05-20T20:08:59.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.414+0000 7f540e000700 1 -- 172.21.15.28:0/1076800606 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f541c0047e0 con 0x5581b931efb0 2024-05-20T20:08:59.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.414+0000 7f540e000700 1 -- 172.21.15.28:0/1076800606 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f54140a81a0 con 0x5581b931efb0 2024-05-20T20:08:59.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.416+0000 7f5435c76b80 10 client.26134 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:59.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.416+0000 7f5435c76b80 10 client.26134 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:08:59.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.416+0000 7f5435c76b80 20 client.26134 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:59.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.416+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93467a0 0x5581b9348c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.417+0000 7f5423e00700 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93467a0 0x5581b9348c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.417+0000 7f5435c76b80 4 client.26134 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:08:59.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.417+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5581b927bd90 con 0x5581b93467a0 2024-05-20T20:08:59.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.418+0000 7f540e000700 10 client.26134 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:59.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.418+0000 7f5423e00700 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93467a0 0x5581b9348c10 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=125a94b428476c82 server_cookie=f6585683e9d25cfb in_seq=0 out_seq=0 2024-05-20T20:08:59.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f540e000700 1 -- 172.21.15.28:0/1076800606 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5581b927bd90 con 0x5581b93467a0 2024-05-20T20:08:59.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f540e000700 10 client.26134 handle_command_reply: tid=0 2024-05-20T20:08:59.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 client.26134 shutdown 2024-05-20T20:08:59.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931efb0 msgr2=0x5581b933cac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931efb0 0x5581b933cac0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f541400b520 tx=0x7f54140a8760 comp rx=0 tx=0).stop 2024-05-20T20:08:59.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 shutdown_connections 2024-05-20T20:08:59.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93467a0 0x5581b9348c10 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 0x5581b93369a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931e5f0 0x5581b933c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1076800606 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931efb0 0x5581b933cac0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 >> 172.21.15.28:0/1076800606 conn(0x5581b9249be0 msgr2=0x5581b9315fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:59.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.420+0000 7f540e000700 0 client.26134 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:59.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.421+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 shutdown_connections 2024-05-20T20:08:59.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.421+0000 7f5435c76b80 1 -- 172.21.15.28:0/1076800606 wait complete. 2024-05-20T20:08:59.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.421+0000 7f5435c76b80 20 client.26134 trim_cache size 0 max 16384 2024-05-20T20:08:59.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 Processor -- start 2024-05-20T20:08:59.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 -- start start 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9337140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931efb0 0x5581b9316740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 0x5581b934e500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f54241ab930 con 0x5581b9320950 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f54241ad470 con 0x5581b931efb0 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.461+0000 7f5435c76b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f54241ac000 con 0x5581b931e5f0 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9337140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5429200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 0x5581b934e500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9337140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41438/0 (socket says 172.21.15.28:41438) 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423e00700 1 -- 172.21.15.28:0/1175102568 learned_addr learned my addr 172.21.15.28:0/1175102568 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423400700 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931efb0 0x5581b9316740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423400700 1 -- 172.21.15.28:0/1175102568 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 msgr2=0x5581b9337140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423400700 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9337140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423400700 1 -- 172.21.15.28:0/1175102568 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 msgr2=0x5581b934e500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423400700 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 0x5581b934e500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.462+0000 7f5423400700 1 -- 172.21.15.28:0/1175102568 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5581b93174d0 con 0x5581b931efb0 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.463+0000 7f5423e00700 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9337140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.463+0000 7f5423400700 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931efb0 0x5581b9316740 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f54140a8c40 tx=0x7f54140a97a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=49c191c85f8f0fb6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.463+0000 7f5418c00700 1 -- 172.21.15.28:0/1175102568 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f54140c3040 con 0x5581b931efb0 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.463+0000 7f5418c00700 1 -- 172.21.15.28:0/1175102568 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f54140ad2f0 con 0x5581b931efb0 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.463+0000 7f5418c00700 1 -- 172.21.15.28:0/1175102568 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f54140a7c10 con 0x5581b931efb0 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.464+0000 7f5435c76b80 1 -- 172.21.15.28:0/1175102568 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931efb0 msgr2=0x5581b9316740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.464+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931efb0 0x5581b9316740 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f54140a8c40 tx=0x7f54140a97a0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.465+0000 7f5435c76b80 1 -- 172.21.15.28:0/1175102568 shutdown_connections 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.465+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b9320950 0x5581b934e500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.465+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9337140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.465+0000 7f5435c76b80 1 --2- 172.21.15.28:0/1175102568 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b931efb0 0x5581b9316740 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.465+0000 7f5435c76b80 1 -- 172.21.15.28:0/1175102568 >> 172.21.15.28:0/1175102568 conn(0x5581b9249be0 msgr2=0x5581b933c580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:59.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.468+0000 7f5435c76b80 1 -- 172.21.15.28:0/1175102568 shutdown_connections 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.468+0000 7f5435c76b80 1 -- 172.21.15.28:0/1175102568 wait complete. 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.469+0000 7f5435c76b80 1 -- start start 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.469+0000 7f5435c76b80 1 Processor -- start 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9342e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b931c990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5435c76b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b931ced0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5435c76b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f54241acda0 con 0x5581b931efb0 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5435c76b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f54241ac6d0 con 0x5581b9320950 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5435c76b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f542407bb10 con 0x5581b931e5f0 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5435c76b80 10 client.0 _ll_register_callbacks cb 0x5581b9213c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.470+0000 7f5423400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b931c990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.471+0000 7f5423400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b931c990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52258/0 (socket says 172.21.15.28:52258) 2024-05-20T20:08:59.471 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.471+0000 7f5423400700 1 -- 172.21.15.28:0/2135973789 learned_addr learned my addr 172.21.15.28:0/2135973789 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:08:59.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.471+0000 7f5429200700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b931ced0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.472+0000 7f5423e00700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9342e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.473+0000 7f5423e00700 1 -- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 msgr2=0x5581b931ced0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.473+0000 7f5423e00700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b931ced0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.473+0000 7f5423e00700 1 -- 172.21.15.28:0/2135973789 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 msgr2=0x5581b931c990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:08:59.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.473+0000 7f5423e00700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b931c990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.473+0000 7f5423e00700 1 -- 172.21.15.28:0/2135973789 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5581b931d510 con 0x5581b931e5f0 2024-05-20T20:08:59.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.473+0000 7f5429200700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b931ced0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:08:59.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.474+0000 7f5423e00700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9342e10 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f541c007f20 tx=0x7f541c0073e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=85cf495fc07968e7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:08:59.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.474+0000 7f540e000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:08:59.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.474+0000 7f540e000700 1 -- 172.21.15.28:0/2135973789 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f541c03d060 con 0x5581b931e5f0 2024-05-20T20:08:59.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.474+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f54140ad2f0 con 0x5581b931e5f0 2024-05-20T20:08:59.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.474+0000 7f540e000700 1 -- 172.21.15.28:0/2135973789 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f541c004bb0 con 0x5581b931e5f0 2024-05-20T20:08:59.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.475+0000 7f540e000700 1 -- 172.21.15.28:0/2135973789 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f541c034040 con 0x5581b931e5f0 2024-05-20T20:08:59.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.475+0000 7f5435c76b80 10 client.26140 fetch_fsmap learned FSMap version 29 2024-05-20T20:08:59.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.475+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5581b93b1220 con 0x5581b931e5f0 2024-05-20T20:08:59.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.475+0000 7f540e000700 1 -- 172.21.15.28:0/2135973789 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f54140ad2f0 con 0x5581b931e5f0 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.475+0000 7f540e000700 1 -- 172.21.15.28:0/2135973789 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f541c018040 con 0x5581b931e5f0 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.476+0000 7f5435c76b80 10 client.26140 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.476+0000 7f5435c76b80 10 client.26140 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.476+0000 7f5435c76b80 20 client.26140 populate_metadata read hostname 'smithi028' 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.476+0000 7f5435c76b80 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93b17a0 0x5581b93b3bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.476+0000 7f5435c76b80 4 client.26140 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.476+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5581b9275570 con 0x5581b93b17a0 2024-05-20T20:08:59.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.476+0000 7f5423400700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93b17a0 0x5581b93b3bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:08:59.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.477+0000 7f5423400700 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93b17a0 0x5581b93b3bc0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=d645a2359225b03e server_cookie=584859480a9b07f7 in_seq=0 out_seq=0 2024-05-20T20:08:59.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.477+0000 7f540e000700 10 client.26140 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:59.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.477+0000 7f540e000700 1 -- 172.21.15.28:0/2135973789 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5581b9275570 con 0x5581b93b17a0 2024-05-20T20:08:59.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.477+0000 7f540e000700 10 client.26140 handle_command_reply: tid=0 2024-05-20T20:08:59.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.477+0000 7f5435c76b80 1 client.26140 shutdown 2024-05-20T20:08:59.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 msgr2=0x5581b9342e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9342e10 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f541c007f20 tx=0x7f541c0073e0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 shutdown_connections 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5581b93b17a0 0x5581b93b3bc0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5581b9320950 0x5581b931ced0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5581b931e5f0 0x5581b9342e10 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 --2- 172.21.15.28:0/2135973789 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5581b931efb0 0x5581b931c990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 >> 172.21.15.28:0/2135973789 conn(0x5581b9249be0 msgr2=0x5581b933c580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:59.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f540e000700 0 client.26140 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:08:59.481 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:08:59.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.478+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 shutdown_connections 2024-05-20T20:08:59.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.479+0000 7f5435c76b80 1 -- 172.21.15.28:0/2135973789 wait complete. 2024-05-20T20:08:59.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.479+0000 7f5435c76b80 20 client.26140 trim_cache size 0 max 16384 2024-05-20T20:08:59.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.480+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f54080b2b40 msgr2=0x7f54080b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.480+0000 7f5429c00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f54080b2b40 0x7f54080b5000 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f541c021000 tx=0x7f541c01e000 comp rx=0 tx=0).stop 2024-05-20T20:08:59.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424106010 msgr2=0x7f54241a1d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:08:59.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424106010 0x7f54241a1d20 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f541400b550 tx=0x7f541400b860 comp rx=0 tx=0).stop 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 shutdown_connections 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f542410b650 0x7f542419b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5424105650 0x7f54241a17e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f54080b2b40 0x7f54080b5000 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 --2- 172.21.15.28:0/3162539649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5424106010 0x7f54241a1d20 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 >> 172.21.15.28:0/3162539649 conn(0x7f5424101120 msgr2=0x7f5424101500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.481+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 shutdown_connections 2024-05-20T20:08:59.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:08:59.482+0000 7f5429c00700 1 -- 172.21.15.28:0/3162539649 wait complete. 2024-05-20T20:09:00.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:08:59 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:09:00.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:08:59 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:09:00.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:08:59 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:09:00.675 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:09:01.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:00 smithi028 ceph-mon[31178]: pgmap v104: 97 pgs: 97 active+clean; 6.6 GiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 413 MiB/s wr, 122 op/s 2024-05-20T20:09:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:00 smithi154 ceph-mon[36314]: pgmap v104: 97 pgs: 97 active+clean; 6.6 GiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 413 MiB/s wr, 122 op/s 2024-05-20T20:09:01.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:00 smithi033 ceph-mon[36179]: pgmap v104: 97 pgs: 97 active+clean; 6.6 GiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 413 MiB/s wr, 122 op/s 2024-05-20T20:09:01.558 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:02.058 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:01 smithi028 ceph-mon[31178]: pgmap v105: 97 pgs: 97 active+clean; 6.6 GiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 279 MiB/s wr, 83 op/s 2024-05-20T20:09:02.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:01 smithi154 ceph-mon[36314]: pgmap v105: 97 pgs: 97 active+clean; 6.6 GiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 279 MiB/s wr, 83 op/s 2024-05-20T20:09:02.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:01 smithi033 ceph-mon[36179]: pgmap v105: 97 pgs: 97 active+clean; 6.6 GiB data, 1.6 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 279 MiB/s wr, 83 op/s 2024-05-20T20:09:03.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:02 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:09:03.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:02 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:09:03.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:02 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:09:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:02 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:09:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:02 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:09:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:02 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:09:03.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:02 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:09:03.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:02 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:09:03.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:02 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:09:03.756 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.748+0000 7f8be1800640 1 -- 172.21.15.28:0/587290634 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40aa2a0 msgr2=0x7f8bd40b1e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 --2- 172.21.15.28:0/587290634 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40aa2a0 0x7f8bd40b1e70 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc050ab0 tx=0x7f8bdc08eba0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 -- 172.21.15.28:0/587290634 shutdown_connections 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 --2- 172.21.15.28:0/587290634 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40aa2a0 0x7f8bd40b1e70 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 --2- 172.21.15.28:0/587290634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bd40a5b80 0x7f8bd40a9bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 --2- 172.21.15.28:0/587290634 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bd40a51d0 0x7f8bd40a55b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 -- 172.21.15.28:0/587290634 >> 172.21.15.28:0/587290634 conn(0x7f8bd401a670 msgr2=0x7f8bd401aa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 -- 172.21.15.28:0/587290634 shutdown_connections 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.749+0000 7f8be1800640 1 -- 172.21.15.28:0/587290634 wait complete. 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.751+0000 7f8be1800640 1 Processor -- start 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.752+0000 7f8be1800640 1 -- start start 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.753+0000 7f8be1800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40a51d0 0x7f8bd41442c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.757 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.753+0000 7f8be1800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bd40aa2a0 0x7f8bd4144800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.753+0000 7f8bdaa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40a51d0 0x7f8bd41442c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.753+0000 7f8bdaa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40a51d0 0x7f8bd41442c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41468/0 (socket says 172.21.15.28:41468) 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.754+0000 7f8bda000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bd40aa2a0 0x7f8bd4144800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.754+0000 7f8bda000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bd40aa2a0 0x7f8bd4144800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34378/0 (socket says 172.21.15.28:34378) 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.754+0000 7f8be1800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bd413f760 0x7f8bd413fbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.754+0000 7f8bdb400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bd413f760 0x7f8bd413fbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.754+0000 7f8bdaa00640 1 -- 172.21.15.28:0/2463637971 learned_addr learned my addr 172.21.15.28:0/2463637971 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.754+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd40b4610 con 0x7f8bd413f760 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.755+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd40b4490 con 0x7f8bd40aa2a0 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.755+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd40b4790 con 0x7f8bd40a51d0 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.756+0000 7f8bdaa00640 1 -- 172.21.15.28:0/2463637971 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bd40aa2a0 msgr2=0x7f8bd4144800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.756+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bd40aa2a0 0x7f8bd4144800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.756+0000 7f8bdaa00640 1 -- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bd413f760 msgr2=0x7f8bd413fbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.756+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bd413f760 0x7f8bd413fbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.756+0000 7f8bdaa00640 1 -- 172.21.15.28:0/2463637971 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8bd4140310 con 0x7f8bd40a51d0 2024-05-20T20:09:03.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.757+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40a51d0 0x7f8bd41442c0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f8bcc00e9c0 tx=0x7f8bcc00ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:03.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.758+0000 7f8bd3400640 1 -- 172.21.15.28:0/2463637971 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bcc00ccc0 con 0x7f8bd40a51d0 2024-05-20T20:09:03.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.758+0000 7f8bd3400640 1 -- 172.21.15.28:0/2463637971 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8bcc0054d0 con 0x7f8bd40a51d0 2024-05-20T20:09:03.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.758+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8bd414c230 con 0x7f8bd40a51d0 2024-05-20T20:09:03.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.758+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8bd414c630 con 0x7f8bd40a51d0 2024-05-20T20:09:03.763 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.759+0000 7f8bd3400640 1 -- 172.21.15.28:0/2463637971 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bcc010430 con 0x7f8bd40a51d0 2024-05-20T20:09:03.770 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.770+0000 7f8be4767740 1 Processor -- start 2024-05-20T20:09:03.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.771+0000 7f8be4767740 1 -- start start 2024-05-20T20:09:03.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.771+0000 7f8be4767740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a77f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.771+0000 7f8be4767740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a77f710 0x55707a792360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.771+0000 7f8be4767740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a794c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.771+0000 7f8be4767740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55707a6efb90 con 0x55707a7928a0 2024-05-20T20:09:03.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.771+0000 7f8be4767740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55707a6df730 con 0x55707a77edf0 2024-05-20T20:09:03.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.771+0000 7f8be4767740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55707a6e3770 con 0x55707a77f710 2024-05-20T20:09:03.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.773+0000 7f8bdb400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a77f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.777 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.775+0000 7f8bd3400640 1 -- 172.21.15.28:0/2463637971 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f8bcc004f80 con 0x7f8bd40a51d0 2024-05-20T20:09:03.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.778+0000 7f8bda000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a794c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.778+0000 7f8bda000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a794c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52288/0 (socket says 172.21.15.28:52288) 2024-05-20T20:09:03.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.778+0000 7f8bda000640 1 -- 172.21.15.28:0/393767158 learned_addr learned my addr 172.21.15.28:0/393767158 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:03.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.778+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a77f710 0x55707a792360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.778 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.779+0000 7f8bdb400640 1 -- 172.21.15.28:0/393767158 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a77f710 msgr2=0x55707a792360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:03.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.779+0000 7f8bd3400640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8bc00b29e0 0x7f8bc00b4ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.780 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.779+0000 7f8bd3400640 1 -- 172.21.15.28:0/2463637971 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8bcc0e29c0 con 0x7f8bd40a51d0 2024-05-20T20:09:03.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.780+0000 7f8bdb400640 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a77f710 0x55707a792360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.780+0000 7f8bdb400640 1 -- 172.21.15.28:0/393767158 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 msgr2=0x55707a794c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:03.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.780+0000 7f8bdb400640 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a794c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.780+0000 7f8bdb400640 1 -- 172.21.15.28:0/393767158 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55707a6788b0 con 0x55707a77edf0 2024-05-20T20:09:03.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.780+0000 7f8bda000640 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a794c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:09:03.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.781+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a77f710 0x55707a792360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:03.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.782+0000 7f8bdb400640 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a77f1d0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc068330 tx=0x7f8bdc051710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=81db1b0aa82e8147 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:03.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.783+0000 7f8bc7e00640 1 -- 172.21.15.28:0/393767158 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bdc0905f0 con 0x55707a77edf0 2024-05-20T20:09:03.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.783+0000 7f8bdb400640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8bc00b29e0 0x7f8bc00b4ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.784+0000 7f8bc7e00640 1 -- 172.21.15.28:0/393767158 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8bdc069420 con 0x55707a77edf0 2024-05-20T20:09:03.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.784+0000 7f8bc7e00640 1 -- 172.21.15.28:0/393767158 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bdc098400 con 0x55707a77edf0 2024-05-20T20:09:03.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.786+0000 7f8bdb400640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8bc00b29e0 0x7f8bc00b4ea0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc069fd0 tx=0x7f8bdc0987d0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:03.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.786+0000 7f8be4767740 1 -- 172.21.15.28:0/393767158 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 msgr2=0x55707a77f1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.786+0000 7f8be4767740 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a77f1d0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc068330 tx=0x7f8bdc051710 comp rx=0 tx=0).stop 2024-05-20T20:09:03.790 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.787+0000 7f8be4767740 1 -- 172.21.15.28:0/393767158 shutdown_connections 2024-05-20T20:09:03.790 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.787+0000 7f8be4767740 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a794c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.790 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.787+0000 7f8be4767740 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a77f710 0x55707a792360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.790 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.787+0000 7f8be4767740 1 --2- 172.21.15.28:0/393767158 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a77f1d0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.790 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.787+0000 7f8be4767740 1 -- 172.21.15.28:0/393767158 >> 172.21.15.28:0/393767158 conn(0x55707a77b3c0 msgr2=0x55707a77b7a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:03.790 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.788+0000 7f8be4767740 1 -- 172.21.15.28:0/393767158 shutdown_connections 2024-05-20T20:09:03.791 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.788+0000 7f8be4767740 1 -- 172.21.15.28:0/393767158 wait complete. 2024-05-20T20:09:03.791 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.789+0000 7f8be4767740 1 -- start start 2024-05-20T20:09:03.791 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.790+0000 7f8be4767740 1 Processor -- start 2024-05-20T20:09:03.791 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.791+0000 7f8be4767740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77edf0 0x55707a78bf20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.791+0000 7f8be4767740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77f710 0x55707a78c460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.791+0000 7f8be4767740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a791c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.791+0000 7f8be4767740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55707a3c5790 con 0x55707a77edf0 2024-05-20T20:09:03.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.791+0000 7f8be4767740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55707a687bc0 con 0x55707a77f710 2024-05-20T20:09:03.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.791+0000 7f8be4767740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55707a781d30 con 0x55707a7928a0 2024-05-20T20:09:03.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.791+0000 7f8be4767740 10 client.0 _ll_register_callbacks cb 0x55707a665400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:03.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.792+0000 7f8bdaa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77edf0 0x55707a78bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.794 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.792+0000 7f8bdaa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77edf0 0x55707a78bf20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52290/0 (socket says 172.21.15.28:52290) 2024-05-20T20:09:03.794 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.792+0000 7f8bdaa00640 1 -- 172.21.15.28:0/4049653109 learned_addr learned my addr 172.21.15.28:0/4049653109 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:03.794 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.792+0000 7f8bda000640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77f710 0x55707a78c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.794 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.793+0000 7f8bdb400640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a791c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.794 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.794+0000 7f8bdb400640 1 -- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77f710 msgr2=0x55707a78c460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:03.795 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.794+0000 7f8bdb400640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77f710 0x55707a78c460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.795 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.794+0000 7f8bdb400640 1 -- 172.21.15.28:0/4049653109 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77edf0 msgr2=0x55707a78bf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:03.795 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.794+0000 7f8bdb400640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77edf0 0x55707a78bf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.795 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.794+0000 7f8bdb400640 1 -- 172.21.15.28:0/4049653109 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55707a3f4210 con 0x55707a7928a0 2024-05-20T20:09:03.795 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.795+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77edf0 0x55707a78bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:03.795 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.795+0000 7f8bdb400640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a791c40 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc068330 tx=0x7f8bdc098660 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cbab736f1ba48c7d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:03.796 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.795+0000 7f8bc4c00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:09:03.797 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.796+0000 7f8bc4c00640 1 -- 172.21.15.28:0/4049653109 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bdc098400 con 0x55707a7928a0 2024-05-20T20:09:03.797 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.796+0000 7f8bc4c00640 1 -- 172.21.15.28:0/4049653109 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8bdc090a30 con 0x55707a7928a0 2024-05-20T20:09:03.797 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.798+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8bdc069420 con 0x55707a7928a0 2024-05-20T20:09:03.798 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.798+0000 7f8bc4c00640 1 -- 172.21.15.28:0/4049653109 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8bdc0695d0 con 0x55707a7928a0 2024-05-20T20:09:03.798 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.798+0000 7f8be4767740 10 client.26149 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:03.798 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.798+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55707a4c0620 con 0x55707a7928a0 2024-05-20T20:09:03.799 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.799+0000 7f8bc4c00640 1 -- 172.21.15.28:0/4049653109 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8bdc069420 con 0x55707a7928a0 2024-05-20T20:09:03.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.802+0000 7f8bc4c00640 1 -- 172.21.15.28:0/4049653109 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8bdc0a2070 con 0x55707a7928a0 2024-05-20T20:09:03.802 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.802+0000 7f8be4767740 10 client.26149 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:03.802 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.802+0000 7f8be4767740 10 client.26149 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:03.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.803+0000 7f8be4767740 20 client.26149 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:03.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.803+0000 7f8be4767740 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a7aa610 0x55707a7aca00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.803+0000 7f8be4767740 4 client.26149 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:03.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.803+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55707a6f1ae0 con 0x55707a7aa610 2024-05-20T20:09:03.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.804+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a7aa610 0x55707a7aca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.805+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a7aa610 0x55707a7aca00 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=48062a77f984ff2a server_cookie=b185ee7259be47e2 in_seq=0 out_seq=0 2024-05-20T20:09:03.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.805+0000 7f8bc4c00640 10 client.26149 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:03.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.807+0000 7f8bc4c00640 1 -- 172.21.15.28:0/4049653109 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55707a6f1ae0 con 0x55707a7aa610 2024-05-20T20:09:03.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.807+0000 7f8bc4c00640 10 client.26149 handle_command_reply: tid=0 2024-05-20T20:09:03.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.807+0000 7f8be4767740 1 client.26149 shutdown 2024-05-20T20:09:03.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.808+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 msgr2=0x55707a791c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.808+0000 7f8be4767740 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a791c40 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc068330 tx=0x7f8bdc098660 comp rx=0 tx=0).stop 2024-05-20T20:09:03.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.809+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 shutdown_connections 2024-05-20T20:09:03.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.809+0000 7f8be4767740 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a7aa610 0x55707a7aca00 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.809+0000 7f8be4767740 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77f710 0x55707a78c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.809+0000 7f8be4767740 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a791c40 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.809+0000 7f8be4767740 1 --2- 172.21.15.28:0/4049653109 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77edf0 0x55707a78bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.809+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 >> 172.21.15.28:0/4049653109 conn(0x55707a77b3c0 msgr2=0x55707a781140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:03.810 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.809+0000 7f8bc4c00640 0 client.26149 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:03.810 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.810+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 shutdown_connections 2024-05-20T20:09:03.810 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.810+0000 7f8be4767740 1 -- 172.21.15.28:0/4049653109 wait complete. 2024-05-20T20:09:03.811 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.811+0000 7f8be4767740 20 client.26149 trim_cache size 0 max 16384 2024-05-20T20:09:03.877 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.878+0000 7f8be4767740 1 Processor -- start 2024-05-20T20:09:03.883 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.883+0000 7f8be4767740 1 -- start start 2024-05-20T20:09:03.884 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.884+0000 7f8be4767740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a791b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.884 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.884+0000 7f8be4767740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77f710 0x55707a7862b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.884 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.884+0000 7f8be4767740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a7b2000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.885 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.884+0000 7f8be4767740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd4150830 con 0x55707a77f710 2024-05-20T20:09:03.885 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.884+0000 7f8be4767740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd4152370 con 0x55707a77edf0 2024-05-20T20:09:03.885 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.884+0000 7f8be4767740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd4150f00 con 0x55707a7928a0 2024-05-20T20:09:03.885 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a7b2000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.885 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a7b2000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41484/0 (socket says 172.21.15.28:41484) 2024-05-20T20:09:03.885 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 -- 172.21.15.28:0/1868274663 learned_addr learned my addr 172.21.15.28:0/1868274663 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:03.886 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 -- 172.21.15.28:0/1868274663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 msgr2=0x55707a791b50 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:03.886 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 --2- 172.21.15.28:0/1868274663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a791b50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.886 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 -- 172.21.15.28:0/1868274663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77f710 msgr2=0x55707a7862b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:03.886 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 --2- 172.21.15.28:0/1868274663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77f710 0x55707a7862b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.886 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.885+0000 7f8bdb400640 1 -- 172.21.15.28:0/1868274663 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55707a77b1f0 con 0x55707a7928a0 2024-05-20T20:09:03.886 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.886+0000 7f8bdb400640 1 --2- 172.21.15.28:0/1868274663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a7b2000 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc064010 tx=0x7f8bdc0688b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6f9364062ccbb66d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:03.887 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.886+0000 7f8bc7400640 1 -- 172.21.15.28:0/1868274663 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bdc0cb070 con 0x55707a7928a0 2024-05-20T20:09:03.887 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.886+0000 7f8bc7400640 1 -- 172.21.15.28:0/1868274663 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8bdc0be070 con 0x55707a7928a0 2024-05-20T20:09:03.887 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.886+0000 7f8bc7400640 1 -- 172.21.15.28:0/1868274663 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bdc069c20 con 0x55707a7928a0 2024-05-20T20:09:03.887 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.887+0000 7f8be4767740 1 -- 172.21.15.28:0/1868274663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 msgr2=0x55707a7b2000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.887 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.887+0000 7f8be4767740 1 --2- 172.21.15.28:0/1868274663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a7b2000 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc064010 tx=0x7f8bdc0688b0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.887 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.888+0000 7f8be4767740 1 -- 172.21.15.28:0/1868274663 shutdown_connections 2024-05-20T20:09:03.888 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.888+0000 7f8be4767740 1 --2- 172.21.15.28:0/1868274663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7928a0 0x55707a7b2000 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.888 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.888+0000 7f8be4767740 1 --2- 172.21.15.28:0/1868274663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a77f710 0x55707a7862b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.888 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.888+0000 7f8be4767740 1 --2- 172.21.15.28:0/1868274663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a77edf0 0x55707a791b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.888 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.888+0000 7f8be4767740 1 -- 172.21.15.28:0/1868274663 >> 172.21.15.28:0/1868274663 conn(0x55707a77b3c0 msgr2=0x55707a782850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:03.889 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.890+0000 7f8be4767740 1 -- 172.21.15.28:0/1868274663 shutdown_connections 2024-05-20T20:09:03.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.890+0000 7f8be4767740 1 -- 172.21.15.28:0/1868274663 wait complete. 2024-05-20T20:09:03.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.891+0000 7f8be4767740 1 -- start start 2024-05-20T20:09:03.891 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.891+0000 7f8be4767740 1 Processor -- start 2024-05-20T20:09:03.893 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.893+0000 7f8be4767740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a7b8bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.893+0000 7f8be4767740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7b9130 0x55707a7b95b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.893+0000 7f8be4767740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a810cb0 0x55707a8130a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.895+0000 7f8be4767740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd4151ca0 con 0x55707a7928a0 2024-05-20T20:09:03.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.895+0000 7f8bda000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7b9130 0x55707a7b95b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.895+0000 7f8bda000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7b9130 0x55707a7b95b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41486/0 (socket says 172.21.15.28:41486) 2024-05-20T20:09:03.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.895+0000 7f8bda000640 1 -- 172.21.15.28:0/2949082556 learned_addr learned my addr 172.21.15.28:0/2949082556 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:03.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.895+0000 7f8bdb400640 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a810cb0 0x55707a8130a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.895+0000 7f8be4767740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd41515d0 con 0x55707a810cb0 2024-05-20T20:09:03.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.895+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8bd414e810 con 0x55707a7b9130 2024-05-20T20:09:03.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.898+0000 7f8bdb400640 1 -- 172.21.15.28:0/2949082556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7b9130 msgr2=0x55707a7b95b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:03.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.898+0000 7f8bdb400640 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7b9130 0x55707a7b95b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.898+0000 7f8bdb400640 1 -- 172.21.15.28:0/2949082556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 msgr2=0x55707a7b8bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:03.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.898+0000 7f8bdb400640 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a7b8bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.898+0000 7f8bdb400640 1 -- 172.21.15.28:0/2949082556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55707a813760 con 0x55707a810cb0 2024-05-20T20:09:03.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.899+0000 7f8be4767740 10 client.0 _ll_register_callbacks cb 0x7f8bdc0b3070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:03.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.899+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8bdc0be070 con 0x55707a810cb0 2024-05-20T20:09:03.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.899+0000 7f8bdb400640 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a810cb0 0x55707a8130a0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc068760 tx=0x7f8bdc068ba0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ccf69b1d70ac51c2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:03.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.901+0000 7f8bc4c00640 10 client.26210 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:03.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.901+0000 7f8bc4c00640 1 -- 172.21.15.28:0/2949082556 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8bdc0cb070 con 0x55707a810cb0 2024-05-20T20:09:03.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.901+0000 7f8bc4c00640 1 -- 172.21.15.28:0/2949082556 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8bdc069770 con 0x55707a810cb0 2024-05-20T20:09:03.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.901+0000 7f8bc4c00640 1 -- 172.21.15.28:0/2949082556 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8bdc069970 con 0x55707a810cb0 2024-05-20T20:09:03.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.901+0000 7f8be4767740 10 client.26210 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:03.902 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.901+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x55707a814860 con 0x55707a810cb0 2024-05-20T20:09:03.903 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.903+0000 7f8bc4c00640 1 -- 172.21.15.28:0/2949082556 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8bdc0a2070 con 0x55707a810cb0 2024-05-20T20:09:03.904 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.904+0000 7f8be4767740 10 client.26210 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:03.904 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.904+0000 7f8be4767740 10 client.26210 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:03.904 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.905+0000 7f8be4767740 20 client.26210 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:03.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.905+0000 7f8be4767740 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a814d80 0x55707a8171a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:03.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.905+0000 7f8be4767740 4 client.26210 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:09:03.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.905+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55707a6eb2c0 con 0x55707a814d80 2024-05-20T20:09:03.906 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.906+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a814d80 0x55707a8171a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:03.907 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.907+0000 7f8bc4c00640 10 client.26210 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:03.907 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.907+0000 7f8bdaa00640 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a814d80 0x55707a8171a0 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8565f354bafee84d server_cookie=6397707d48582988 in_seq=0 out_seq=0 2024-05-20T20:09:03.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.942+0000 7f8bc4c00640 1 -- 172.21.15.28:0/2949082556 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55707a6eb2c0 con 0x55707a814d80 2024-05-20T20:09:03.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.942+0000 7f8bc4c00640 10 client.26210 handle_command_reply: tid=0 2024-05-20T20:09:03.943 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.942+0000 7f8be4767740 1 client.26210 shutdown 2024-05-20T20:09:03.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a810cb0 msgr2=0x55707a8130a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a810cb0 0x55707a8130a0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc068760 tx=0x7f8bdc068ba0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 shutdown_connections 2024-05-20T20:09:03.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55707a814d80 0x55707a8171a0 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55707a810cb0 0x55707a8130a0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8bc4c00640 0 client.26210 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:03.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55707a7b9130 0x55707a7b95b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 --2- 172.21.15.28:0/2949082556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55707a7928a0 0x55707a7b8bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.943+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 >> 172.21.15.28:0/2949082556 conn(0x55707a77b3c0 msgr2=0x55707a77a520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:03.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.944+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 shutdown_connections 2024-05-20T20:09:03.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.944+0000 7f8be4767740 1 -- 172.21.15.28:0/2949082556 wait complete. 2024-05-20T20:09:03.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.944+0000 7f8be4767740 20 client.26210 trim_cache size 0 max 16384 2024-05-20T20:09:03.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:09:03.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:09:03.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:09:03.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.947+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8bc00b29e0 msgr2=0x7f8bc00b4ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.947+0000 7f8be1800640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8bc00b29e0 0x7f8bc00b4ea0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f8bdc069fd0 tx=0x7f8bdc0987d0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.947+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40a51d0 msgr2=0x7f8bd41442c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.947+0000 7f8be1800640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40a51d0 0x7f8bd41442c0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f8bcc00e9c0 tx=0x7f8bcc00ee80 comp rx=0 tx=0).stop 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.948+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 shutdown_connections 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.948+0000 7f8be1800640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8bd40aa2a0 0x7f8bd4144800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.948+0000 7f8be1800640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8bc00b29e0 0x7f8bc00b4ea0 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.948+0000 7f8be1800640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8bd413f760 0x7f8bd413fbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.948+0000 7f8be1800640 1 --2- 172.21.15.28:0/2463637971 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8bd40a51d0 0x7f8bd41442c0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:03.948 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.948+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 >> 172.21.15.28:0/2463637971 conn(0x7f8bd401a670 msgr2=0x7f8bd40a8ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:03.949 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.948+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 shutdown_connections 2024-05-20T20:09:03.949 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:03.949+0000 7f8be1800640 1 -- 172.21.15.28:0/2463637971 wait complete. 2024-05-20T20:09:04.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:03 smithi028 ceph-mon[31178]: pgmap v106: 97 pgs: 97 active+clean; 7.8 GiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 379 MiB/s wr, 242 op/s 2024-05-20T20:09:04.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:03 smithi154 ceph-mon[36314]: pgmap v106: 97 pgs: 97 active+clean; 7.8 GiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 379 MiB/s wr, 242 op/s 2024-05-20T20:09:04.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:03 smithi033 ceph-mon[36179]: pgmap v106: 97 pgs: 97 active+clean; 7.8 GiB data, 1.8 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 379 MiB/s wr, 242 op/s 2024-05-20T20:09:05.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 -- 172.21.15.28:0/576693565 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f434810b750 msgr2=0x7f434810bb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 --2- 172.21.15.28:0/576693565 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f434810b750 0x7f434810bb30 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f433c009990 tx=0x7f433c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 -- 172.21.15.28:0/576693565 shutdown_connections 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 --2- 172.21.15.28:0/576693565 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 0x7f4348112620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 --2- 172.21.15.28:0/576693565 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f434810b750 0x7f434810bb30 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 --2- 172.21.15.28:0/576693565 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4348105750 0x7f4348105bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 -- 172.21.15.28:0/576693565 >> 172.21.15.28:0/576693565 conn(0x7f4348101220 msgr2=0x7f4348103640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 -- 172.21.15.28:0/576693565 shutdown_connections 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.074+0000 7f4350800700 1 -- 172.21.15.28:0/576693565 wait complete. 2024-05-20T20:09:05.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.075+0000 7f4350800700 1 Processor -- start 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.076+0000 7f4350800700 1 -- start start 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.076+0000 7f4350800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4348105750 0x7f4348074cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.076+0000 7f4350800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 0x7f4348075210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f4350800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f434810b750 0x7f434806df90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f4350800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f43481159e0 con 0x7f4348105750 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f4350800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4348115860 con 0x7f434810b750 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f4350800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4348115b60 con 0x7f4348106110 2024-05-20T20:09:05.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f434d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 0x7f4348075210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f434d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 0x7f4348075210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41502/0 (socket says 172.21.15.28:41502) 2024-05-20T20:09:05.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f434d400700 1 -- 172.21.15.28:0/1407777554 learned_addr learned my addr 172.21.15.28:0/1407777554 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:05.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f434e800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f434810b750 0x7f434806df90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f434d400700 1 -- 172.21.15.28:0/1407777554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f434810b750 msgr2=0x7f434806df90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f434d400700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f434810b750 0x7f434806df90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.077+0000 7f434d400700 1 -- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4348105750 msgr2=0x7f4348074cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.078+0000 7f434de00700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4348105750 0x7f4348074cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.078+0000 7f434d400700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4348105750 0x7f4348074cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.078+0000 7f434d400700 1 -- 172.21.15.28:0/1407777554 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f434806e890 con 0x7f4348106110 2024-05-20T20:09:05.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.078+0000 7f434de00700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4348105750 0x7f4348074cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:05.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.079+0000 7f434d400700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 0x7f4348075210 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f433800d950 tx=0x7f433800dd10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:05.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.080+0000 7f4342a00700 1 -- 172.21.15.28:0/1407777554 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4338016070 con 0x7f4348106110 2024-05-20T20:09:05.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.080+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f434806eb80 con 0x7f4348106110 2024-05-20T20:09:05.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.080+0000 7f4342a00700 1 -- 172.21.15.28:0/1407777554 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4338010460 con 0x7f4348106110 2024-05-20T20:09:05.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.080+0000 7f4342a00700 1 -- 172.21.15.28:0/1407777554 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f433800f620 con 0x7f4348106110 2024-05-20T20:09:05.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.081+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4348072b60 con 0x7f4348106110 2024-05-20T20:09:05.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.096+0000 7f435b28cb80 1 Processor -- start 2024-05-20T20:09:05.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.097+0000 7f435b28cb80 1 -- start start 2024-05-20T20:09:05.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.097+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7f79f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.097+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7f30 0x55e86d7f1b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.097+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f2140 0x55e86d7f25c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.097+0000 7f435b28cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e86d74fe40 con 0x55e86d7f7610 2024-05-20T20:09:05.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.097+0000 7f435b28cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e86d73f9e0 con 0x55e86d7f2140 2024-05-20T20:09:05.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.097+0000 7f435b28cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e86d743a20 con 0x55e86d7f7f30 2024-05-20T20:09:05.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.098+0000 7f434e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7f79f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.098+0000 7f434de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7f30 0x55e86d7f1b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.099+0000 7f434de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7f30 0x55e86d7f1b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41508/0 (socket says 172.21.15.28:41508) 2024-05-20T20:09:05.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.099+0000 7f434de00700 1 -- 172.21.15.28:0/643893931 learned_addr learned my addr 172.21.15.28:0/643893931 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:05.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.099+0000 7f4342a00700 1 -- 172.21.15.28:0/1407777554 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4338019070 con 0x7f4348106110 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.101+0000 7f434d400700 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f2140 0x55e86d7f25c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.101+0000 7f434d400700 1 -- 172.21.15.28:0/643893931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7f30 msgr2=0x55e86d7f1b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.101+0000 7f434d400700 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7f30 0x55e86d7f1b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.101+0000 7f434d400700 1 -- 172.21.15.28:0/643893931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 msgr2=0x55e86d7f79f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.102+0000 7f434d400700 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7f79f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.102+0000 7f434d400700 1 -- 172.21.15.28:0/643893931 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e86d708420 con 0x55e86d7f2140 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.102+0000 7f434e800700 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7f79f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.103+0000 7f4342a00700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43300b2a70 0x7f43300b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.103+0000 7f4342a00700 1 -- 172.21.15.28:0/1407777554 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f43380e1750 con 0x7f4348106110 2024-05-20T20:09:05.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.104+0000 7f434e800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43300b2a70 0x7f43300b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.105+0000 7f434e800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43300b2a70 0x7f43300b4f30 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f434400af70 tx=0x7f434400a590 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:05.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.108+0000 7f434d400700 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f2140 0x55e86d7f25c0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f43380deeb0 tx=0x7f43380e7ef0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=feff0646b5f1fd39 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:05.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.108+0000 7f4337400700 1 -- 172.21.15.28:0/643893931 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43380a4b30 con 0x55e86d7f2140 2024-05-20T20:09:05.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.108+0000 7f4337400700 1 -- 172.21.15.28:0/643893931 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43380a43f0 con 0x55e86d7f2140 2024-05-20T20:09:05.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.108+0000 7f4337400700 1 -- 172.21.15.28:0/643893931 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43380a83e0 con 0x55e86d7f2140 2024-05-20T20:09:05.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.109+0000 7f435b28cb80 1 -- 172.21.15.28:0/643893931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f2140 msgr2=0x55e86d7f25c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.109+0000 7f435b28cb80 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f2140 0x55e86d7f25c0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f43380deeb0 tx=0x7f43380e7ef0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.109+0000 7f435b28cb80 1 -- 172.21.15.28:0/643893931 shutdown_connections 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.109+0000 7f435b28cb80 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f2140 0x55e86d7f25c0 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.109+0000 7f435b28cb80 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7f79f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.109+0000 7f435b28cb80 1 --2- 172.21.15.28:0/643893931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7f30 0x55e86d7f1b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.109+0000 7f435b28cb80 1 -- 172.21.15.28:0/643893931 >> 172.21.15.28:0/643893931 conn(0x55e86d71fbe0 msgr2=0x55e86d7ec690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.110+0000 7f435b28cb80 1 -- 172.21.15.28:0/643893931 shutdown_connections 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.110+0000 7f435b28cb80 1 -- 172.21.15.28:0/643893931 wait complete. 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.111+0000 7f435b28cb80 1 -- start start 2024-05-20T20:09:05.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.111+0000 7f435b28cb80 1 Processor -- start 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f2140 0x55e86d7fdc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7fb020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7f30 0x55e86d7fb560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f435b28cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e86d645fa0 con 0x55e86d7f7610 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f435b28cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e86d6e0760 con 0x55e86d7f7f30 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f435b28cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e86d80f9f0 con 0x55e86d7f2140 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f435b28cb80 10 client.0 _ll_register_callbacks cb 0x55e86d6e9c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.112+0000 7f434d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7fb020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7fb020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52344/0 (socket says 172.21.15.28:52344) 2024-05-20T20:09:05.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 -- 172.21.15.28:0/842315230 learned_addr learned my addr 172.21.15.28:0/842315230 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434de00700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f2140 0x55e86d7fdc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434e800700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7f30 0x55e86d7fb560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 -- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f2140 msgr2=0x55e86d7fdc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f2140 0x55e86d7fdc80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 -- 172.21.15.28:0/842315230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7f30 msgr2=0x55e86d7fb560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7f30 0x55e86d7fb560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 -- 172.21.15.28:0/842315230 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e86d68f8d0 con 0x55e86d7f7610 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434e800700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7f30 0x55e86d7fb560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f434d400700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7fb020 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f43380a8970 tx=0x7f4338010b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5e3a81ef2d6884fc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.113+0000 7f432be00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.114+0000 7f432be00700 1 -- 172.21.15.28:0/842315230 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43380a85c0 con 0x55e86d7f7610 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.114+0000 7f432be00700 1 -- 172.21.15.28:0/842315230 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f43380df130 con 0x55e86d7f7610 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.114+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f43380a43f0 con 0x55e86d7f7610 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.115+0000 7f432be00700 1 -- 172.21.15.28:0/842315230 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f43380a8a40 con 0x55e86d7f7610 2024-05-20T20:09:05.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.115+0000 7f435b28cb80 10 client.16056 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.115+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e86d710060 con 0x55e86d7f7610 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f432be00700 1 -- 172.21.15.28:0/842315230 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f43380a43f0 con 0x55e86d7f7610 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f432be00700 1 -- 172.21.15.28:0/842315230 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f43380a4f00 con 0x55e86d7f7610 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f435b28cb80 10 client.16056 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f435b28cb80 10 client.16056 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f435b28cb80 20 client.16056 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f435b28cb80 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d81c520 0x55e86d81e960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f435b28cb80 4 client.16056 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.116+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e86d751d90 con 0x55e86d81c520 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.117+0000 7f434de00700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d81c520 0x55e86d81e960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.118+0000 7f432be00700 10 client.16056 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:09:05.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.118+0000 7f434de00700 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d81c520 0x55e86d81e960 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=4a44234955c1e0e0 server_cookie=ae511ffc66d4347c in_seq=0 out_seq=0 2024-05-20T20:09:05.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.119+0000 7f432be00700 1 -- 172.21.15.28:0/842315230 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55e86d751d90 con 0x55e86d81c520 2024-05-20T20:09:05.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.119+0000 7f432be00700 10 client.16056 handle_command_reply: tid=0 2024-05-20T20:09:05.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.119+0000 7f435b28cb80 1 client.16056 shutdown 2024-05-20T20:09:05.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.119+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 msgr2=0x55e86d7fb020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.119+0000 7f435b28cb80 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7fb020 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f43380a8970 tx=0x7f4338010b40 comp rx=0 tx=0).stop 2024-05-20T20:09:05.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.120+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 shutdown_connections 2024-05-20T20:09:05.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.120+0000 7f435b28cb80 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7f30 0x55e86d7fb560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.120+0000 7f435b28cb80 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f2140 0x55e86d7fdc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.120+0000 7f435b28cb80 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d81c520 0x55e86d81e960 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.120+0000 7f435b28cb80 1 --2- 172.21.15.28:0/842315230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f7610 0x55e86d7fb020 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.120+0000 7f432be00700 0 client.16056 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:09:05.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.120+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 >> 172.21.15.28:0/842315230 conn(0x55e86d71fbe0 msgr2=0x55e86d7ebd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:05.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.122+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 shutdown_connections 2024-05-20T20:09:05.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.122+0000 7f435b28cb80 1 -- 172.21.15.28:0/842315230 wait complete. 2024-05-20T20:09:05.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.123+0000 7f435b28cb80 20 client.16056 trim_cache size 0 max 16384 2024-05-20T20:09:05.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.152+0000 7f435b28cb80 1 Processor -- start 2024-05-20T20:09:05.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.152+0000 7f435b28cb80 1 -- start start 2024-05-20T20:09:05.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.153+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d71fbe0 0x55e86d7f1b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.154+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d7f8a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.154+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7610 0x55e86d7ec4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.154+0000 7f435b28cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f434819d5a0 con 0x55e86d7f2140 2024-05-20T20:09:05.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.154+0000 7f435b28cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f434819f0e0 con 0x55e86d71fbe0 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.154+0000 7f435b28cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f434819dc70 con 0x55e86d7f7610 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.154+0000 7f434d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d7f8a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.155+0000 7f434d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d7f8a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52360/0 (socket says 172.21.15.28:52360) 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.155+0000 7f434d400700 1 -- 172.21.15.28:0/521243407 learned_addr learned my addr 172.21.15.28:0/521243407 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.156+0000 7f434d400700 1 -- 172.21.15.28:0/521243407 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7610 msgr2=0x55e86d7ec4f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.156+0000 7f434d400700 1 --2- 172.21.15.28:0/521243407 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7610 0x55e86d7ec4f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.156+0000 7f434d400700 1 -- 172.21.15.28:0/521243407 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d71fbe0 msgr2=0x55e86d7f1b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:05.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.156+0000 7f434d400700 1 --2- 172.21.15.28:0/521243407 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d71fbe0 0x55e86d7f1b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.156+0000 7f434d400700 1 -- 172.21.15.28:0/521243407 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e86d7f9590 con 0x55e86d7f2140 2024-05-20T20:09:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.157+0000 7f434d400700 1 --2- 172.21.15.28:0/521243407 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d7f8a20 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f4338010cb0 tx=0x7f43380e1b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d7f446f7ecf0d9b1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.159+0000 7f4336a00700 1 -- 172.21.15.28:0/521243407 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43380a8a40 con 0x55e86d7f2140 2024-05-20T20:09:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.159+0000 7f4336a00700 1 -- 172.21.15.28:0/521243407 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4338042020 con 0x55e86d7f2140 2024-05-20T20:09:05.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.159+0000 7f4336a00700 1 -- 172.21.15.28:0/521243407 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43380a43f0 con 0x55e86d7f2140 2024-05-20T20:09:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.160+0000 7f435b28cb80 1 -- 172.21.15.28:0/521243407 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 msgr2=0x55e86d7f8a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.160+0000 7f435b28cb80 1 --2- 172.21.15.28:0/521243407 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d7f8a20 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f4338010cb0 tx=0x7f43380e1b50 comp rx=0 tx=0).stop 2024-05-20T20:09:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.161+0000 7f435b28cb80 1 -- 172.21.15.28:0/521243407 shutdown_connections 2024-05-20T20:09:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.161+0000 7f435b28cb80 1 --2- 172.21.15.28:0/521243407 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d7f7610 0x55e86d7ec4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.161+0000 7f435b28cb80 1 --2- 172.21.15.28:0/521243407 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d71fbe0 0x55e86d7f1b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.161+0000 7f435b28cb80 1 --2- 172.21.15.28:0/521243407 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d7f8a20 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.161+0000 7f435b28cb80 1 -- 172.21.15.28:0/521243407 >> 172.21.15.28:0/521243407 conn(0x55e86d7f58e0 msgr2=0x55e86d7f8310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.162+0000 7f435b28cb80 1 -- 172.21.15.28:0/521243407 shutdown_connections 2024-05-20T20:09:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.162+0000 7f435b28cb80 1 -- 172.21.15.28:0/521243407 wait complete. 2024-05-20T20:09:05.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.163+0000 7f435b28cb80 1 -- start start 2024-05-20T20:09:05.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.163+0000 7f435b28cb80 1 Processor -- start 2024-05-20T20:09:05.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.164+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d71fbe0 0x55e86d815290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.164+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d823980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.165+0000 7f434d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d823980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.165+0000 7f434de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d71fbe0 0x55e86d815290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.165+0000 7f435b28cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7610 0x55e86d823ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.165+0000 7f435b28cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f434819ea10 con 0x55e86d7f2140 2024-05-20T20:09:05.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.165+0000 7f435b28cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f434819e340 con 0x55e86d7f7610 2024-05-20T20:09:05.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.165+0000 7f435b28cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f434807b1c0 con 0x55e86d71fbe0 2024-05-20T20:09:05.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.165+0000 7f435b28cb80 10 client.0 _ll_register_callbacks cb 0x55e86d6e9c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:05.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.166+0000 7f434d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d823980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52370/0 (socket says 172.21.15.28:52370) 2024-05-20T20:09:05.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.166+0000 7f434d400700 1 -- 172.21.15.28:0/2050073591 learned_addr learned my addr 172.21.15.28:0/2050073591 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:05.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.166+0000 7f434de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d71fbe0 0x55e86d815290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41536/0 (socket says 172.21.15.28:41536) 2024-05-20T20:09:05.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.167+0000 7f434e800700 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7610 0x55e86d823ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.167+0000 7f434d400700 1 -- 172.21.15.28:0/2050073591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d71fbe0 msgr2=0x55e86d815290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:05.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.167+0000 7f434d400700 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d71fbe0 0x55e86d815290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.167+0000 7f434d400700 1 -- 172.21.15.28:0/2050073591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7610 msgr2=0x55e86d823ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:05.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.167+0000 7f434d400700 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7610 0x55e86d823ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.167+0000 7f434d400700 1 -- 172.21.15.28:0/2050073591 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e86d824600 con 0x55e86d7f2140 2024-05-20T20:09:05.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.169+0000 7f434d400700 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d823980 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f43380e75d0 tx=0x7f4338010b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=81e7c01f247fac server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:05.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.169+0000 7f432be00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:05.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.170+0000 7f432be00700 1 -- 172.21.15.28:0/2050073591 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f43380a8a40 con 0x55e86d7f2140 2024-05-20T20:09:05.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.170+0000 7f432be00700 1 -- 172.21.15.28:0/2050073591 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f43380a5260 con 0x55e86d7f2140 2024-05-20T20:09:05.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.170+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4338042020 con 0x55e86d7f2140 2024-05-20T20:09:05.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.171+0000 7f432be00700 1 -- 172.21.15.28:0/2050073591 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f43380a83e0 con 0x55e86d7f2140 2024-05-20T20:09:05.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.172+0000 7f435b28cb80 10 client.16062 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:05.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.172+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e86d824600 con 0x55e86d7f2140 2024-05-20T20:09:05.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.173+0000 7f432be00700 1 -- 172.21.15.28:0/2050073591 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4338042020 con 0x55e86d7f2140 2024-05-20T20:09:05.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.173+0000 7f432be00700 1 -- 172.21.15.28:0/2050073591 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f43380a4f00 con 0x55e86d7f2140 2024-05-20T20:09:05.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.173+0000 7f435b28cb80 10 client.16062 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:05.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.174+0000 7f435b28cb80 10 client.16062 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:09:05.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.174+0000 7f435b28cb80 20 client.16062 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:05.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.174+0000 7f435b28cb80 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d7f7f30 0x55e86d889280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:05.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.174+0000 7f435b28cb80 4 client.16062 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:09:05.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.174+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55e86d74b570 con 0x55e86d7f7f30 2024-05-20T20:09:05.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.175+0000 7f434de00700 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d7f7f30 0x55e86d889280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:05.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.178+0000 7f434de00700 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d7f7f30 0x55e86d889280 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=234e527148d44f2d server_cookie=9856f1c986de688d in_seq=0 out_seq=0 2024-05-20T20:09:05.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.178+0000 7f432be00700 10 client.16062 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:09:05.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.178+0000 7f432be00700 1 -- 172.21.15.28:0/2050073591 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55e86d74b570 con 0x55e86d7f7f30 2024-05-20T20:09:05.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.178+0000 7f432be00700 10 client.16062 handle_command_reply: tid=0 2024-05-20T20:09:05.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.178+0000 7f435b28cb80 1 client.16062 shutdown 2024-05-20T20:09:05.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 msgr2=0x55e86d823980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d823980 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f43380e75d0 tx=0x7f4338010b60 comp rx=0 tx=0).stop 2024-05-20T20:09:05.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 shutdown_connections 2024-05-20T20:09:05.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e86d7f7610 0x55e86d823ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e86d71fbe0 0x55e86d815290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55e86d7f7f30 0x55e86d889280 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 --2- 172.21.15.28:0/2050073591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e86d7f2140 0x55e86d823980 secure :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f43380e75d0 tx=0x7f4338010b60 comp rx=0 tx=0).stop 2024-05-20T20:09:05.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 >> 172.21.15.28:0/2050073591 conn(0x55e86d7f58e0 msgr2=0x55e86d7f8ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:05.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.179+0000 7f432be00700 0 client.16062 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:09:05.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.180+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 shutdown_connections 2024-05-20T20:09:05.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.181+0000 7f435b28cb80 1 -- 172.21.15.28:0/2050073591 wait complete. 2024-05-20T20:09:05.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.181+0000 7f435b28cb80 20 client.16062 trim_cache size 0 max 16384 2024-05-20T20:09:05.182 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43300b2a70 msgr2=0x7f43300b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43300b2a70 0x7f43300b4f30 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f434400af70 tx=0x7f434400a590 comp rx=0 tx=0).stop 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 msgr2=0x7f4348075210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 0x7f4348075210 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f433800d950 tx=0x7f433800dd10 comp rx=0 tx=0).stop 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 shutdown_connections 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f43300b2a70 0x7f43300b4f30 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f434810b750 0x7f434806df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4348105750 0x7f4348074cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.182+0000 7f4350800700 1 --2- 172.21.15.28:0/1407777554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4348106110 0x7f4348075210 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:05.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.183+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 >> 172.21.15.28:0/1407777554 conn(0x7f4348101220 msgr2=0x7f43481018d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:05.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.185+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 shutdown_connections 2024-05-20T20:09:05.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:05.185+0000 7f4350800700 1 -- 172.21.15.28:0/1407777554 wait complete. 2024-05-20T20:09:06.690 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:09:07.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:06 smithi028 ceph-mon[31178]: pgmap v107: 97 pgs: 97 active+clean; 8.2 GiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 299 MiB/s wr, 252 op/s 2024-05-20T20:09:07.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:06 smithi154 ceph-mon[36314]: pgmap v107: 97 pgs: 97 active+clean; 8.2 GiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 299 MiB/s wr, 252 op/s 2024-05-20T20:09:07.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:06 smithi033 ceph-mon[36179]: pgmap v107: 97 pgs: 97 active+clean; 8.2 GiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 299 MiB/s wr, 252 op/s 2024-05-20T20:09:07.370 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:08 smithi028 ceph-mon[31178]: pgmap v108: 97 pgs: 97 active+clean; 8.2 GiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 267 MiB/s wr, 243 op/s 2024-05-20T20:09:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:08 smithi154 ceph-mon[36314]: pgmap v108: 97 pgs: 97 active+clean; 8.2 GiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 267 MiB/s wr, 243 op/s 2024-05-20T20:09:09.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:08 smithi033 ceph-mon[36179]: pgmap v108: 97 pgs: 97 active+clean; 8.2 GiB data, 1.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 267 MiB/s wr, 243 op/s 2024-05-20T20:09:10.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.552+0000 7f0597800700 1 -- 172.21.15.28:0/3964790564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f059011cd20 msgr2=0x7f059011f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.552+0000 7f0597800700 1 --2- 172.21.15.28:0/3964790564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f059011cd20 0x7f059011f110 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f0588009f90 tx=0x7f058800b330 comp rx=0 tx=0).stop 2024-05-20T20:09:10.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.552+0000 7f0597800700 1 -- 172.21.15.28:0/3964790564 shutdown_connections 2024-05-20T20:09:10.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.552+0000 7f0597800700 1 --2- 172.21.15.28:0/3964790564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f059011cd20 0x7f059011f110 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.552+0000 7f0597800700 1 --2- 172.21.15.28:0/3964790564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0590111d20 0x7f059010a540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.552+0000 7f0597800700 1 --2- 172.21.15.28:0/3964790564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f059010aa80 0x7f059010af00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.552+0000 7f0597800700 1 -- 172.21.15.28:0/3964790564 >> 172.21.15.28:0/3964790564 conn(0x7f059006e6b0 msgr2=0x7f059006eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:10.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.553+0000 7f0597800700 1 -- 172.21.15.28:0/3964790564 shutdown_connections 2024-05-20T20:09:10.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.553+0000 7f0597800700 1 -- 172.21.15.28:0/3964790564 wait complete. 2024-05-20T20:09:10.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 Processor -- start 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 -- start start 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f059010aa80 0x7f05901ae5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0590111d20 0x7f05901aeb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f059011cd20 0x7f05901a86c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0590122560 con 0x7f059010aa80 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f05901223e0 con 0x7f059011cd20 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.554+0000 7f0597800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f05901226e0 con 0x7f0590111d20 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.555+0000 7f0594e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f059010aa80 0x7f05901ae5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.555+0000 7f058fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0590111d20 0x7f05901aeb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.555+0000 7f058fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0590111d20 0x7f05901aeb30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58622/0 (socket says 172.21.15.28:58622) 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.555+0000 7f058fe00700 1 -- 172.21.15.28:0/1126626235 learned_addr learned my addr 172.21.15.28:0/1126626235 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.555+0000 7f0595800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f059011cd20 0x7f05901a86c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058fe00700 1 -- 172.21.15.28:0/1126626235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f059011cd20 msgr2=0x7f05901a86c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058fe00700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f059011cd20 0x7f05901a86c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058fe00700 1 -- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f059010aa80 msgr2=0x7f05901ae5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058fe00700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f059010aa80 0x7f05901ae5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058fe00700 1 -- 172.21.15.28:0/1126626235 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f05901a8fa0 con 0x7f0590111d20 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058fe00700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0590111d20 0x7f05901aeb30 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f057c00e9c0 tx=0x7f057c00ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058d600700 1 -- 172.21.15.28:0/1126626235 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f057c004a30 con 0x7f0590111d20 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058d600700 1 -- 172.21.15.28:0/1126626235 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f057c00cc70 con 0x7f0590111d20 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f058d600700 1 -- 172.21.15.28:0/1126626235 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f057c004a30 con 0x7f0590111d20 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.556+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f05901a9290 con 0x7f0590111d20 2024-05-20T20:09:10.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.557+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f059007b970 con 0x7f0590111d20 2024-05-20T20:09:10.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.562+0000 7f058d600700 1 -- 172.21.15.28:0/1126626235 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f057c004bd0 con 0x7f0590111d20 2024-05-20T20:09:10.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.563+0000 7f05a210eb80 1 Processor -- start 2024-05-20T20:09:10.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.564+0000 7f058d600700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f05780b2c70 0x7f05780b5130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.565+0000 7f058d600700 1 -- 172.21.15.28:0/1126626235 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f057c0a5b20 con 0x7f0590111d20 2024-05-20T20:09:10.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.566+0000 7f05a210eb80 1 -- start start 2024-05-20T20:09:10.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.566+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 0x559358489290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.566+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x559358483260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.566+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.566+0000 7f05a210eb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5593583e1e40 con 0x5593584897d0 2024-05-20T20:09:10.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.566+0000 7f05a210eb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5593583d19e0 con 0x559358488eb0 2024-05-20T20:09:10.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.566+0000 7f05a210eb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5593583d5a20 con 0x559358483880 2024-05-20T20:09:10.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.567+0000 7f058fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x559358483260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.567+0000 7f0595800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f05780b2c70 0x7f05780b5130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.567+0000 7f058fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x559358483260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:39034/0 (socket says 172.21.15.28:39034) 2024-05-20T20:09:10.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.567+0000 7f058fe00700 1 -- 172.21.15.28:0/752672630 learned_addr learned my addr 172.21.15.28:0/752672630 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.567+0000 7f0595800700 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.568+0000 7f0594e00700 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 0x559358489290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.568+0000 7f0595800700 1 -- 172.21.15.28:0/752672630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 msgr2=0x559358489290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.568+0000 7f0595800700 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 0x559358489290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.568+0000 7f0595800700 1 -- 172.21.15.28:0/752672630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 msgr2=0x559358483260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.568+0000 7f0595800700 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x559358483260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.569+0000 7f0595800700 1 -- 172.21.15.28:0/752672630 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55935839a420 con 0x559358483880 2024-05-20T20:09:10.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.570+0000 7f0595800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f05780b2c70 0x7f05780b5130 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f0588009f90 tx=0x7f0588023000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:10.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.570+0000 7f0595800700 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483ce0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f059006ccb0 tx=0x7f0588003f30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=304ffdeee61168f1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.570+0000 7f0582000700 1 -- 172.21.15.28:0/752672630 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0588009b80 con 0x559358483880 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.570+0000 7f0582000700 1 -- 172.21.15.28:0/752672630 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0588021460 con 0x559358483880 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.570+0000 7f0582000700 1 -- 172.21.15.28:0/752672630 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0588005040 con 0x559358483880 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.571+0000 7f05a210eb80 1 -- 172.21.15.28:0/752672630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 msgr2=0x559358483ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.571+0000 7f05a210eb80 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483ce0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f059006ccb0 tx=0x7f0588003f30 comp rx=0 tx=0).stop 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.572+0000 7f05a210eb80 1 -- 172.21.15.28:0/752672630 shutdown_connections 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.572+0000 7f05a210eb80 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483ce0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.572+0000 7f05a210eb80 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 0x559358489290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.572+0000 7f05a210eb80 1 --2- 172.21.15.28:0/752672630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x559358483260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.572+0000 7f05a210eb80 1 -- 172.21.15.28:0/752672630 >> 172.21.15.28:0/752672630 conn(0x5593583b1be0 msgr2=0x55935847e010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:10.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.573+0000 7f05a210eb80 1 -- 172.21.15.28:0/752672630 shutdown_connections 2024-05-20T20:09:10.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.573+0000 7f05a210eb80 1 -- 172.21.15.28:0/752672630 wait complete. 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.573+0000 7f05a210eb80 1 -- start start 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.573+0000 7f05a210eb80 1 Processor -- start 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358483880 0x55935848f510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 0x55935848c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cdf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f05a210eb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5593582d7fa0 con 0x5593584897d0 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f05a210eb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x559358372760 con 0x559358483880 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f05a210eb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5593584901e0 con 0x559358488eb0 2024-05-20T20:09:10.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f05a210eb80 10 client.0 _ll_register_callbacks cb 0x55935837bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:10.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.574+0000 7f0594e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358483880 0x55935848f510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.575+0000 7f0594e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358483880 0x55935848f510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36718/0 (socket says 172.21.15.28:36718) 2024-05-20T20:09:10.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.575+0000 7f0594e00700 1 -- 172.21.15.28:0/591983605 learned_addr learned my addr 172.21.15.28:0/591983605 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:10.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.575+0000 7f058fe00700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 0x55935848c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.575+0000 7f0595800700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.575+0000 7f0594e00700 1 -- 172.21.15.28:0/591983605 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 msgr2=0x55935848c8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:10.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.575+0000 7f0594e00700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 0x55935848c8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.576+0000 7f0594e00700 1 -- 172.21.15.28:0/591983605 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 msgr2=0x55935848cdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:10.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.576+0000 7f0594e00700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.576+0000 7f0594e00700 1 -- 172.21.15.28:0/591983605 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5593583218d0 con 0x559358483880 2024-05-20T20:09:10.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.576+0000 7f0595800700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:10.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.576+0000 7f0594e00700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358483880 0x55935848f510 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f0584011800 tx=0x7f0584011b10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bef1854249045dc6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:10.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.576+0000 7f0576a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:10.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.577+0000 7f0576a00700 1 -- 172.21.15.28:0/591983605 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f058400f750 con 0x559358483880 2024-05-20T20:09:10.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.577+0000 7f0576a00700 1 -- 172.21.15.28:0/591983605 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f058400f8f0 con 0x559358483880 2024-05-20T20:09:10.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.578+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0588021460 con 0x559358483880 2024-05-20T20:09:10.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.578+0000 7f0576a00700 1 -- 172.21.15.28:0/591983605 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f058400faf0 con 0x559358483880 2024-05-20T20:09:10.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.578+0000 7f05a210eb80 10 client.26234 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:10.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.578+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5593583a2060 con 0x559358483880 2024-05-20T20:09:10.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.579+0000 7f0576a00700 1 -- 172.21.15.28:0/591983605 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f058400fc90 con 0x559358483880 2024-05-20T20:09:10.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.580+0000 7f0576a00700 1 -- 172.21.15.28:0/591983605 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0584022030 con 0x559358483880 2024-05-20T20:09:10.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.580+0000 7f05a210eb80 10 client.26234 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:10.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.580+0000 7f05a210eb80 10 client.26234 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:09:10.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.580+0000 7f05a210eb80 20 client.26234 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:10.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.580+0000 7f05a210eb80 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55935849cd90 0x55935849f180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.580+0000 7f05a210eb80 4 client.26234 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:10.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.580+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f0588021460 con 0x55935849cd90 2024-05-20T20:09:10.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.582+0000 7f058fe00700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55935849cd90 0x55935849f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.582+0000 7f058fe00700 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55935849cd90 0x55935849f180 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=88d11cc28e2c2f9b server_cookie=c41b4d57287b39cb in_seq=0 out_seq=0 2024-05-20T20:09:10.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.582+0000 7f0576a00700 10 client.26234 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:09:10.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f0576a00700 1 -- 172.21.15.28:0/591983605 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x7f0588021460 con 0x55935849cd90 2024-05-20T20:09:10.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f0576a00700 10 client.26234 handle_command_reply: tid=0 2024-05-20T20:09:10.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 client.26234 shutdown 2024-05-20T20:09:10.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358483880 msgr2=0x55935848f510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358483880 0x55935848f510 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f0584011800 tx=0x7f0584011b10 comp rx=0 tx=0).stop 2024-05-20T20:09:10.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 shutdown_connections 2024-05-20T20:09:10.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55935849cd90 0x55935849f180 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358483880 0x55935848f510 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 --2- 172.21.15.28:0/591983605 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 0x55935848c8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 >> 172.21.15.28:0/591983605 conn(0x5593583b1be0 msgr2=0x559358482250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:10.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.584+0000 7f0576a00700 0 client.26234 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:09:10.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.585+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 shutdown_connections 2024-05-20T20:09:10.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.586+0000 7f05a210eb80 1 -- 172.21.15.28:0/591983605 wait complete. 2024-05-20T20:09:10.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.586+0000 7f05a210eb80 20 client.26234 trim_cache size 0 max 16384 2024-05-20T20:09:10.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 Processor -- start 2024-05-20T20:09:10.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 -- start start 2024-05-20T20:09:10.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 0x55935848c690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0590119620 con 0x5593584897d0 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f059007c080 con 0x559358488eb0 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.620+0000 7f05a210eb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0590119cf0 con 0x559358483880 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.621+0000 7f0595800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.621+0000 7f0595800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cbd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:39056/0 (socket says 172.21.15.28:39056) 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.621+0000 7f0595800700 1 -- 172.21.15.28:0/3843468284 learned_addr learned my addr 172.21.15.28:0/3843468284 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.622+0000 7f0595800700 1 -- 172.21.15.28:0/3843468284 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 msgr2=0x559358483260 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.622+0000 7f0595800700 1 --2- 172.21.15.28:0/3843468284 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.623+0000 7f0595800700 1 -- 172.21.15.28:0/3843468284 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 msgr2=0x55935848c690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.623+0000 7f0595800700 1 --2- 172.21.15.28:0/3843468284 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 0x55935848c690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.623+0000 7f0595800700 1 -- 172.21.15.28:0/3843468284 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55935848ae40 con 0x5593584897d0 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.623+0000 7f0595800700 1 --2- 172.21.15.28:0/3843468284 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cbd0 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f058800cb40 tx=0x7f05880094f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=daf0d95988db1170 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.624+0000 7f0581600700 1 -- 172.21.15.28:0/3843468284 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0588058030 con 0x5593584897d0 2024-05-20T20:09:10.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.624+0000 7f0581600700 1 -- 172.21.15.28:0/3843468284 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0588007b40 con 0x5593584897d0 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.624+0000 7f0581600700 1 -- 172.21.15.28:0/3843468284 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f058801e430 con 0x5593584897d0 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.625+0000 7f05a210eb80 1 -- 172.21.15.28:0/3843468284 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 msgr2=0x55935848cbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.625+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3843468284 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cbd0 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f058800cb40 tx=0x7f05880094f0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.626+0000 7f05a210eb80 1 -- 172.21.15.28:0/3843468284 shutdown_connections 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.626+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3843468284 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593584897d0 0x55935848cbd0 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.626+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3843468284 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358483880 0x559358483260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.626+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3843468284 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559358488eb0 0x55935848c690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.626+0000 7f05a210eb80 1 -- 172.21.15.28:0/3843468284 >> 172.21.15.28:0/3843468284 conn(0x5593583b1be0 msgr2=0x559358477e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:10.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.633+0000 7f05a210eb80 1 -- 172.21.15.28:0/3843468284 shutdown_connections 2024-05-20T20:09:10.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.633+0000 7f05a210eb80 1 -- 172.21.15.28:0/3843468284 wait complete. 2024-05-20T20:09:10.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.635+0000 7f05a210eb80 1 -- start start 2024-05-20T20:09:10.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.636+0000 7f05a210eb80 1 Processor -- start 2024-05-20T20:09:10.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.637+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559358483880 0x559358495650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.637+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 0x559358495b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.637+0000 7f05a210eb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593584897d0 0x559358492440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.638+0000 7f05a210eb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f059011aa90 con 0x559358483880 2024-05-20T20:09:10.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.638+0000 7f05a210eb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f059011a3c0 con 0x5593584897d0 2024-05-20T20:09:10.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.638+0000 7f05a210eb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f05901175c0 con 0x559358488eb0 2024-05-20T20:09:10.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.638+0000 7f05a210eb80 10 client.0 _ll_register_callbacks cb 0x55935837bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:10.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.639+0000 7f0595800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593584897d0 0x559358492440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.639+0000 7f0595800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593584897d0 0x559358492440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36732/0 (socket says 172.21.15.28:36732) 2024-05-20T20:09:10.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.639+0000 7f0595800700 1 -- 172.21.15.28:0/3220660600 learned_addr learned my addr 172.21.15.28:0/3220660600 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:10.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.640+0000 7f0595800700 1 -- 172.21.15.28:0/3220660600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 msgr2=0x559358495b90 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:10.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.640+0000 7f0595800700 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 0x559358495b90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.640+0000 7f0595800700 1 -- 172.21.15.28:0/3220660600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559358483880 msgr2=0x559358495650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:10.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.639+0000 7f0594e00700 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559358483880 0x559358495650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.641+0000 7f0595800700 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559358483880 0x559358495650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.641+0000 7f0595800700 1 -- 172.21.15.28:0/3220660600 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559358492ab0 con 0x5593584897d0 2024-05-20T20:09:10.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.641+0000 7f0594e00700 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559358483880 0x559358495650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:10.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.642+0000 7f0595800700 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593584897d0 0x559358492440 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f0588021bd0 tx=0x7f0588021c00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=36bb3e31a2f6dc7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:10.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.642+0000 7f0576a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:10.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.643+0000 7f0576a00700 1 -- 172.21.15.28:0/3220660600 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0588060020 con 0x5593584897d0 2024-05-20T20:09:10.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.643+0000 7f0576a00700 1 -- 172.21.15.28:0/3220660600 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0588027040 con 0x5593584897d0 2024-05-20T20:09:10.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.643+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0588007b40 con 0x5593584897d0 2024-05-20T20:09:10.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.644+0000 7f0576a00700 1 -- 172.21.15.28:0/3220660600 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f058801e640 con 0x5593584897d0 2024-05-20T20:09:10.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.644+0000 7f05a210eb80 10 client.26237 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:10.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.644+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559358508260 con 0x5593584897d0 2024-05-20T20:09:10.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.645+0000 7f0576a00700 1 -- 172.21.15.28:0/3220660600 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0588007b40 con 0x5593584897d0 2024-05-20T20:09:10.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.645+0000 7f0576a00700 1 -- 172.21.15.28:0/3220660600 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0588011070 con 0x5593584897d0 2024-05-20T20:09:10.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.648+0000 7f05a210eb80 10 client.26237 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:10.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.648+0000 7f05a210eb80 10 client.26237 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:09:10.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.648+0000 7f05a210eb80 20 client.26237 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:10.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.648+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5593585084e0 0x55935850a8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:10.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.649+0000 7f05a210eb80 4 client.26237 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:09:10.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.649+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5593583e3d90 con 0x5593585084e0 2024-05-20T20:09:10.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.649+0000 7f0594e00700 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5593585084e0 0x55935850a8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:10.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.650+0000 7f0594e00700 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5593585084e0 0x55935850a8d0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=a5f1f9d84ace77d8 server_cookie=741044d85ff06f61 in_seq=0 out_seq=0 2024-05-20T20:09:10.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.650+0000 7f0576a00700 10 client.26237 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:09:10.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.651+0000 7f0576a00700 1 -- 172.21.15.28:0/3220660600 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5593583e3d90 con 0x5593585084e0 2024-05-20T20:09:10.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.651+0000 7f0576a00700 10 client.26237 handle_command_reply: tid=0 2024-05-20T20:09:10.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.651+0000 7f05a210eb80 1 client.26237 shutdown 2024-05-20T20:09:10.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.651+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593584897d0 msgr2=0x559358492440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.651+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593584897d0 0x559358492440 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f0588021bd0 tx=0x7f0588021c00 comp rx=0 tx=0).stop 2024-05-20T20:09:10.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.651+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 shutdown_connections 2024-05-20T20:09:10.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.651+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5593585084e0 0x55935850a8d0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.652+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593584897d0 0x559358492440 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.652+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559358483880 0x559358495650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.652+0000 7f05a210eb80 1 --2- 172.21.15.28:0/3220660600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559358488eb0 0x559358495b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.652+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 >> 172.21.15.28:0/3220660600 conn(0x5593583b1be0 msgr2=0x55935847d9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:10.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.652+0000 7f0576a00700 0 client.26237 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:09:10.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.652+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 shutdown_connections 2024-05-20T20:09:10.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.653+0000 7f05a210eb80 1 -- 172.21.15.28:0/3220660600 wait complete. 2024-05-20T20:09:10.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.653+0000 7f05a210eb80 20 client.26237 trim_cache size 0 max 16384 2024-05-20T20:09:10.655 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:09:10.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f05780b2c70 msgr2=0x7f05780b5130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f05780b2c70 0x7f05780b5130 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f0588009f90 tx=0x7f0588023000 comp rx=0 tx=0).stop 2024-05-20T20:09:10.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0590111d20 msgr2=0x7f05901aeb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:10.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0590111d20 0x7f05901aeb30 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f057c00e9c0 tx=0x7f057c00ecd0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 shutdown_connections 2024-05-20T20:09:10.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f05780b2c70 0x7f05780b5130 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f059011cd20 0x7f05901a86c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f059010aa80 0x7f05901ae5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 --2- 172.21.15.28:0/1126626235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0590111d20 0x7f05901aeb30 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:10.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 >> 172.21.15.28:0/1126626235 conn(0x7f059006e6b0 msgr2=0x7f059011ad10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:10.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.654+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 shutdown_connections 2024-05-20T20:09:10.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:10.655+0000 7f0597800700 1 -- 172.21.15.28:0/1126626235 wait complete. 2024-05-20T20:09:11.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:10 smithi028 ceph-mon[31178]: pgmap v109: 97 pgs: 97 active+clean; 9.7 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 400 MiB/s wr, 524 op/s 2024-05-20T20:09:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:10 smithi154 ceph-mon[36314]: pgmap v109: 97 pgs: 97 active+clean; 9.7 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 400 MiB/s wr, 524 op/s 2024-05-20T20:09:11.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:10 smithi033 ceph-mon[36179]: pgmap v109: 97 pgs: 97 active+clean; 9.7 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 400 MiB/s wr, 524 op/s 2024-05-20T20:09:11.657 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:09:12.408 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:12.987 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:12 smithi028 ceph-mon[31178]: pgmap v110: 97 pgs: 97 active+clean; 9.7 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 259 MiB/s wr, 483 op/s 2024-05-20T20:09:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:12 smithi154 ceph-mon[36314]: pgmap v110: 97 pgs: 97 active+clean; 9.7 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 259 MiB/s wr, 483 op/s 2024-05-20T20:09:13.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:12 smithi033 ceph-mon[36179]: pgmap v110: 97 pgs: 97 active+clean; 9.7 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 259 MiB/s wr, 483 op/s 2024-05-20T20:09:15.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:14 smithi028 ceph-mon[31178]: pgmap v111: 97 pgs: 97 active+clean; 9.9 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 358 MiB/s wr, 1.36k op/s 2024-05-20T20:09:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:14 smithi154 ceph-mon[36314]: pgmap v111: 97 pgs: 97 active+clean; 9.9 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 358 MiB/s wr, 1.36k op/s 2024-05-20T20:09:15.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:14 smithi033 ceph-mon[36179]: pgmap v111: 97 pgs: 97 active+clean; 9.9 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 358 MiB/s wr, 1.36k op/s 2024-05-20T20:09:15.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.891+0000 7f7b63600700 1 -- 172.21.15.28:0/2356049201 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c10b750 msgr2=0x7f7b5c10bb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.891+0000 7f7b63600700 1 --2- 172.21.15.28:0/2356049201 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c10b750 0x7f7b5c10bb30 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f7b50009990 tx=0x7f7b50009ca0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.891+0000 7f7b63600700 1 -- 172.21.15.28:0/2356049201 shutdown_connections 2024-05-20T20:09:15.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.891+0000 7f7b63600700 1 --2- 172.21.15.28:0/2356049201 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7b5c106110 0x7f7b5c112620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.891+0000 7f7b63600700 1 --2- 172.21.15.28:0/2356049201 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c10b750 0x7f7b5c10bb30 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.891+0000 7f7b63600700 1 --2- 172.21.15.28:0/2356049201 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c105750 0x7f7b5c105bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.891+0000 7f7b63600700 1 -- 172.21.15.28:0/2356049201 >> 172.21.15.28:0/2356049201 conn(0x7f7b5c101280 msgr2=0x7f7b5c1036a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:15.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.892+0000 7f7b63600700 1 -- 172.21.15.28:0/2356049201 shutdown_connections 2024-05-20T20:09:15.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.892+0000 7f7b63600700 1 -- 172.21.15.28:0/2356049201 wait complete. 2024-05-20T20:09:15.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 Processor -- start 2024-05-20T20:09:15.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 -- start start 2024-05-20T20:09:15.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c105750 0x7f7b5c19bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c106110 0x7f7b5c19c320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7b5c10b750 0x7f7b5c1a06b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c1159d0 con 0x7f7b5c105750 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c115850 con 0x7f7b5c106110 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.894+0000 7f7b63600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c115b50 con 0x7f7b5c10b750 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c106110 0x7f7b5c19c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b60c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c105750 0x7f7b5c19bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c106110 0x7f7b5c19c320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36758/0 (socket says 172.21.15.28:36758) 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 -- 172.21.15.28:0/252855383 learned_addr learned my addr 172.21.15.28:0/252855383 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:15.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b61600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7b5c10b750 0x7f7b5c1a06b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 -- 172.21.15.28:0/252855383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7b5c10b750 msgr2=0x7f7b5c1a06b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7b5c10b750 0x7f7b5c1a06b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 -- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c105750 msgr2=0x7f7b5c19bde0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c105750 0x7f7b5c19bde0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 -- 172.21.15.28:0/252855383 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7b5c1a0df0 con 0x7f7b5c106110 2024-05-20T20:09:15.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.895+0000 7f7b5be00700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c106110 0x7f7b5c19c320 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f7b4800e9a0 tx=0x7f7b4800ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:15.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.896+0000 7f7b59600700 1 -- 172.21.15.28:0/252855383 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b4800ca10 con 0x7f7b5c106110 2024-05-20T20:09:15.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.896+0000 7f7b59600700 1 -- 172.21.15.28:0/252855383 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7b4800cbb0 con 0x7f7b5c106110 2024-05-20T20:09:15.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.896+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7b5c1a10e0 con 0x7f7b5c106110 2024-05-20T20:09:15.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.896+0000 7f7b59600700 1 -- 172.21.15.28:0/252855383 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b48018680 con 0x7f7b5c106110 2024-05-20T20:09:15.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.896+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7b5c1a8960 con 0x7f7b5c106110 2024-05-20T20:09:15.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.902+0000 7f7b59600700 1 -- 172.21.15.28:0/252855383 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7b48010430 con 0x7f7b5c106110 2024-05-20T20:09:15.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.902+0000 7f7b6df82b80 1 Processor -- start 2024-05-20T20:09:15.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.903+0000 7f7b6df82b80 1 -- start start 2024-05-20T20:09:15.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.904+0000 7f7b59600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7b440b2b20 0x7f7b440b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b59600700 1 -- 172.21.15.28:0/252855383 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7b480e2760 con 0x7f7b5c106110 2024-05-20T20:09:15.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b61600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7b440b2b20 0x7f7b440b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 0x5599f1961aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195ba70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f195c090 0x5599f195c4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5599f18b1e40 con 0x5599f195c090 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5599f18a19e0 con 0x5599f19616c0 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5599f18a5a20 con 0x5599f1961fe0 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b5be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195ba70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b5be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195ba70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58666/0 (socket says 172.21.15.28:58666) 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.905+0000 7f7b5be00700 1 -- 172.21.15.28:0/453783858 learned_addr learned my addr 172.21.15.28:0/453783858 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.906+0000 7f7b5be00700 1 -- 172.21.15.28:0/453783858 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 msgr2=0x5599f1961aa0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.906+0000 7f7b5be00700 1 --2- 172.21.15.28:0/453783858 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 0x5599f1961aa0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.906+0000 7f7b5be00700 1 -- 172.21.15.28:0/453783858 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f195c090 msgr2=0x5599f195c4f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.906+0000 7f7b5be00700 1 --2- 172.21.15.28:0/453783858 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f195c090 0x5599f195c4f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.906+0000 7f7b5be00700 1 -- 172.21.15.28:0/453783858 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5599f186a420 con 0x5599f1961fe0 2024-05-20T20:09:15.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.906+0000 7f7b5be00700 1 --2- 172.21.15.28:0/453783858 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195ba70 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f7b480a5a70 tx=0x7f7b480a9470 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f6fb4b347c236a27 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:15.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.906+0000 7f7b4e000700 1 -- 172.21.15.28:0/453783858 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b480a99f0 con 0x5599f1961fe0 2024-05-20T20:09:15.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.907+0000 7f7b61600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7b440b2b20 0x7f7b440b4fe0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f7b5400f3c0 tx=0x7f7b54007950 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:15.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.907+0000 7f7b4e000700 1 -- 172.21.15.28:0/453783858 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7b480adda0 con 0x5599f1961fe0 2024-05-20T20:09:15.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.907+0000 7f7b4e000700 1 -- 172.21.15.28:0/453783858 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b480a9eb0 con 0x5599f1961fe0 2024-05-20T20:09:15.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 -- 172.21.15.28:0/453783858 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 msgr2=0x5599f195ba70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/453783858 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195ba70 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f7b480a5a70 tx=0x7f7b480a9470 comp rx=0 tx=0).stop 2024-05-20T20:09:15.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 -- 172.21.15.28:0/453783858 shutdown_connections 2024-05-20T20:09:15.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/453783858 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f195c090 0x5599f195c4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/453783858 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 0x5599f1961aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/453783858 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195ba70 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 -- 172.21.15.28:0/453783858 >> 172.21.15.28:0/453783858 conn(0x5599f1881be0 msgr2=0x5599f194e010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:15.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.908+0000 7f7b6df82b80 1 -- 172.21.15.28:0/453783858 shutdown_connections 2024-05-20T20:09:15.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.909+0000 7f7b6df82b80 1 -- 172.21.15.28:0/453783858 wait complete. 2024-05-20T20:09:15.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.909+0000 7f7b6df82b80 1 -- start start 2024-05-20T20:09:15.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.909+0000 7f7b6df82b80 1 Processor -- start 2024-05-20T20:09:15.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.910+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1967dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 0x5599f19650e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f1961fe0 0x5599f1965620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5599f17a7fa0 con 0x5599f1961fe0 2024-05-20T20:09:15.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5599f1842760 con 0x5599f19616c0 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5599f19689f0 con 0x5599f195c090 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b61600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f1961fe0 0x5599f1965620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b61600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f1961fe0 0x5599f1965620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:39102/0 (socket says 172.21.15.28:39102) 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.911+0000 7f7b61600700 1 -- 172.21.15.28:0/1545725590 learned_addr learned my addr 172.21.15.28:0/1545725590 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b60c00700 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1967dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b61600700 1 -- 172.21.15.28:0/1545725590 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 msgr2=0x5599f1967dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b5be00700 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 0x5599f19650e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b61600700 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1967dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b61600700 1 -- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 msgr2=0x5599f19650e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:15.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b61600700 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 0x5599f19650e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b6df82b80 10 client.0 _ll_register_callbacks cb 0x5599f184bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:15.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b61600700 1 -- 172.21.15.28:0/1545725590 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5599f17f18d0 con 0x5599f1961fe0 2024-05-20T20:09:15.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.912+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7b480adda0 con 0x5599f1961fe0 2024-05-20T20:09:15.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.913+0000 7f7b61600700 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f1961fe0 0x5599f1965620 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f7b54014e40 tx=0x7f7b54004ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7f258564741cc737 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:15.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.913+0000 7f7b42a00700 10 client.16083 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:15.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.913+0000 7f7b42a00700 1 -- 172.21.15.28:0/1545725590 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b5401b050 con 0x5599f1961fe0 2024-05-20T20:09:15.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.913+0000 7f7b42a00700 1 -- 172.21.15.28:0/1545725590 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7b540159f0 con 0x5599f1961fe0 2024-05-20T20:09:15.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.913+0000 7f7b42a00700 1 -- 172.21.15.28:0/1545725590 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f7b54015bf0 con 0x5599f1961fe0 2024-05-20T20:09:15.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.914+0000 7f7b6df82b80 10 client.16083 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:15.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.914+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({fsmap=29}) -- 0x5599f1872060 con 0x5599f1961fe0 2024-05-20T20:09:15.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.915+0000 7f7b42a00700 1 -- 172.21.15.28:0/1545725590 <== mon.0 v2:172.21.15.28:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7b54020440 con 0x5599f1961fe0 2024-05-20T20:09:15.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.916+0000 7f7b6df82b80 10 client.16083 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:15.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.916+0000 7f7b6df82b80 10 client.16083 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:15.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.916+0000 7f7b6df82b80 20 client.16083 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:15.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.916+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f1975640 0x5599f1977a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.916+0000 7f7b6df82b80 4 client.16083 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:15.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.916+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5599f18b3d90 con 0x5599f1975640 2024-05-20T20:09:15.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.916+0000 7f7b60c00700 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f1975640 0x5599f1977a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.918+0000 7f7b42a00700 10 client.16083 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:15.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.918+0000 7f7b60c00700 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f1975640 0x5599f1977a30 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4fd20a730e1136b7 server_cookie=b159036d507ec942 in_seq=0 out_seq=0 2024-05-20T20:09:15.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.919+0000 7f7b42a00700 1 -- 172.21.15.28:0/1545725590 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5599f18b3d90 con 0x5599f1975640 2024-05-20T20:09:15.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.919+0000 7f7b42a00700 10 client.16083 handle_command_reply: tid=0 2024-05-20T20:09:15.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.919+0000 7f7b6df82b80 1 client.16083 shutdown 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f1961fe0 msgr2=0x5599f1965620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f1961fe0 0x5599f1965620 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f7b54014e40 tx=0x7f7b54004ad0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 shutdown_connections 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f1975640 0x5599f1977a30 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f1961fe0 0x5599f1965620 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1967dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1545725590 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f19616c0 0x5599f19650e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.920+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 >> 172.21.15.28:0/1545725590 conn(0x5599f1881be0 msgr2=0x5599f195aa60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:15.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.921+0000 7f7b42a00700 0 client.16083 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:15.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.921+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 shutdown_connections 2024-05-20T20:09:15.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.921+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1545725590 wait complete. 2024-05-20T20:09:15.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.921+0000 7f7b6df82b80 20 client.16083 trim_cache size 0 max 16384 2024-05-20T20:09:15.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 Processor -- start 2024-05-20T20:09:15.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 -- start start 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f195c090 0x5599f195ba70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f195c470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c1abc50 con 0x5599f19616c0 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c1ad790 con 0x5599f195c090 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c1ac320 con 0x5599f1961fe0 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b60c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f195c090 0x5599f195ba70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b5be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f195c470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b60c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f195c090 0x5599f195ba70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36780/0 (socket says 172.21.15.28:36780) 2024-05-20T20:09:15.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.948+0000 7f7b60c00700 1 -- 172.21.15.28:0/1886743321 learned_addr learned my addr 172.21.15.28:0/1886743321 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:15.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b61600700 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b60c00700 1 -- 172.21.15.28:0/1886743321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 msgr2=0x5599f195f1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b60c00700 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195f1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b60c00700 1 -- 172.21.15.28:0/1886743321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 msgr2=0x5599f195c470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b60c00700 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f195c470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b60c00700 1 -- 172.21.15.28:0/1886743321 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5599f1965930 con 0x5599f195c090 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b60c00700 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f195c090 0x5599f195ba70 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f7b50009990 tx=0x7f7b50005fb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d53805f1cd3ff927 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b4d600700 1 -- 172.21.15.28:0/1886743321 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b5000bd40 con 0x5599f195c090 2024-05-20T20:09:15.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b4d600700 1 -- 172.21.15.28:0/1886743321 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7b50003680 con 0x5599f195c090 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.949+0000 7f7b4d600700 1 -- 172.21.15.28:0/1886743321 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b5000f6a0 con 0x5599f195c090 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.950+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1886743321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f195c090 msgr2=0x5599f195ba70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.950+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f195c090 0x5599f195ba70 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f7b50009990 tx=0x7f7b50005fb0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.950+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1886743321 shutdown_connections 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.950+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f1961fe0 0x5599f195f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.950+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f195c090 0x5599f195ba70 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.950+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/1886743321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f195c470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.950+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1886743321 >> 172.21.15.28:0/1886743321 conn(0x5599f1881be0 msgr2=0x5599f19639d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:15.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.951+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1886743321 shutdown_connections 2024-05-20T20:09:15.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.951+0000 7f7b6df82b80 1 -- 172.21.15.28:0/1886743321 wait complete. 2024-05-20T20:09:15.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.951+0000 7f7b6df82b80 1 -- start start 2024-05-20T20:09:15.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.951+0000 7f7b6df82b80 1 Processor -- start 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1978420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f1978960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b6df82b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f1961fe0 0x5599f1978ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c1ad0c0 con 0x5599f19616c0 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c1ac9f0 con 0x5599f1961fe0 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b6df82b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7b5c07bbe0 con 0x5599f195c090 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b6df82b80 10 client.0 _ll_register_callbacks cb 0x5599f184bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b60c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1978420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b60c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1978420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58686/0 (socket says 172.21.15.28:58686) 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b60c00700 1 -- 172.21.15.28:0/3237029294 learned_addr learned my addr 172.21.15.28:0/3237029294 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b61600700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f1961fe0 0x5599f1978ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b5be00700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f1978960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b60c00700 1 -- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f1961fe0 msgr2=0x5599f1978ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b60c00700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f1961fe0 0x5599f1978ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b60c00700 1 -- 172.21.15.28:0/3237029294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 msgr2=0x5599f1978960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.952+0000 7f7b60c00700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f1978960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b60c00700 1 -- 172.21.15.28:0/3237029294 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5599f19deeb0 con 0x5599f195c090 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b61600700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f1961fe0 0x5599f1978ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b5be00700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f1978960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b60c00700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1978420 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f7b50005950 tx=0x7f7b50005980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e67dc4ed7c2433b7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b42a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b42a00700 1 -- 172.21.15.28:0/3237029294 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b50005310 con 0x5599f195c090 2024-05-20T20:09:15.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b42a00700 1 -- 172.21.15.28:0/3237029294 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7b50003830 con 0x5599f195c090 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7b50003680 con 0x5599f195c090 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b42a00700 1 -- 172.21.15.28:0/3237029294 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f7b50042050 con 0x5599f195c090 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b6df82b80 10 client.26191 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.953+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5599f19e0190 con 0x5599f195c090 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.954+0000 7f7b42a00700 1 -- 172.21.15.28:0/3237029294 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7b50003680 con 0x5599f195c090 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.954+0000 7f7b42a00700 1 -- 172.21.15.28:0/3237029294 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7b5000f9f0 con 0x5599f195c090 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.954+0000 7f7b6df82b80 10 client.26191 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.954+0000 7f7b6df82b80 10 client.26191 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.954+0000 7f7b6df82b80 20 client.26191 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:15.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.955+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f19e06d0 0x5599f19e2ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:15.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.955+0000 7f7b6df82b80 4 client.26191 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:09:15.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.955+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5599f18ad570 con 0x5599f19e06d0 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.955+0000 7f7b5be00700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f19e06d0 0x5599f19e2ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.957+0000 7f7b5be00700 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f19e06d0 0x5599f19e2ac0 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=83d25bae3fbadd0e server_cookie=eee7ec83bd156fb4 in_seq=0 out_seq=0 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.957+0000 7f7b42a00700 10 client.26191 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.957+0000 7f7b42a00700 1 -- 172.21.15.28:0/3237029294 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5599f18ad570 con 0x5599f19e06d0 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.957+0000 7f7b42a00700 10 client.26191 handle_command_reply: tid=0 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.957+0000 7f7b6df82b80 1 client.26191 shutdown 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.957+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 msgr2=0x5599f1978420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.957+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1978420 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f7b50005950 tx=0x7f7b50005980 comp rx=0 tx=0).stop 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 shutdown_connections 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5599f19e06d0 0x5599f19e2ac0 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5599f1961fe0 0x5599f1978ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5599f195c090 0x5599f1978420 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 --2- 172.21.15.28:0/3237029294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5599f19616c0 0x5599f1978960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b42a00700 0 client.26191 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 >> 172.21.15.28:0/3237029294 conn(0x5599f1881be0 msgr2=0x5599f1963950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 shutdown_connections 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 1 -- 172.21.15.28:0/3237029294 wait complete. 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.958+0000 7f7b6df82b80 20 client.26191 trim_cache size 0 max 16384 2024-05-20T20:09:15.961 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7b440b2b20 msgr2=0x7f7b440b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7b440b2b20 0x7f7b440b4fe0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f7b5400f3c0 tx=0x7f7b54007950 comp rx=0 tx=0).stop 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c106110 msgr2=0x7f7b5c19c320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c106110 0x7f7b5c19c320 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f7b4800e9a0 tx=0x7f7b4800ecb0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 shutdown_connections 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7b5c10b750 0x7f7b5c1a06b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7b5c105750 0x7f7b5c19bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7b440b2b20 0x7f7b440b4fe0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 --2- 172.21.15.28:0/252855383 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7b5c106110 0x7f7b5c19c320 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:15.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 >> 172.21.15.28:0/252855383 conn(0x7f7b5c101280 msgr2=0x7f7b5c101960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:15.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.959+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 shutdown_connections 2024-05-20T20:09:15.963 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:15.960+0000 7f7b63600700 1 -- 172.21.15.28:0/252855383 wait complete. 2024-05-20T20:09:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:16 smithi154 ceph-mon[36314]: pgmap v112: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 283 MiB/s wr, 1.43k op/s 2024-05-20T20:09:17.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:16 smithi033 ceph-mon[36179]: pgmap v112: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 283 MiB/s wr, 1.43k op/s 2024-05-20T20:09:17.165 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.164+0000 7f751de00640 1 -- 172.21.15.28:0/703309764 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f751810ce30 msgr2=0x7f751810f2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.165 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.164+0000 7f751de00640 1 --2- 172.21.15.28:0/703309764 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f751810ce30 0x7f751810f2d0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f7510009ca0 tx=0x7f751002a940 comp rx=0 tx=0).stop 2024-05-20T20:09:17.167 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.166+0000 7f751de00640 1 -- 172.21.15.28:0/703309764 shutdown_connections 2024-05-20T20:09:17.167 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.166+0000 7f751de00640 1 --2- 172.21.15.28:0/703309764 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f751810ce30 0x7f751810f2d0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.167 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.166+0000 7f751de00640 1 --2- 172.21.15.28:0/703309764 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518104330 0x7f751810c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.166+0000 7f751de00640 1 --2- 172.21.15.28:0/703309764 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7518103a10 0x7f7518103df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.166+0000 7f751de00640 1 -- 172.21.15.28:0/703309764 >> 172.21.15.28:0/703309764 conn(0x7f75180fd270 msgr2=0x7f75180ff690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:17.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.166+0000 7f751de00640 1 -- 172.21.15.28:0/703309764 shutdown_connections 2024-05-20T20:09:17.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.167+0000 7f751de00640 1 -- 172.21.15.28:0/703309764 wait complete. 2024-05-20T20:09:17.169 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.168+0000 7f751de00640 1 Processor -- start 2024-05-20T20:09:17.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.169+0000 7f751de00640 1 -- start start 2024-05-20T20:09:17.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.170+0000 7f751de00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518103a10 0x7f751810baf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.170+0000 7f751de00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7518104330 0x7f751810c030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.170+0000 7f751de00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f751810ce30 0x7f75181a3e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751de00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7518112b30 con 0x7f751810ce30 2024-05-20T20:09:17.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751de00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f75181129b0 con 0x7f7518104330 2024-05-20T20:09:17.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751de00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7518112cb0 con 0x7f7518103a10 2024-05-20T20:09:17.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f7517e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7518104330 0x7f751810c030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751ca00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518103a10 0x7f751810baf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751ca00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518103a10 0x7f751810baf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60562/0 (socket says 172.21.15.28:60562) 2024-05-20T20:09:17.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751ca00640 1 -- 172.21.15.28:0/2307530152 learned_addr learned my addr 172.21.15.28:0/2307530152 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:17.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751ca00640 1 -- 172.21.15.28:0/2307530152 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7518104330 msgr2=0x7f751810c030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.171+0000 7f751ca00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7518104330 0x7f751810c030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.172+0000 7f751ca00640 1 -- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f751810ce30 msgr2=0x7f75181a3e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.172+0000 7f751ca00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f751810ce30 0x7f75181a3e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.172+0000 7f751ca00640 1 -- 172.21.15.28:0/2307530152 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f75181a4580 con 0x7f7518103a10 2024-05-20T20:09:17.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.172+0000 7f751ca00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518103a10 0x7f751810baf0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f750800d950 tx=0x7f750800de10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:17.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.173+0000 7f7515600640 1 -- 172.21.15.28:0/2307530152 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7508017070 con 0x7f7518103a10 2024-05-20T20:09:17.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.173+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f75181a4870 con 0x7f7518103a10 2024-05-20T20:09:17.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.173+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f75181ac100 con 0x7f7518103a10 2024-05-20T20:09:17.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.173+0000 7f7515600640 1 -- 172.21.15.28:0/2307530152 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7508009c30 con 0x7f7518103a10 2024-05-20T20:09:17.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.173+0000 7f7515600640 1 -- 172.21.15.28:0/2307530152 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f750800f620 con 0x7f7518103a10 2024-05-20T20:09:17.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.186+0000 7f752220a740 1 Processor -- start 2024-05-20T20:09:17.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.186+0000 7f752220a740 1 -- start start 2024-05-20T20:09:17.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.187+0000 7f752220a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec4a230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.187+0000 7f752220a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 0x559e0ec54af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.187+0000 7f752220a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec57420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.188+0000 7f752220a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559e0ebbab90 con 0x559e0ec4a770 2024-05-20T20:09:17.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.188+0000 7f752220a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x559e0ebaa730 con 0x559e0ec55030 2024-05-20T20:09:17.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.188+0000 7f752220a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559e0ebae770 con 0x559e0ec49e50 2024-05-20T20:09:17.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.188+0000 7f7517e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec57420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.188+0000 7f7517e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec57420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37622/0 (socket says 172.21.15.28:37622) 2024-05-20T20:09:17.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.188+0000 7f7517e00640 1 -- 172.21.15.28:0/967311615 learned_addr learned my addr 172.21.15.28:0/967311615 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:17.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.192+0000 7f751d400640 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec4a230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.193+0000 7f7517e00640 1 -- 172.21.15.28:0/967311615 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 msgr2=0x559e0ec4a230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:17.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.193+0000 7f7517e00640 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec4a230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.193+0000 7f7517e00640 1 -- 172.21.15.28:0/967311615 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 msgr2=0x559e0ec54af0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:17.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.193+0000 7f7517e00640 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 0x559e0ec54af0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.193+0000 7f7517e00640 1 -- 172.21.15.28:0/967311615 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559e0eb438b0 con 0x559e0ec55030 2024-05-20T20:09:17.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.195+0000 7f7517e00640 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec57420 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f750400ac00 tx=0x7f750400f790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eaf45e43f6e3cecc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:17.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.197+0000 7f750dc00640 1 -- 172.21.15.28:0/967311615 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f750401d070 con 0x559e0ec55030 2024-05-20T20:09:17.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.197+0000 7f7515600640 1 -- 172.21.15.28:0/2307530152 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7508010460 con 0x7f7518103a10 2024-05-20T20:09:17.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.198+0000 7f750dc00640 1 -- 172.21.15.28:0/967311615 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7504018af0 con 0x559e0ec55030 2024-05-20T20:09:17.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.199+0000 7f752220a740 1 -- 172.21.15.28:0/967311615 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 msgr2=0x559e0ec57420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.199+0000 7f752220a740 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec57420 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f750400ac00 tx=0x7f750400f790 comp rx=0 tx=0).stop 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.201+0000 7f7515600640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74fc0b2b20 0x7f74fc0b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.201+0000 7f7515600640 1 -- 172.21.15.28:0/2307530152 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f75080e2150 con 0x7f7518103a10 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.202+0000 7f752220a740 1 -- 172.21.15.28:0/967311615 shutdown_connections 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.202+0000 7f752220a740 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec57420 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.202+0000 7f752220a740 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 0x559e0ec54af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.202+0000 7f752220a740 1 --2- 172.21.15.28:0/967311615 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec4a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.202+0000 7f752220a740 1 -- 172.21.15.28:0/967311615 >> 172.21.15.28:0/967311615 conn(0x559e0ec46360 msgr2=0x559e0ec46740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:17.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.202+0000 7f751d400640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74fc0b2b20 0x7f74fc0b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.204 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.203+0000 7f752220a740 1 -- 172.21.15.28:0/967311615 shutdown_connections 2024-05-20T20:09:17.204 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.204+0000 7f752220a740 1 -- 172.21.15.28:0/967311615 wait complete. 2024-05-20T20:09:17.205 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.204+0000 7f752220a740 1 -- start start 2024-05-20T20:09:17.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.207+0000 7f752220a740 1 Processor -- start 2024-05-20T20:09:17.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.209+0000 7f751d400640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74fc0b2b20 0x7f74fc0b4fe0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f751002ae60 tx=0x7f7510005df0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:17.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.211+0000 7f752220a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec49e50 0x559e0ec54980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.211+0000 7f752220a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec4a770 0x559e0ec51c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.212+0000 7f752220a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec521d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.212+0000 7f752220a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559e0e890790 con 0x559e0ec49e50 2024-05-20T20:09:17.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.212+0000 7f752220a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x559e0eb52bc0 con 0x559e0ec55030 2024-05-20T20:09:17.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.212+0000 7f752220a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559e0ec57c60 con 0x559e0ec4a770 2024-05-20T20:09:17.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.212+0000 7f752220a740 10 client.0 _ll_register_callbacks cb 0x559e0eb30400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:17.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.213+0000 7f751ca00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec49e50 0x559e0ec54980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.213+0000 7f751ca00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec49e50 0x559e0ec54980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36594/0 (socket says 172.21.15.28:36594) 2024-05-20T20:09:17.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.213+0000 7f751ca00640 1 -- 172.21.15.28:0/2155777865 learned_addr learned my addr 172.21.15.28:0/2155777865 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:17.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.213+0000 7f751d400640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec521d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.214+0000 7f7517e00640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec4a770 0x559e0ec51c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.214+0000 7f751d400640 1 -- 172.21.15.28:0/2155777865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec4a770 msgr2=0x559e0ec51c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.214+0000 7f751d400640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec4a770 0x559e0ec51c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.214+0000 7f751d400640 1 -- 172.21.15.28:0/2155777865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec49e50 msgr2=0x559e0ec54980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.214+0000 7f751d400640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec49e50 0x559e0ec54980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.214+0000 7f751d400640 1 -- 172.21.15.28:0/2155777865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559e0e8bf210 con 0x559e0ec55030 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.214+0000 7f751ca00640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec49e50 0x559e0ec54980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.215+0000 7f751d400640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec521d0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f751002af30 tx=0x7f7510005d80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=16bb54acee7b59fa server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.215+0000 7f7517e00640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec4a770 0x559e0ec51c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:17.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.215+0000 7f7502000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:17.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.215+0000 7f7502000640 1 -- 172.21.15.28:0/2155777865 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7510034070 con 0x559e0ec55030 2024-05-20T20:09:17.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.215+0000 7f7502000640 1 -- 172.21.15.28:0/2155777865 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7510030d10 con 0x559e0ec55030 2024-05-20T20:09:17.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.216+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f7504018af0 con 0x559e0ec55030 2024-05-20T20:09:17.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.217+0000 7f7502000640 1 -- 172.21.15.28:0/2155777865 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f7510003f80 con 0x559e0ec55030 2024-05-20T20:09:17.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.217+0000 7f752220a740 10 client.26261 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:17.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.218+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559e0e98b620 con 0x559e0ec55030 2024-05-20T20:09:17.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.218+0000 7f7502000640 1 -- 172.21.15.28:0/2155777865 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7504018af0 con 0x559e0ec55030 2024-05-20T20:09:17.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.219+0000 7f7502000640 1 -- 172.21.15.28:0/2155777865 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f751004c070 con 0x559e0ec55030 2024-05-20T20:09:17.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.221+0000 7f752220a740 10 client.26261 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:17.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.221+0000 7f752220a740 10 client.26261 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:17.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.221+0000 7f752220a740 20 client.26261 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:17.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.221+0000 7f752220a740 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0ec64b00 0x559e0ec66f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.221+0000 7f752220a740 4 client.26261 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:17.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.221+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559e0ebbcae0 con 0x559e0ec64b00 2024-05-20T20:09:17.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.222+0000 7f751ca00640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0ec64b00 0x559e0ec66f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.223+0000 7f751ca00640 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0ec64b00 0x559e0ec66f40 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=eded588a3aedfc14 server_cookie=bff74fb40b4939c7 in_seq=0 out_seq=0 2024-05-20T20:09:17.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.223+0000 7f7502000640 10 client.26261 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:17.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.224+0000 7f7502000640 1 -- 172.21.15.28:0/2155777865 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x559e0ebbcae0 con 0x559e0ec64b00 2024-05-20T20:09:17.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.224+0000 7f7502000640 10 client.26261 handle_command_reply: tid=0 2024-05-20T20:09:17.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.225+0000 7f752220a740 1 client.26261 shutdown 2024-05-20T20:09:17.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.227+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 msgr2=0x559e0ec521d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.227+0000 7f752220a740 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec521d0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f751002af30 tx=0x7f7510005d80 comp rx=0 tx=0).stop 2024-05-20T20:09:17.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.228+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 shutdown_connections 2024-05-20T20:09:17.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.228+0000 7f752220a740 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0ec64b00 0x559e0ec66f40 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.228+0000 7f752220a740 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec521d0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.228+0000 7f752220a740 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec4a770 0x559e0ec51c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.228+0000 7f752220a740 1 --2- 172.21.15.28:0/2155777865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec49e50 0x559e0ec54980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.228+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 >> 172.21.15.28:0/2155777865 conn(0x559e0ec46360 msgr2=0x559e0ec4c0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:17.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.228+0000 7f7502000640 0 client.26261 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:17.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.230+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 shutdown_connections 2024-05-20T20:09:17.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.230+0000 7f752220a740 1 -- 172.21.15.28:0/2155777865 wait complete. 2024-05-20T20:09:17.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.231+0000 7f752220a740 20 client.26261 trim_cache size 0 max 16384 2024-05-20T20:09:17.268 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.267+0000 7f752220a740 1 Processor -- start 2024-05-20T20:09:17.268 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.267+0000 7f752220a740 1 -- start start 2024-05-20T20:09:17.270 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f752220a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec60d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f752220a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 0x559e0ec45c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f752220a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec6c810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f752220a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f74f800e7a0 con 0x559e0ec4a770 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f752220a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f74f80102e0 con 0x559e0ec55030 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f752220a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f74f800ee70 con 0x559e0ec49e50 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f751ca00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec60d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f751ca00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec60d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60580/0 (socket says 172.21.15.28:60580) 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.268+0000 7f751ca00640 1 -- 172.21.15.28:0/1520832081 learned_addr learned my addr 172.21.15.28:0/1520832081 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.269+0000 7f751d400640 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec6c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.269+0000 7f751ca00640 1 -- 172.21.15.28:0/1520832081 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 msgr2=0x559e0ec6c810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.269+0000 7f751ca00640 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec6c810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.269+0000 7f751ca00640 1 -- 172.21.15.28:0/1520832081 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 msgr2=0x559e0ec45c40 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:17.271 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.270+0000 7f751ca00640 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 0x559e0ec45c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.270+0000 7f751ca00640 1 -- 172.21.15.28:0/1520832081 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x559e0ec51c00 con 0x559e0ec49e50 2024-05-20T20:09:17.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.271+0000 7f751ca00640 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec60d30 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f75080e20c0 tx=0x7f7508000bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=784d0c44338e5cf2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:17.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.271+0000 7f750d200640 1 -- 172.21.15.28:0/1520832081 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f75080c3070 con 0x559e0ec49e50 2024-05-20T20:09:17.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.271+0000 7f750d200640 1 -- 172.21.15.28:0/1520832081 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f75080a93e0 con 0x559e0ec49e50 2024-05-20T20:09:17.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.271+0000 7f750d200640 1 -- 172.21.15.28:0/1520832081 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7508014030 con 0x559e0ec49e50 2024-05-20T20:09:17.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 -- 172.21.15.28:0/1520832081 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 msgr2=0x559e0ec60d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec60d30 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f75080e20c0 tx=0x7f7508000bc0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 -- 172.21.15.28:0/1520832081 shutdown_connections 2024-05-20T20:09:17.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec55030 0x559e0ec6c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec4a770 0x559e0ec45c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 --2- 172.21.15.28:0/1520832081 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec60d30 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 -- 172.21.15.28:0/1520832081 >> 172.21.15.28:0/1520832081 conn(0x559e0ec46360 msgr2=0x559e0ec60620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:17.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.274+0000 7f752220a740 1 -- 172.21.15.28:0/1520832081 shutdown_connections 2024-05-20T20:09:17.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.275+0000 7f752220a740 1 -- 172.21.15.28:0/1520832081 wait complete. 2024-05-20T20:09:17.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.275+0000 7f752220a740 1 -- start start 2024-05-20T20:09:17.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.276+0000 7f752220a740 1 Processor -- start 2024-05-20T20:09:17.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.279+0000 7f752220a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec5d710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.279+0000 7f752220a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec4a770 0x559e0ec5aaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.279+0000 7f752220a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec55030 0x559e0ec5b020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f7517e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec4a770 0x559e0ec5aaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f752220a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f74f800fc10 con 0x559e0ec55030 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f752220a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f74f800f540 con 0x559e0ec4a770 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f752220a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f74f800c780 con 0x559e0ec49e50 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f752220a740 10 client.0 _ll_register_callbacks cb 0x7f7510009b20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f751ca00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec5d710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f751ca00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec5d710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60584/0 (socket says 172.21.15.28:60584) 2024-05-20T20:09:17.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.282+0000 7f751ca00640 1 -- 172.21.15.28:0/74981411 learned_addr learned my addr 172.21.15.28:0/74981411 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:17.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.283+0000 7f7517e00640 1 -- 172.21.15.28:0/74981411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 msgr2=0x559e0ec5d710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:17.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.283+0000 7f7517e00640 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec5d710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.283+0000 7f7517e00640 1 -- 172.21.15.28:0/74981411 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec55030 msgr2=0x559e0ec5b020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:17.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.283+0000 7f7517e00640 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec55030 0x559e0ec5b020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.283+0000 7f7517e00640 1 -- 172.21.15.28:0/74981411 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559e0ec5b850 con 0x559e0ec4a770 2024-05-20T20:09:17.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.283+0000 7f751ca00640 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec5d710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:17.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.284+0000 7f7517e00640 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec4a770 0x559e0ec5aaa0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x559e0ec514f0 tx=0x7f7504010bb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5a5f4a156275ce03 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:17.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.284+0000 7f7502000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:17.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.286+0000 7f7502000640 1 -- 172.21.15.28:0/74981411 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7504027020 con 0x559e0ec4a770 2024-05-20T20:09:17.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.286+0000 7f7502000640 1 -- 172.21.15.28:0/74981411 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7504005040 con 0x559e0ec4a770 2024-05-20T20:09:17.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.286+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f75080a93e0 con 0x559e0ec4a770 2024-05-20T20:09:17.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.287+0000 7f7502000640 1 -- 172.21.15.28:0/74981411 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f750401d040 con 0x559e0ec4a770 2024-05-20T20:09:17.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.287+0000 7f752220a740 10 client.26267 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:17.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.287+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559e0eccf7a0 con 0x559e0ec4a770 2024-05-20T20:09:17.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.288+0000 7f7502000640 1 -- 172.21.15.28:0/74981411 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f75080a93e0 con 0x559e0ec4a770 2024-05-20T20:09:17.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.288+0000 7f7502000640 1 -- 172.21.15.28:0/74981411 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7504018440 con 0x559e0ec4a770 2024-05-20T20:09:17.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.290+0000 7f752220a740 10 client.26267 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:17.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.291+0000 7f752220a740 10 client.26267 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:17.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.291+0000 7f752220a740 20 client.26267 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:17.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.291+0000 7f752220a740 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0eccfbe0 0x559e0ecd1fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:17.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.291+0000 7f752220a740 4 client.26267 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:09:17.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.291+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x559e0ebb62c0 con 0x559e0eccfbe0 2024-05-20T20:09:17.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.291+0000 7f751ca00640 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0eccfbe0 0x559e0ecd1fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:17.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.292+0000 7f751ca00640 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0eccfbe0 0x559e0ecd1fd0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f90eb4e704d65161 server_cookie=2022fbfcb10fb0c0 in_seq=0 out_seq=0 2024-05-20T20:09:17.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.292+0000 7f7502000640 10 client.26267 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:17.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:16 smithi028 ceph-mon[31178]: pgmap v112: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 283 MiB/s wr, 1.43k op/s 2024-05-20T20:09:17.689 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f7502000640 1 -- 172.21.15.28:0/74981411 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x559e0ebb62c0 con 0x559e0eccfbe0 2024-05-20T20:09:17.689 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f7502000640 10 client.26267 handle_command_reply: tid=0 2024-05-20T20:09:17.689 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 client.26267 shutdown 2024-05-20T20:09:17.689 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec4a770 msgr2=0x559e0ec5aaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.689 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec4a770 0x559e0ec5aaa0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x559e0ec514f0 tx=0x7f7504010bb0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.690 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 shutdown_connections 2024-05-20T20:09:17.690 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559e0eccfbe0 0x559e0ecd1fd0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.690 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x559e0ec4a770 0x559e0ec5aaa0 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.690 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x559e0ec55030 0x559e0ec5b020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.690 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 --2- 172.21.15.28:0/74981411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x559e0ec49e50 0x559e0ec5d710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.690 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.688+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 >> 172.21.15.28:0/74981411 conn(0x559e0ec46360 msgr2=0x559e0ec45bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:17.690 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.689+0000 7f7502000640 0 client.26267 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:17.691 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.689+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 shutdown_connections 2024-05-20T20:09:17.691 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.690+0000 7f752220a740 1 -- 172.21.15.28:0/74981411 wait complete. 2024-05-20T20:09:17.691 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.690+0000 7f752220a740 20 client.26267 trim_cache size 0 max 16384 2024-05-20T20:09:17.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:09:17.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:09:17.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:09:17.693 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.692+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74fc0b2b20 msgr2=0x7f74fc0b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.692+0000 7f751de00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74fc0b2b20 0x7f74fc0b4fe0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f751002ae60 tx=0x7f7510005df0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.692+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518103a10 msgr2=0x7f751810baf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:17.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.692+0000 7f751de00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518103a10 0x7f751810baf0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f750800d950 tx=0x7f750800de10 comp rx=0 tx=0).stop 2024-05-20T20:09:17.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 shutdown_connections 2024-05-20T20:09:17.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7518104330 0x7f751810c030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f74fc0b2b20 0x7f74fc0b4fe0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f751810ce30 0x7f75181a3e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 --2- 172.21.15.28:0/2307530152 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7518103a10 0x7f751810baf0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:17.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 >> 172.21.15.28:0/2307530152 conn(0x7f75180fd270 msgr2=0x7f7518101590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:17.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 shutdown_connections 2024-05-20T20:09:17.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:17.693+0000 7f751de00640 1 -- 172.21.15.28:0/2307530152 wait complete. 2024-05-20T20:09:17.926 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:09:17.927 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:09:18.790 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:18 smithi154 ceph-mon[36314]: pgmap v113: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 258 MiB/s wr, 1.39k op/s 2024-05-20T20:09:19.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:18 smithi033 ceph-mon[36179]: pgmap v113: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 258 MiB/s wr, 1.39k op/s 2024-05-20T20:09:19.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:18 smithi028 ceph-mon[31178]: pgmap v113: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 258 MiB/s wr, 1.39k op/s 2024-05-20T20:09:21.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:20 smithi154 ceph-mon[36314]: pgmap v114: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 369 MiB/s wr, 2.56k op/s 2024-05-20T20:09:21.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:20 smithi033 ceph-mon[36179]: pgmap v114: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 369 MiB/s wr, 2.56k op/s 2024-05-20T20:09:21.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:20 smithi028 ceph-mon[31178]: pgmap v114: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 369 MiB/s wr, 2.56k op/s 2024-05-20T20:09:23.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:22 smithi154 ceph-mon[36314]: pgmap v115: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 235 MiB/s wr, 2.28k op/s 2024-05-20T20:09:23.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:22 smithi033 ceph-mon[36179]: pgmap v115: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 235 MiB/s wr, 2.28k op/s 2024-05-20T20:09:23.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:22 smithi028 ceph-mon[31178]: pgmap v115: 97 pgs: 97 active+clean; 10 GiB data, 2.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 235 MiB/s wr, 2.28k op/s 2024-05-20T20:09:24.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 -- 172.21.15.28:0/1234291974 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb29810b780 msgr2=0x7fb29810bb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 --2- 172.21.15.28:0/1234291974 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb29810b780 0x7fb29810bb60 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fb28c009990 tx=0x7fb28c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 -- 172.21.15.28:0/1234291974 shutdown_connections 2024-05-20T20:09:24.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 --2- 172.21.15.28:0/1234291974 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb298106140 0x7fb298112650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 --2- 172.21.15.28:0/1234291974 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb29810b780 0x7fb29810bb60 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 --2- 172.21.15.28:0/1234291974 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb298105780 0x7fb298105c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 -- 172.21.15.28:0/1234291974 >> 172.21.15.28:0/1234291974 conn(0x7fb298101270 msgr2=0x7fb298103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:24.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 -- 172.21.15.28:0/1234291974 shutdown_connections 2024-05-20T20:09:24.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.857+0000 7fb2a0600700 1 -- 172.21.15.28:0/1234291974 wait complete. 2024-05-20T20:09:24.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.858+0000 7fb2a0600700 1 Processor -- start 2024-05-20T20:09:24.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.858+0000 7fb2a0600700 1 -- start start 2024-05-20T20:09:24.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb2a0600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb298105780 0x7fb2981124f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb2a0600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb298106140 0x7fb29810f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb2a0600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb29810b780 0x7fb29810fb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb2a0600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb298115a10 con 0x7fb298105780 2024-05-20T20:09:24.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb2a0600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb298115890 con 0x7fb29810b780 2024-05-20T20:09:24.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb2a0600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb298115b90 con 0x7fb298106140 2024-05-20T20:09:24.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb29dc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb298105780 0x7fb2981124f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.859+0000 7fb29d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb298106140 0x7fb29810f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb298106140 0x7fb29810f5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60608/0 (socket says 172.21.15.28:60608) 2024-05-20T20:09:24.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29d200700 1 -- 172.21.15.28:0/399008765 learned_addr learned my addr 172.21.15.28:0/399008765 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29e600700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb29810b780 0x7fb29810fb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29d200700 1 -- 172.21.15.28:0/399008765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb29810b780 msgr2=0x7fb29810fb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29d200700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb29810b780 0x7fb29810fb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29d200700 1 -- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb298105780 msgr2=0x7fb2981124f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29d200700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb298105780 0x7fb2981124f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29d200700 1 -- 172.21.15.28:0/399008765 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb2981ac750 con 0x7fb298106140 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.860+0000 7fb29dc00700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb298105780 0x7fb2981124f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:24.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.861+0000 7fb29d200700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb298106140 0x7fb29810f5f0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fb28400eb30 tx=0x7fb28400ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:24.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.861+0000 7fb296000700 1 -- 172.21.15.28:0/399008765 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb28400cbd0 con 0x7fb298106140 2024-05-20T20:09:24.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.861+0000 7fb296000700 1 -- 172.21.15.28:0/399008765 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb2840044d0 con 0x7fb298106140 2024-05-20T20:09:24.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.861+0000 7fb296000700 1 -- 172.21.15.28:0/399008765 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb284003e60 con 0x7fb298106140 2024-05-20T20:09:24.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.861+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb2981ac920 con 0x7fb298106140 2024-05-20T20:09:24.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.861+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb2981acd20 con 0x7fb298106140 2024-05-20T20:09:24.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.866+0000 7fb296000700 1 -- 172.21.15.28:0/399008765 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb284004000 con 0x7fb298106140 2024-05-20T20:09:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.867+0000 7fb2ab021b80 1 Processor -- start 2024-05-20T20:09:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.867+0000 7fb2ab021b80 1 -- start start 2024-05-20T20:09:24.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0d7a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0d1ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0d2640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5627aa02fec0 con 0x5627aa0d21c0 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5627aa01fa60 con 0x5627aa0d7690 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5627aa023aa0 con 0x5627aa0d7fb0 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb29e600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0d7a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb29e600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0d7a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37676/0 (socket says 172.21.15.28:37676) 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb29e600700 1 -- 172.21.15.28:0/4002546341 learned_addr learned my addr 172.21.15.28:0/4002546341 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.868+0000 7fb29dc00700 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0d1ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.869+0000 7fb29e600700 1 -- 172.21.15.28:0/4002546341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 msgr2=0x5627aa0d1ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.869+0000 7fb29e600700 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0d1ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.869+0000 7fb29e600700 1 -- 172.21.15.28:0/4002546341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 msgr2=0x5627aa0d2640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.871+0000 7fb29d200700 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0d2640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.871+0000 7fb29e600700 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0d2640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.871+0000 7fb29e600700 1 -- 172.21.15.28:0/4002546341 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5627a9fe84a0 con 0x5627aa0d7690 2024-05-20T20:09:24.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.871+0000 7fb29d200700 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0d2640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.871+0000 7fb296000700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb2800b2a70 0x7fb2800b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.872+0000 7fb29e600700 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0d7a70 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7fb29000a4b0 tx=0x7fb290009650 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b54cdf80beb3870 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.872+0000 7fb296000700 1 -- 172.21.15.28:0/399008765 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb2840e28b0 con 0x7fb298106140 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.872+0000 7fb29dc00700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb2800b2a70 0x7fb2800b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.872+0000 7fb28aa00700 1 -- 172.21.15.28:0/4002546341 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb290014420 con 0x5627aa0d7690 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.872+0000 7fb28aa00700 1 -- 172.21.15.28:0/4002546341 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb290019070 con 0x5627aa0d7690 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.872+0000 7fb28aa00700 1 -- 172.21.15.28:0/4002546341 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2900138c0 con 0x5627aa0d7690 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.873+0000 7fb2ab021b80 1 -- 172.21.15.28:0/4002546341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 msgr2=0x5627aa0d7a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.873+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0d7a70 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7fb29000a4b0 tx=0x7fb290009650 comp rx=0 tx=0).stop 2024-05-20T20:09:24.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.874+0000 7fb2ab021b80 1 -- 172.21.15.28:0/4002546341 shutdown_connections 2024-05-20T20:09:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.874+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0d2640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.874+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0d7a70 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.874+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/4002546341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0d1ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.874+0000 7fb29dc00700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb2800b2a70 0x7fb2800b4f30 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fb28c006010 tx=0x7fb28c00b280 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:24.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.874+0000 7fb2ab021b80 1 -- 172.21.15.28:0/4002546341 >> 172.21.15.28:0/4002546341 conn(0x5627a9fffc60 msgr2=0x5627aa0cc710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:24.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.877+0000 7fb2ab021b80 1 -- 172.21.15.28:0/4002546341 shutdown_connections 2024-05-20T20:09:24.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.877+0000 7fb2ab021b80 1 -- 172.21.15.28:0/4002546341 wait complete. 2024-05-20T20:09:24.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.878+0000 7fb2ab021b80 1 -- start start 2024-05-20T20:09:24.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.878+0000 7fb2ab021b80 1 Processor -- start 2024-05-20T20:09:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.880+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0ddd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.880+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d7690 0x5627aa0db120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.880+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0db660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.880+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5627a9f26020 con 0x5627aa0d7690 2024-05-20T20:09:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.880+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5627a9fc07e0 con 0x5627aa0d21c0 2024-05-20T20:09:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.880+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5627aa0dea60 con 0x5627aa0d7fb0 2024-05-20T20:09:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.880+0000 7fb2ab021b80 10 client.0 _ll_register_callbacks cb 0x5627a9fc9cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0ddd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0ddd80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37682/0 (socket says 172.21.15.28:37682) 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 -- 172.21.15.28:0/1222852457 learned_addr learned my addr 172.21.15.28:0/1222852457 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 -- 172.21.15.28:0/1222852457 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 msgr2=0x5627aa0db660 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0db660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 -- 172.21.15.28:0/1222852457 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d7690 msgr2=0x5627aa0db120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d7690 0x5627aa0db120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.881+0000 7fb29dc00700 1 -- 172.21.15.28:0/1222852457 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627a9f6f950 con 0x5627aa0d21c0 2024-05-20T20:09:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.882+0000 7fb29dc00700 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0ddd80 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7fb28c005620 tx=0x7fb28c009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c7e49e594520ae41 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.882+0000 7fb27f400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.882+0000 7fb27f400700 1 -- 172.21.15.28:0/1222852457 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb28c034060 con 0x5627aa0d21c0 2024-05-20T20:09:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.883+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb290019070 con 0x5627aa0d21c0 2024-05-20T20:09:24.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.883+0000 7fb27f400700 1 -- 172.21.15.28:0/1222852457 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb28c00f460 con 0x5627aa0d21c0 2024-05-20T20:09:24.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.890+0000 7fb27f400700 1 -- 172.21.15.28:0/1222852457 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb28c00f660 con 0x5627aa0d21c0 2024-05-20T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:24 smithi028 ceph-mon[31178]: pgmap v116: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 300 MiB/s wr, 2.89k op/s 2024-05-20T20:09:24.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.893+0000 7fb2ab021b80 10 client.26276 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:24.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.893+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5627a9ff00e0 con 0x5627aa0d21c0 2024-05-20T20:09:24.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.894+0000 7fb27f400700 1 -- 172.21.15.28:0/1222852457 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb290019070 con 0x5627aa0d21c0 2024-05-20T20:09:24.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.894+0000 7fb27f400700 1 -- 172.21.15.28:0/1222852457 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb28c0258b0 con 0x5627aa0d21c0 2024-05-20T20:09:24.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.895+0000 7fb2ab021b80 10 client.26276 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:24.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.895+0000 7fb2ab021b80 10 client.26276 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:24.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.895+0000 7fb2ab021b80 20 client.26276 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:24.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.895+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa0eb540 0x5627aa0ed9b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.895+0000 7fb2ab021b80 4 client.26276 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:24.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.895+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5627aa031e10 con 0x5627aa0eb540 2024-05-20T20:09:24.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.895+0000 7fb29d200700 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa0eb540 0x5627aa0ed9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.896+0000 7fb29d200700 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa0eb540 0x5627aa0ed9b0 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=61086762d956f5bf server_cookie=c974c642fd38f780 in_seq=0 out_seq=0 2024-05-20T20:09:24.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.896+0000 7fb27f400700 10 client.26276 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.899+0000 7fb27f400700 1 -- 172.21.15.28:0/1222852457 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5627aa031e10 con 0x5627aa0eb540 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.899+0000 7fb27f400700 10 client.26276 handle_command_reply: tid=0 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.899+0000 7fb2ab021b80 1 client.26276 shutdown 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.899+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 msgr2=0x5627aa0ddd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.899+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0ddd80 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7fb28c005620 tx=0x7fb28c009310 comp rx=0 tx=0).stop 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 shutdown_connections 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa0eb540 0x5627aa0ed9b0 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0db660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0ddd80 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1222852457 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d7690 0x5627aa0db120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 >> 172.21.15.28:0/1222852457 conn(0x5627a9fffc60 msgr2=0x5627aa0cbdf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:24.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb27f400700 0 client.26276 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:24.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 shutdown_connections 2024-05-20T20:09:24.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.900+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1222852457 wait complete. 2024-05-20T20:09:24.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.901+0000 7fb2ab021b80 20 client.26276 trim_cache size 0 max 16384 2024-05-20T20:09:24.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.944+0000 7fb2ab021b80 1 Processor -- start 2024-05-20T20:09:24.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.944+0000 7fb2ab021b80 1 -- start start 2024-05-20T20:09:24.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.944+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0cc570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.944+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0da120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.945+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0f3280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.945+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb2981b4470 con 0x5627aa0d21c0 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.945+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb2981b5fb0 con 0x5627aa0d7690 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.945+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb2981b4b40 con 0x5627aa0d7fb0 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.945+0000 7fb29d200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0da120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.945+0000 7fb29d200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0da120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37686/0 (socket says 172.21.15.28:37686) 2024-05-20T20:09:24.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.945+0000 7fb29d200700 1 -- 172.21.15.28:0/408018683 learned_addr learned my addr 172.21.15.28:0/408018683 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:24.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.946+0000 7fb29d200700 1 -- 172.21.15.28:0/408018683 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 msgr2=0x5627aa0f3280 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:09:24.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.946+0000 7fb29d200700 1 --2- 172.21.15.28:0/408018683 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0f3280 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.946+0000 7fb29d200700 1 -- 172.21.15.28:0/408018683 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 msgr2=0x5627aa0cc570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:24.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.946+0000 7fb29d200700 1 --2- 172.21.15.28:0/408018683 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0cc570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.946+0000 7fb29d200700 1 -- 172.21.15.28:0/408018683 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5627aa0cd520 con 0x5627aa0d7690 2024-05-20T20:09:24.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.946+0000 7fb29d200700 1 --2- 172.21.15.28:0/408018683 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0da120 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fb28400c5b0 tx=0x7fb284003800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8439d1c0cf0e7651 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:24.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.947+0000 7fb28a000700 1 -- 172.21.15.28:0/408018683 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2840a9650 con 0x5627aa0d7690 2024-05-20T20:09:24.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.947+0000 7fb28a000700 1 -- 172.21.15.28:0/408018683 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb2840042f0 con 0x5627aa0d7690 2024-05-20T20:09:24.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.947+0000 7fb28a000700 1 -- 172.21.15.28:0/408018683 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2840b29a0 con 0x5627aa0d7690 2024-05-20T20:09:24.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.948+0000 7fb2ab021b80 1 -- 172.21.15.28:0/408018683 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 msgr2=0x5627aa0da120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.948+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/408018683 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0da120 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fb28400c5b0 tx=0x7fb284003800 comp rx=0 tx=0).stop 2024-05-20T20:09:24.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.948+0000 7fb2ab021b80 1 -- 172.21.15.28:0/408018683 shutdown_connections 2024-05-20T20:09:24.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.948+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/408018683 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0f3280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.948+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/408018683 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0d21c0 0x5627aa0cc570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.948+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/408018683 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d7690 0x5627aa0da120 secure :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7fb28400c5b0 tx=0x7fb284003800 comp rx=0 tx=0).stop 2024-05-20T20:09:24.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.948+0000 7fb2ab021b80 1 -- 172.21.15.28:0/408018683 >> 172.21.15.28:0/408018683 conn(0x5627a9fffc60 msgr2=0x5627aa0d9a10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:24.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.949+0000 7fb2ab021b80 1 -- 172.21.15.28:0/408018683 shutdown_connections 2024-05-20T20:09:24.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.949+0000 7fb2ab021b80 1 -- 172.21.15.28:0/408018683 wait complete. 2024-05-20T20:09:24.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.949+0000 7fb2ab021b80 1 -- start start 2024-05-20T20:09:24.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.950+0000 7fb2ab021b80 1 Processor -- start 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.950+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0e0cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.950+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0e1200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.950+0000 7fb2ab021b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0e46c0 0x5627aa156160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.950+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb2981b58e0 con 0x5627aa0e46c0 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.950+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb2981b5210 con 0x5627aa0d21c0 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.950+0000 7fb2ab021b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb2981b2410 con 0x5627aa0d7fb0 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0e1200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29dc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0e0cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0e1200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60628/0 (socket says 172.21.15.28:60628) 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 -- 172.21.15.28:0/1972673077 learned_addr learned my addr 172.21.15.28:0/1972673077 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29e600700 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0e46c0 0x5627aa156160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 -- 172.21.15.28:0/1972673077 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 msgr2=0x5627aa0e0cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0e0cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 -- 172.21.15.28:0/1972673077 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0e46c0 msgr2=0x5627aa156160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0e46c0 0x5627aa156160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.951+0000 7fb29d200700 1 -- 172.21.15.28:0/1972673077 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627aa0e1ad0 con 0x5627aa0d7fb0 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.952+0000 7fb29d200700 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0e1200 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fb284000c00 tx=0x7fb2840b23a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8fd577274fb00700 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.952+0000 7fb27f400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.952+0000 7fb27f400700 1 -- 172.21.15.28:0/1972673077 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2840a9650 con 0x5627aa0d7fb0 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.952+0000 7fb27f400700 1 -- 172.21.15.28:0/1972673077 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb2840a9da0 con 0x5627aa0d7fb0 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.952+0000 7fb27f400700 10 client.0 inactive, discarding osd_map(81..81 src has 1..81) 2024-05-20T20:09:24.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.953+0000 7fb2ab021b80 10 client.0 _ll_register_callbacks cb 0x5627a9fc9cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:24.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.953+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb2840042f0 con 0x5627aa0d7fb0 2024-05-20T20:09:24.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.955+0000 7fb27f400700 1 -- 172.21.15.28:0/1972673077 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb2840e0440 con 0x5627aa0d7fb0 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.955+0000 7fb2ab021b80 10 client.26212 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.955+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x5627aa1575c0 con 0x5627aa0d7fb0 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.956+0000 7fb27f400700 1 -- 172.21.15.28:0/1972673077 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb2840b2510 con 0x5627aa0d7fb0 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.956+0000 7fb2ab021b80 10 client.26212 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.956+0000 7fb2ab021b80 10 client.26212 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.956+0000 7fb2ab021b80 20 client.26212 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.956+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa157ad0 0x5627aa159ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.956+0000 7fb2ab021b80 4 client.26212 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:09:24.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.956+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x5627aa02b5f0 con 0x5627aa157ad0 2024-05-20T20:09:24.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.957+0000 7fb29dc00700 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa157ad0 0x5627aa159ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:24.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.963+0000 7fb29dc00700 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa157ad0 0x5627aa159ef0 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=83707c718e345fad server_cookie=86267167f3275536 in_seq=0 out_seq=0 2024-05-20T20:09:24.965 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.963+0000 7fb27f400700 10 client.26212 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:24.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.965+0000 7fb27f400700 1 -- 172.21.15.28:0/1972673077 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x5627aa02b5f0 con 0x5627aa157ad0 2024-05-20T20:09:24.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.965+0000 7fb27f400700 10 client.26212 handle_command_reply: tid=0 2024-05-20T20:09:24.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.966+0000 7fb2ab021b80 1 client.26212 shutdown 2024-05-20T20:09:24.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.967+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 msgr2=0x5627aa0e1200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.967+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0e1200 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fb284000c00 tx=0x7fb2840b23a0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.968+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 shutdown_connections 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.968+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627aa157ad0 0x5627aa159ef0 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.968+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627aa0e46c0 0x5627aa156160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.968+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627aa0d21c0 0x5627aa0e0cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.968+0000 7fb2ab021b80 1 --2- 172.21.15.28:0/1972673077 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627aa0d7fb0 0x5627aa0e1200 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.968+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 >> 172.21.15.28:0/1972673077 conn(0x5627a9fffc60 msgr2=0x5627aa0da2a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.971+0000 7fb27f400700 0 client.26212 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:24.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.971+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 shutdown_connections 2024-05-20T20:09:24.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.971+0000 7fb2ab021b80 1 -- 172.21.15.28:0/1972673077 wait complete. 2024-05-20T20:09:24.977 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:09:24.977 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:09:24.977 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "a4ece64d-f8b8-447d-99a5-0dfe52cac045", 2024-05-20T20:09:24.977 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:09:24.977 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:09:24.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.973+0000 7fb2ab021b80 20 client.26212 trim_cache size 0 max 16384 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.976+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb2800b2a70 msgr2=0x7fb2800b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.976+0000 7fb2a0600700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb2800b2a70 0x7fb2800b4f30 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fb28c006010 tx=0x7fb28c00b280 comp rx=0 tx=0).stop 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.976+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb298106140 msgr2=0x7fb29810f5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.976+0000 7fb2a0600700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb298106140 0x7fb29810f5f0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fb28400eb30 tx=0x7fb28400ee40 comp rx=0 tx=0).stop 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.979+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 shutdown_connections 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.979+0000 7fb2a0600700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb2800b2a70 0x7fb2800b4f30 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.979+0000 7fb2a0600700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb29810b780 0x7fb29810fb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.979+0000 7fb2a0600700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb298105780 0x7fb2981124f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.979+0000 7fb2a0600700 1 --2- 172.21.15.28:0/399008765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb298106140 0x7fb29810f5f0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:24.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.979+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 >> 172.21.15.28:0/399008765 conn(0x7fb298101270 msgr2=0x7fb298101a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:24.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.980+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 shutdown_connections 2024-05-20T20:09:24.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:24.980+0000 7fb2a0600700 1 -- 172.21.15.28:0/399008765 wait complete. 2024-05-20T20:09:25.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:24 smithi154 ceph-mon[36314]: pgmap v116: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 300 MiB/s wr, 2.89k op/s 2024-05-20T20:09:25.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:24 smithi033 ceph-mon[36179]: pgmap v116: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 300 MiB/s wr, 2.89k op/s 2024-05-20T20:09:26.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:25 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:09:26.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:25 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:09:26.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:25 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:09:26.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:25 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:09:26.278 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:25 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:09:26.278 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:25 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:09:26.278 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:25 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:09:26.278 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:25 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:09:26.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:25 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:09:26.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:25 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:09:26.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:25 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:09:26.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:25 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:09:26.487 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:09:27.266 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:27.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:26 smithi028 ceph-mon[31178]: pgmap v117: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 218 MiB/s wr, 2.18k op/s 2024-05-20T20:09:27.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:26 smithi154 ceph-mon[36314]: pgmap v117: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 218 MiB/s wr, 2.18k op/s 2024-05-20T20:09:27.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:26 smithi033 ceph-mon[36179]: pgmap v117: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 218 MiB/s wr, 2.18k op/s 2024-05-20T20:09:28.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:27 smithi028 ceph-mon[31178]: pgmap v118: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 193 MiB/s wr, 1.95k op/s 2024-05-20T20:09:28.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:27 smithi154 ceph-mon[36314]: pgmap v118: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 193 MiB/s wr, 1.95k op/s 2024-05-20T20:09:28.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:27 smithi033 ceph-mon[36179]: pgmap v118: 97 pgs: 97 active+clean; 11 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 193 MiB/s wr, 1.95k op/s 2024-05-20T20:09:29.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:28 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:09:29.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:28 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:09:29.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:28 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:09:30.309 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:29 smithi028 ceph-mon[31178]: pgmap v119: 97 pgs: 97 active+clean; 12 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 308 MiB/s wr, 2.71k op/s 2024-05-20T20:09:30.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:29 smithi154 ceph-mon[36314]: pgmap v119: 97 pgs: 97 active+clean; 12 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 308 MiB/s wr, 2.71k op/s 2024-05-20T20:09:30.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:29 smithi033 ceph-mon[36179]: pgmap v119: 97 pgs: 97 active+clean; 12 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 308 MiB/s wr, 2.71k op/s 2024-05-20T20:09:30.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.945+0000 7f123ee00640 1 -- 172.21.15.28:0/350980446 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1238115ba0 msgr2=0x7f1238117f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:30.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.945+0000 7f123ee00640 1 --2- 172.21.15.28:0/350980446 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1238115ba0 0x7f1238117f90 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f1230009ca0 tx=0x7f123002a950 comp rx=0 tx=0).stop 2024-05-20T20:09:30.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.946+0000 7f123ee00640 1 -- 172.21.15.28:0/350980446 shutdown_connections 2024-05-20T20:09:30.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.946+0000 7f123ee00640 1 --2- 172.21.15.28:0/350980446 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1238115ba0 0x7f1238117f90 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.946+0000 7f123ee00640 1 --2- 172.21.15.28:0/350980446 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f123810a370 0x7f1238115660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.946+0000 7f123ee00640 1 --2- 172.21.15.28:0/350980446 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238109a50 0x7f1238109e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.946+0000 7f123ee00640 1 -- 172.21.15.28:0/350980446 >> 172.21.15.28:0/350980446 conn(0x7f123806db20 msgr2=0x7f123806df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:30.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.946+0000 7f123ee00640 1 -- 172.21.15.28:0/350980446 shutdown_connections 2024-05-20T20:09:30.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.947+0000 7f123ee00640 1 -- 172.21.15.28:0/350980446 wait complete. 2024-05-20T20:09:30.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.952+0000 7f123ee00640 1 Processor -- start 2024-05-20T20:09:30.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.953+0000 7f123ee00640 1 -- start start 2024-05-20T20:09:30.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.958+0000 7f123ee00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1238109a50 0x7f12381a7600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.958+0000 7f123ee00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f123810a370 0x7f12381a7b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.958+0000 7f123ee00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238115ba0 0x7f12381a16c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.958+0000 7f123ee00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f123811b4e0 con 0x7f123810a370 2024-05-20T20:09:30.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.958+0000 7f123ee00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f123811b360 con 0x7f1238115ba0 2024-05-20T20:09:30.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.958+0000 7f123ee00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f123811b660 con 0x7f1238109a50 2024-05-20T20:09:30.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.959+0000 7f123ce00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238115ba0 0x7f12381a16c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.959+0000 7f123ce00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238115ba0 0x7f12381a16c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52152/0 (socket says 172.21.15.28:52152) 2024-05-20T20:09:30.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.959+0000 7f123ce00640 1 -- 172.21.15.28:0/870830295 learned_addr learned my addr 172.21.15.28:0/870830295 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:30.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.959+0000 7f1237e00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1238109a50 0x7f12381a7600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.962+0000 7f1237400640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f123810a370 0x7f12381a7b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.962+0000 7f123ce00640 1 -- 172.21.15.28:0/870830295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1238109a50 msgr2=0x7f12381a7600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:30.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.962+0000 7f123ce00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1238109a50 0x7f12381a7600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.962+0000 7f123ce00640 1 -- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f123810a370 msgr2=0x7f12381a7b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:30.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.963+0000 7f123ce00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f123810a370 0x7f12381a7b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.963+0000 7f123ce00640 1 -- 172.21.15.28:0/870830295 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f12381a1f00 con 0x7f1238115ba0 2024-05-20T20:09:30.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.963+0000 7f123ce00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238115ba0 0x7f12381a16c0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f1230006010 tx=0x7f1230003f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:30.966 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.966+0000 7f1237e00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1238109a50 0x7f12381a7600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:30.967 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.966+0000 7f1237400640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f123810a370 0x7f12381a7b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:30.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.969+0000 7f1234c00640 1 -- 172.21.15.28:0/870830295 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1230034040 con 0x7f1238115ba0 2024-05-20T20:09:30.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.969+0000 7f1234c00640 1 -- 172.21.15.28:0/870830295 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1230030850 con 0x7f1238115ba0 2024-05-20T20:09:30.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.969+0000 7f1234c00640 1 -- 172.21.15.28:0/870830295 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1230004640 con 0x7f1238115ba0 2024-05-20T20:09:30.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.969+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f12381a2190 con 0x7f1238115ba0 2024-05-20T20:09:30.971 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.969+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f123810b3f0 con 0x7f1238115ba0 2024-05-20T20:09:30.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.979+0000 7f1241c00740 1 Processor -- start 2024-05-20T20:09:30.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.979+0000 7f1234c00640 1 -- 172.21.15.28:0/870830295 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1230037020 con 0x7f1238115ba0 2024-05-20T20:09:30.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.982+0000 7f1234c00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f121c0b2b50 0x7f121c0b5010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.983+0000 7f1241c00740 1 -- start start 2024-05-20T20:09:30.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.983+0000 7f1241c00740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae53bd0 0x55a4eae53fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.983+0000 7f1241c00740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae67140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.983+0000 7f1241c00740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae69a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.983+0000 7f1241c00740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a4eadc4b90 con 0x55a4eae544f0 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.983+0000 7f1241c00740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a4eadb4730 con 0x55a4eae53bd0 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.984+0000 7f1241c00740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a4eadb8770 con 0x55a4eae67680 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.984+0000 7f1234c00640 1 -- 172.21.15.28:0/870830295 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f12300c6c80 con 0x7f1238115ba0 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.984+0000 7f123ce00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae69a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.984+0000 7f1237400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae67140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.984+0000 7f1237400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae67140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48462/0 (socket says 172.21.15.28:48462) 2024-05-20T20:09:30.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.984+0000 7f1237400640 1 -- 172.21.15.28:0/499077386 learned_addr learned my addr 172.21.15.28:0/499077386 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:30.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.985+0000 7f123ce00640 1 -- 172.21.15.28:0/499077386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae53bd0 msgr2=0x55a4eae53fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:30.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.986+0000 7f1237e00640 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae53bd0 0x55a4eae53fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.986+0000 7f1237400640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f121c0b2b50 0x7f121c0b5010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.987+0000 7f123ce00640 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae53bd0 0x55a4eae53fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.987+0000 7f123ce00640 1 -- 172.21.15.28:0/499077386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 msgr2=0x55a4eae67140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:30.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.987+0000 7f123ce00640 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae67140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.987+0000 7f123ce00640 1 -- 172.21.15.28:0/499077386 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a4ead4d8b0 con 0x55a4eae67680 2024-05-20T20:09:30.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.987+0000 7f1237e00640 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae53bd0 0x55a4eae53fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:30.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.988+0000 7f1237400640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f121c0b2b50 0x7f121c0b5010 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f1224009fd0 tx=0x7f12240097b0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:30.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.988+0000 7f123ce00640 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae69a70 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f1230100b00 tx=0x7f12300cfb60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1ebf175dac323c5d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:30.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.988+0000 7f122d600640 1 -- 172.21.15.28:0/499077386 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12300c6e80 con 0x55a4eae67680 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.988+0000 7f122d600640 1 -- 172.21.15.28:0/499077386 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f123003fec0 con 0x55a4eae67680 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.988+0000 7f122d600640 1 -- 172.21.15.28:0/499077386 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12300c7020 con 0x55a4eae67680 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.989+0000 7f1241c00740 1 -- 172.21.15.28:0/499077386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 msgr2=0x55a4eae69a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.989+0000 7f1241c00740 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae69a70 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f1230100b00 tx=0x7f12300cfb60 comp rx=0 tx=0).stop 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.990+0000 7f1241c00740 1 -- 172.21.15.28:0/499077386 shutdown_connections 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.990+0000 7f1241c00740 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae69a70 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.990+0000 7f1241c00740 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae67140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.990+0000 7f1241c00740 1 --2- 172.21.15.28:0/499077386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae53bd0 0x55a4eae53fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.990+0000 7f1241c00740 1 -- 172.21.15.28:0/499077386 >> 172.21.15.28:0/499077386 conn(0x55a4eae501a0 msgr2=0x55a4eae50580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:30.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.991+0000 7f1241c00740 1 -- 172.21.15.28:0/499077386 shutdown_connections 2024-05-20T20:09:30.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.991+0000 7f1241c00740 1 -- 172.21.15.28:0/499077386 wait complete. 2024-05-20T20:09:30.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.991+0000 7f1241c00740 1 -- start start 2024-05-20T20:09:30.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.992+0000 7f1241c00740 1 Processor -- start 2024-05-20T20:09:30.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.993+0000 7f1241c00740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae60870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.993+0000 7f1241c00740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae60db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.994+0000 7f1241c00740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae67680 0x55a4eae5bbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:30.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.994+0000 7f1237e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae60870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.994+0000 7f1237e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae60870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55826/0 (socket says 172.21.15.28:55826) 2024-05-20T20:09:30.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.994+0000 7f1237400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae60db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.994+0000 7f1237400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae60db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52166/0 (socket says 172.21.15.28:52166) 2024-05-20T20:09:30.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.995+0000 7f123ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae67680 0x55a4eae5bbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:30.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.995+0000 7f123ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae67680 0x55a4eae5bbd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48470/0 (socket says 172.21.15.28:48470) 2024-05-20T20:09:30.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.995+0000 7f1241c00740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a4eaa9a790 con 0x55a4eae67680 2024-05-20T20:09:30.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.995+0000 7f1241c00740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a4ead5cbc0 con 0x55a4eae544f0 2024-05-20T20:09:30.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.995+0000 7f1241c00740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a4eadb8770 con 0x55a4eae53bd0 2024-05-20T20:09:30.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.995+0000 7f1241c00740 10 client.0 _ll_register_callbacks cb 0x55a4ead3a400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:30.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.995+0000 7f1237e00640 1 -- 172.21.15.28:0/3286231165 learned_addr learned my addr 172.21.15.28:0/3286231165 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:30.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.996+0000 7f123ce00640 1 -- 172.21.15.28:0/3286231165 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 msgr2=0x55a4eae60870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:30.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.997+0000 7f123ce00640 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae60870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.997+0000 7f123ce00640 1 -- 172.21.15.28:0/3286231165 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 msgr2=0x55a4eae60db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:30.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.997+0000 7f123ce00640 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae60db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:30.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.997+0000 7f123ce00640 1 -- 172.21.15.28:0/3286231165 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a4eaac9210 con 0x55a4eae67680 2024-05-20T20:09:31.000 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.997+0000 7f123ce00640 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae67680 0x55a4eae5bbd0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f12300c7700 tx=0x7f12300c70b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a8232d6e1a6d2574 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.000 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.997+0000 7f1222000640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:31.000 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.998+0000 7f1222000640 1 -- 172.21.15.28:0/3286231165 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12300e3020 con 0x55a4eae67680 2024-05-20T20:09:31.000 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.999+0000 7f1222000640 1 -- 172.21.15.28:0/3286231165 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f1230009890 con 0x55a4eae67680 2024-05-20T20:09:31.000 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:30.999+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f123003fec0 con 0x55a4eae67680 2024-05-20T20:09:31.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.000+0000 7f1222000640 1 -- 172.21.15.28:0/3286231165 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f12300c71f0 con 0x55a4eae67680 2024-05-20T20:09:31.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.000+0000 7f1241c00740 10 client.16110 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:31.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.000+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a4eab95620 con 0x55a4eae67680 2024-05-20T20:09:31.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.001+0000 7f1222000640 1 -- 172.21.15.28:0/3286231165 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f123003fec0 con 0x55a4eae67680 2024-05-20T20:09:31.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.001+0000 7f1222000640 1 -- 172.21.15.28:0/3286231165 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f12300c6380 con 0x55a4eae67680 2024-05-20T20:09:31.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.001+0000 7f1241c00740 10 client.16110 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:31.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.001+0000 7f1241c00740 10 client.16110 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:31.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.002+0000 7f1241c00740 20 client.16110 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:31.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.002+0000 7f1241c00740 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae7f2d0 0x55a4eae816c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.002+0000 7f1241c00740 4 client.16110 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:31.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.002+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a4eadc6ae0 con 0x55a4eae7f2d0 2024-05-20T20:09:31.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.002+0000 7f1237e00640 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae7f2d0 0x55a4eae816c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.004+0000 7f1237e00640 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae7f2d0 0x55a4eae816c0 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5952a7849d9eddab server_cookie=5400334b624fd316 in_seq=0 out_seq=0 2024-05-20T20:09:31.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.004+0000 7f1222000640 10 client.16110 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.005+0000 7f1222000640 1 -- 172.21.15.28:0/3286231165 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a4eadc6ae0 con 0x55a4eae7f2d0 2024-05-20T20:09:31.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.005+0000 7f1222000640 10 client.16110 handle_command_reply: tid=0 2024-05-20T20:09:31.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.005+0000 7f1241c00740 1 client.16110 shutdown 2024-05-20T20:09:31.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.008+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae67680 msgr2=0x55a4eae5bbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.008+0000 7f1241c00740 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae67680 0x55a4eae5bbd0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f12300c7700 tx=0x7f12300c70b0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.011+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 shutdown_connections 2024-05-20T20:09:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.011+0000 7f1241c00740 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae7f2d0 0x55a4eae816c0 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.012+0000 7f1241c00740 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae60db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.012+0000 7f1241c00740 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae67680 0x55a4eae5bbd0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.012+0000 7f1241c00740 1 --2- 172.21.15.28:0/3286231165 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae60870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.012+0000 7f1222000640 0 client.16110 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.012+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 >> 172.21.15.28:0/3286231165 conn(0x55a4eae501a0 msgr2=0x55a4eae55f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.015+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 shutdown_connections 2024-05-20T20:09:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.016+0000 7f1241c00740 1 -- 172.21.15.28:0/3286231165 wait complete. 2024-05-20T20:09:31.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.017+0000 7f1241c00740 20 client.16110 trim_cache size 0 max 16384 2024-05-20T20:09:31.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.089+0000 7f1241c00740 1 Processor -- start 2024-05-20T20:09:31.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.089+0000 7f1241c00740 1 -- start start 2024-05-20T20:09:31.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.093+0000 7f1241c00740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae86de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.093+0000 7f1241c00740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae87320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.093+0000 7f1241c00740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae67680 0x55a4eae60420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.093+0000 7f1241c00740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f121800e800 con 0x55a4eae544f0 2024-05-20T20:09:31.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.093+0000 7f1241c00740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1218010340 con 0x55a4eae67680 2024-05-20T20:09:31.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.093+0000 7f1241c00740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f121800eed0 con 0x55a4eae53bd0 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f1237e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae86de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f1237e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae86de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55834/0 (socket says 172.21.15.28:55834) 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f1237e00640 1 -- 172.21.15.28:0/3318836734 learned_addr learned my addr 172.21.15.28:0/3318836734 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f123ce00640 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae67680 0x55a4eae60420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f123ce00640 1 -- 172.21.15.28:0/3318836734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 msgr2=0x55a4eae86de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f123ce00640 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae86de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f123ce00640 1 -- 172.21.15.28:0/3318836734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 msgr2=0x55a4eae87320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.094+0000 7f123ce00640 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae87320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.095+0000 7f123ce00640 1 -- 172.21.15.28:0/3318836734 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a4eae4ffd0 con 0x55a4eae67680 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.095+0000 7f1237e00640 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae86de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.095+0000 7f123ce00640 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae67680 0x55a4eae60420 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f1230009dc0 tx=0x7f1230005dc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c0d6bc8ca091bc68 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.095+0000 7f122cc00640 1 -- 172.21.15.28:0/3318836734 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12300ec070 con 0x55a4eae67680 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.095+0000 7f122cc00640 1 -- 172.21.15.28:0/3318836734 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1230100c70 con 0x55a4eae67680 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.095+0000 7f122cc00640 1 -- 172.21.15.28:0/3318836734 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12300ec070 con 0x55a4eae67680 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.096+0000 7f1241c00740 1 -- 172.21.15.28:0/3318836734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae67680 msgr2=0x55a4eae60420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.096+0000 7f1241c00740 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae67680 0x55a4eae60420 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f1230009dc0 tx=0x7f1230005dc0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.099+0000 7f1241c00740 1 -- 172.21.15.28:0/3318836734 shutdown_connections 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.099+0000 7f1241c00740 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae67680 0x55a4eae60420 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.099+0000 7f1241c00740 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae544f0 0x55a4eae87320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.099+0000 7f1241c00740 1 --2- 172.21.15.28:0/3318836734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae53bd0 0x55a4eae86de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.099+0000 7f1241c00740 1 -- 172.21.15.28:0/3318836734 >> 172.21.15.28:0/3318836734 conn(0x55a4eae501a0 msgr2=0x55a4eae4f310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.099+0000 7f1241c00740 1 -- 172.21.15.28:0/3318836734 shutdown_connections 2024-05-20T20:09:31.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.099+0000 7f1241c00740 1 -- 172.21.15.28:0/3318836734 wait complete. 2024-05-20T20:09:31.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.100+0000 7f1241c00740 1 -- start start 2024-05-20T20:09:31.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.100+0000 7f1241c00740 1 Processor -- start 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.101+0000 7f1241c00740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae53bd0 0x55a4eae92af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.102+0000 7f1241c00740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae93030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.102+0000 7f1241c00740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae8df50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f1237e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae53bd0 0x55a4eae92af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f1241c00740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f121800fc70 con 0x55a4eae53bd0 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f1241c00740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f121800f5a0 con 0x55a4eae544f0 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f1237e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae53bd0 0x55a4eae92af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48488/0 (socket says 172.21.15.28:48488) 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f1237e00640 1 -- 172.21.15.28:0/1373354286 learned_addr learned my addr 172.21.15.28:0/1373354286 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:31.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f121800c7a0 con 0x55a4eae67680 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f1237400640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae93030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.103+0000 7f123ce00640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae8df50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.104+0000 7f1237e00640 1 -- 172.21.15.28:0/1373354286 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 msgr2=0x55a4eae8df50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.105+0000 7f1237e00640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae8df50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.105+0000 7f1237e00640 1 -- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 msgr2=0x55a4eae93030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.105+0000 7f1237e00640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae93030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.105+0000 7f1237e00640 1 -- 172.21.15.28:0/1373354286 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a4eae8e750 con 0x55a4eae53bd0 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.105+0000 7f1241c00740 10 client.0 _ll_register_callbacks cb 0x7f12300902f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:31.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.105+0000 7f123ce00640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae8df50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.106+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1230100c70 con 0x55a4eae53bd0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.106+0000 7f1222000640 10 client.16113 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.106+0000 7f1237e00640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae53bd0 0x55a4eae92af0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f1228005370 tx=0x7f1228013b00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7b6708f7656104de server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.106+0000 7f1222000640 1 -- 172.21.15.28:0/1373354286 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1228004b40 con 0x55a4eae53bd0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.106+0000 7f1222000640 1 -- 172.21.15.28:0/1373354286 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f1228004ce0 con 0x55a4eae53bd0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.106+0000 7f1222000640 1 -- 172.21.15.28:0/1373354286 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f122801b290 con 0x55a4eae53bd0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.107+0000 7f1241c00740 10 client.16113 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.107+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({fsmap=29}) -- 0x55a4eae718d0 con 0x55a4eae53bd0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.107+0000 7f1222000640 1 -- 172.21.15.28:0/1373354286 <== mon.0 v2:172.21.15.28:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1228022070 con 0x55a4eae53bd0 2024-05-20T20:09:31.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.107+0000 7f1241c00740 10 client.16113 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:31.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.107+0000 7f1241c00740 10 client.16113 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:31.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.107+0000 7f1241c00740 20 client.16113 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:31.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.107+0000 7f1241c00740 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae71f50 0x55a4eae74370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.108+0000 7f1241c00740 4 client.16113 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:09:31.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.108+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a4eadc02c0 con 0x55a4eae71f50 2024-05-20T20:09:31.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.108+0000 7f1237400640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae71f50 0x55a4eae74370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.109+0000 7f1237400640 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae71f50 0x55a4eae74370 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=eec3079b51488c7f server_cookie=9282bedf4179ecef in_seq=0 out_seq=0 2024-05-20T20:09:31.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.109+0000 7f1222000640 10 client.16113 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.228+0000 7f1222000640 1 -- 172.21.15.28:0/1373354286 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55a4eadc02c0 con 0x55a4eae71f50 2024-05-20T20:09:31.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.228+0000 7f1222000640 10 client.16113 handle_command_reply: tid=0 2024-05-20T20:09:31.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.228+0000 7f1241c00740 1 client.16113 shutdown 2024-05-20T20:09:31.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.230+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae53bd0 msgr2=0x55a4eae92af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.230+0000 7f1241c00740 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae53bd0 0x55a4eae92af0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f1228005370 tx=0x7f1228013b00 comp rx=0 tx=0).stop 2024-05-20T20:09:31.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.231+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 shutdown_connections 2024-05-20T20:09:31.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.231+0000 7f1241c00740 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a4eae71f50 0x55a4eae74370 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.231+0000 7f1241c00740 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a4eae544f0 0x55a4eae93030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.231+0000 7f1241c00740 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a4eae67680 0x55a4eae8df50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.231+0000 7f1241c00740 1 --2- 172.21.15.28:0/1373354286 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a4eae53bd0 0x55a4eae92af0 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.231+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 >> 172.21.15.28:0/1373354286 conn(0x55a4eae501a0 msgr2=0x55a4eae65490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.231+0000 7f1222000640 0 client.16113 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.232+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 shutdown_connections 2024-05-20T20:09:31.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.232+0000 7f1241c00740 1 -- 172.21.15.28:0/1373354286 wait complete. 2024-05-20T20:09:31.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.233+0000 7f1241c00740 20 client.16113 trim_cache size 0 max 16384 2024-05-20T20:09:31.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:09:31.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:09:31.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:09:31.239 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.239+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f121c0b2b50 msgr2=0x7f121c0b5010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f121c0b2b50 0x7f121c0b5010 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f1224009fd0 tx=0x7f12240097b0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238115ba0 msgr2=0x7f12381a16c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238115ba0 0x7f12381a16c0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f1230006010 tx=0x7f1230003f10 comp rx=0 tx=0).stop 2024-05-20T20:09:31.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 shutdown_connections 2024-05-20T20:09:31.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1238115ba0 0x7f12381a16c0 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f121c0b2b50 0x7f121c0b5010 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f123810a370 0x7f12381a7b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.240+0000 7f123ee00640 1 --2- 172.21.15.28:0/870830295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1238109a50 0x7f12381a7600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.241+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 >> 172.21.15.28:0/870830295 conn(0x7f123806db20 msgr2=0x7f12380715d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.241+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 shutdown_connections 2024-05-20T20:09:31.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:31.241+0000 7f123ee00640 1 -- 172.21.15.28:0/870830295 wait complete. 2024-05-20T20:09:31.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.350+0000 7f0423200700 1 -- 172.21.15.28:0/662706058 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f042410a540 msgr2=0x7f042410a920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.350+0000 7f0423200700 1 --2- 172.21.15.28:0/662706058 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f042410a540 0x7f042410a920 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f04100099c0 tx=0x7f0410009cd0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.351+0000 7f0423200700 1 -- 172.21.15.28:0/662706058 shutdown_connections 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.351+0000 7f0423200700 1 --2- 172.21.15.28:0/662706058 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0424116690 0x7f0424118a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.351+0000 7f0423200700 1 --2- 172.21.15.28:0/662706058 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f042410a540 0x7f042410a920 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.351+0000 7f0423200700 1 --2- 172.21.15.28:0/662706058 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410ae60 0x7f0424116150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.351+0000 7f0423200700 1 -- 172.21.15.28:0/662706058 >> 172.21.15.28:0/662706058 conn(0x7f042406e520 msgr2=0x7f042406e930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.351+0000 7f0423200700 1 -- 172.21.15.28:0/662706058 shutdown_connections 2024-05-20T20:09:31.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.351+0000 7f0423200700 1 -- 172.21.15.28:0/662706058 wait complete. 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.354+0000 7f0423200700 1 Processor -- start 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.354+0000 7f0423200700 1 -- start start 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0423200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410a540 0x7f04241a22e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0423200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f042410ae60 0x7f04241a2820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0423200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0424116690 0x7f04241a6b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0423200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f042411be50 con 0x7f042410ae60 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0423200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f042411bcd0 con 0x7f042410a540 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0423200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f042411bfd0 con 0x7f0424116690 2024-05-20T20:09:31.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0421e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410a540 0x7f04241a22e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0421e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410a540 0x7f04241a22e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52204/0 (socket says 172.21.15.28:52204) 2024-05-20T20:09:31.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.355+0000 7f0421e00700 1 -- 172.21.15.28:0/3748347303 learned_addr learned my addr 172.21.15.28:0/3748347303 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:31.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.356+0000 7f0422800700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0424116690 0x7f04241a6b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.356+0000 7f0421e00700 1 -- 172.21.15.28:0/3748347303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0424116690 msgr2=0x7f04241a6b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.356+0000 7f0421e00700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0424116690 0x7f04241a6b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.356+0000 7f0421e00700 1 -- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f042410ae60 msgr2=0x7f04241a2820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.356+0000 7f0421e00700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f042410ae60 0x7f04241a2820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.356+0000 7f0421e00700 1 -- 172.21.15.28:0/3748347303 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f04241a70a0 con 0x7f042410a540 2024-05-20T20:09:31.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.356+0000 7f0422800700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0424116690 0x7f04241a6b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:31.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.357+0000 7f0421e00700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410a540 0x7f04241a22e0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f0410005910 tx=0x7f0410004690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.357+0000 7f041aa00700 1 -- 172.21.15.28:0/3748347303 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0410026070 con 0x7f042410a540 2024-05-20T20:09:31.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.357+0000 7f041aa00700 1 -- 172.21.15.28:0/3748347303 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f041000f460 con 0x7f042410a540 2024-05-20T20:09:31.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.358+0000 7f041aa00700 1 -- 172.21.15.28:0/3748347303 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0410009e60 con 0x7f042410a540 2024-05-20T20:09:31.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.359+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04241a7330 con 0x7f042410a540 2024-05-20T20:09:31.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.359+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f04241a7790 con 0x7f042410a540 2024-05-20T20:09:31.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.367+0000 7f041aa00700 1 -- 172.21.15.28:0/3748347303 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f041000f910 con 0x7f042410a540 2024-05-20T20:09:31.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.369+0000 7f04332b7b80 1 Processor -- start 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.370+0000 7f041aa00700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04080b2af0 0x7f04080b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.371+0000 7f04332b7b80 1 -- start start 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.371+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a3400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.371+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a3940 0x561c0809d4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.371+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c0809df30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.371+0000 7f04332b7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561c07ffbe40 con 0x561c0809dad0 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.371+0000 7f04332b7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561c07feb9e0 con 0x561c080a3940 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.371+0000 7f04332b7b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561c07fefa20 con 0x561c080a3020 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.372+0000 7f0421400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a3940 0x561c0809d4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.372+0000 7f0421400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a3940 0x561c0809d4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52210/0 (socket says 172.21.15.28:52210) 2024-05-20T20:09:31.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.372+0000 7f0421400700 1 -- 172.21.15.28:0/2562214329 learned_addr learned my addr 172.21.15.28:0/2562214329 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:31.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.374+0000 7f0421e00700 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a3400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.374+0000 7f041aa00700 1 -- 172.21.15.28:0/3748347303 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f04100e2cd0 con 0x7f042410a540 2024-05-20T20:09:31.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.375+0000 7f0421400700 1 -- 172.21.15.28:0/2562214329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 msgr2=0x561c080a3400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.375+0000 7f0421400700 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a3400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.376+0000 7f0421400700 1 -- 172.21.15.28:0/2562214329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 msgr2=0x561c0809df30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.376+0000 7f0421400700 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c0809df30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.376+0000 7f0421400700 1 -- 172.21.15.28:0/2562214329 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561c07fb4420 con 0x561c080a3940 2024-05-20T20:09:31.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.376+0000 7f0422800700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04080b2af0 0x7f04080b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.378+0000 7f0421400700 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a3940 0x561c0809d4b0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f0414009c70 tx=0x7f0414009f80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a9971f215aa127a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.378+0000 7f0422800700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04080b2af0 0x7f04080b4fb0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f041c009690 tx=0x7f041c0091a0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.379+0000 7f040f400700 1 -- 172.21.15.28:0/2562214329 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f041400ec10 con 0x561c080a3940 2024-05-20T20:09:31.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.379+0000 7f040f400700 1 -- 172.21.15.28:0/2562214329 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f041400edb0 con 0x561c080a3940 2024-05-20T20:09:31.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.379+0000 7f040f400700 1 -- 172.21.15.28:0/2562214329 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0414010430 con 0x561c080a3940 2024-05-20T20:09:31.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.380+0000 7f04332b7b80 1 -- 172.21.15.28:0/2562214329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a3940 msgr2=0x561c0809d4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.380+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a3940 0x561c0809d4b0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f0414009c70 tx=0x7f0414009f80 comp rx=0 tx=0).stop 2024-05-20T20:09:31.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.380+0000 7f04332b7b80 1 -- 172.21.15.28:0/2562214329 shutdown_connections 2024-05-20T20:09:31.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.380+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c0809df30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.380+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a3400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.380+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2562214329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a3940 0x561c0809d4b0 secure :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f0414009c70 tx=0x7f0414009f80 comp rx=0 tx=0).stop 2024-05-20T20:09:31.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.380+0000 7f04332b7b80 1 -- 172.21.15.28:0/2562214329 >> 172.21.15.28:0/2562214329 conn(0x561c07fcbbe0 msgr2=0x561c08098150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.381+0000 7f04332b7b80 1 -- 172.21.15.28:0/2562214329 shutdown_connections 2024-05-20T20:09:31.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.381+0000 7f04332b7b80 1 -- 172.21.15.28:0/2562214329 wait complete. 2024-05-20T20:09:31.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.381+0000 7f04332b7b80 1 -- start start 2024-05-20T20:09:31.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.382+0000 7f04332b7b80 1 Processor -- start 2024-05-20T20:09:31.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.383+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c080a9960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.383+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a6c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.383+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080a7650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.383+0000 7f04332b7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561c07ef1fa0 con 0x561c0809dad0 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.383+0000 7f04332b7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561c07f8c760 con 0x561c080a71f0 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.384+0000 7f0421400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a6c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.384+0000 7f0421400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a6c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55884/0 (socket says 172.21.15.28:55884) 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.384+0000 7f0421400700 1 -- 172.21.15.28:0/2759768451 learned_addr learned my addr 172.21.15.28:0/2759768451 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.384+0000 7f0421e00700 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c080a9960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.384+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561c080aa500 con 0x561c080a3020 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.384+0000 7f04332b7b80 10 client.0 _ll_register_callbacks cb 0x561c07f95c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:31.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.384+0000 7f0422800700 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080a7650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.385+0000 7f0421e00700 1 -- 172.21.15.28:0/2759768451 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 msgr2=0x561c080a6c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.385+0000 7f0421e00700 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a6c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.385+0000 7f0421e00700 1 -- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 msgr2=0x561c080a7650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.385+0000 7f0421e00700 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080a7650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.385+0000 7f0421e00700 1 -- 172.21.15.28:0/2759768451 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561c07f3b8d0 con 0x561c0809dad0 2024-05-20T20:09:31.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.386+0000 7f0421e00700 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c080a9960 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f04100e03d0 tx=0x7f041000bfb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=557d7bc7ec9e7e1a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.386+0000 7f0403e00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:31.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.386+0000 7f0403e00700 1 -- 172.21.15.28:0/2759768451 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04100a9940 con 0x561c0809dad0 2024-05-20T20:09:31.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.386+0000 7f0403e00700 1 -- 172.21.15.28:0/2759768451 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f04100aa460 con 0x561c0809dad0 2024-05-20T20:09:31.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.386+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f041400edb0 con 0x561c0809dad0 2024-05-20T20:09:31.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.387+0000 7f0403e00700 1 -- 172.21.15.28:0/2759768451 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f04100a9760 con 0x561c0809dad0 2024-05-20T20:09:31.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.387+0000 7f04332b7b80 10 client.16119 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:31.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.387+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561c07fbc060 con 0x561c0809dad0 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.387+0000 7f0403e00700 1 -- 172.21.15.28:0/2759768451 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f041400edb0 con 0x561c0809dad0 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f0403e00700 1 -- 172.21.15.28:0/2759768451 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f04100b2930 con 0x561c0809dad0 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f04332b7b80 10 client.16119 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f04332b7b80 10 client.16119 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f04332b7b80 20 client.16119 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c080b76d0 0x561c080b9af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f04332b7b80 4 client.16119 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561c07ffdd90 con 0x561c080b76d0 2024-05-20T20:09:31.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.388+0000 7f0421400700 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c080b76d0 0x561c080b9af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.389+0000 7f0421400700 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c080b76d0 0x561c080b9af0 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=66eaa8ad5426fad1 server_cookie=8c0d9e5673f863b5 in_seq=0 out_seq=0 2024-05-20T20:09:31.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.389+0000 7f0403e00700 10 client.16119 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.392+0000 7f0403e00700 1 -- 172.21.15.28:0/2759768451 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561c07ffdd90 con 0x561c080b76d0 2024-05-20T20:09:31.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.392+0000 7f0403e00700 10 client.16119 handle_command_reply: tid=0 2024-05-20T20:09:31.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.393+0000 7f04332b7b80 1 client.16119 shutdown 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.393+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 msgr2=0x561c080a9960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.393+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c080a9960 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f04100e03d0 tx=0x7f041000bfb0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.394+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 shutdown_connections 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.394+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c080b76d0 0x561c080b9af0 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.394+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080a7650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.394+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c0809dad0 0x561c080a9960 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.394+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2759768451 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a3020 0x561c080a6c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.394+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 >> 172.21.15.28:0/2759768451 conn(0x561c07fcbbe0 msgr2=0x561c0809c4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.394+0000 7f0403e00700 0 client.16119 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.395+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 shutdown_connections 2024-05-20T20:09:31.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.395+0000 7f04332b7b80 1 -- 172.21.15.28:0/2759768451 wait complete. 2024-05-20T20:09:31.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.396+0000 7f04332b7b80 20 client.16119 trim_cache size 0 max 16384 2024-05-20T20:09:31.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.428+0000 7f04332b7b80 1 Processor -- start 2024-05-20T20:09:31.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.428+0000 7f04332b7b80 1 -- start start 2024-05-20T20:09:31.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c0809dad0 0x561c0809deb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 0x561c08097c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a71f0 0x561c08098170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f04332b7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f04241bc800 con 0x561c080a3020 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f04332b7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f04241be340 con 0x561c0809dad0 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f04332b7b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f04241bced0 con 0x561c080a71f0 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a71f0 0x561c08098170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0421e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c0809dad0 0x561c0809deb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a71f0 0x561c08098170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55892/0 (socket says 172.21.15.28:55892) 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 -- 172.21.15.28:0/2721855931 learned_addr learned my addr 172.21.15.28:0/2721855931 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:31.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0421400700 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 0x561c08097c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 -- 172.21.15.28:0/2721855931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c0809dad0 msgr2=0x561c0809deb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c0809dad0 0x561c0809deb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 -- 172.21.15.28:0/2721855931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 msgr2=0x561c08097c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 0x561c08097c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.429+0000 7f0422800700 1 -- 172.21.15.28:0/2721855931 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561c08092650 con 0x561c080a71f0 2024-05-20T20:09:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.430+0000 7f0422800700 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a71f0 0x561c08098170 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f041c007f80 tx=0x7f041c013000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b94508e7e186164b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.430+0000 7f040ea00700 1 -- 172.21.15.28:0/2721855931 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f041c021460 con 0x561c080a71f0 2024-05-20T20:09:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.430+0000 7f040ea00700 1 -- 172.21.15.28:0/2721855931 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f041c026070 con 0x561c080a71f0 2024-05-20T20:09:31.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.430+0000 7f040ea00700 1 -- 172.21.15.28:0/2721855931 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f041c015880 con 0x561c080a71f0 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 -- 172.21.15.28:0/2721855931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a71f0 msgr2=0x561c08098170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a71f0 0x561c08098170 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f041c007f80 tx=0x7f041c013000 comp rx=0 tx=0).stop 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 -- 172.21.15.28:0/2721855931 shutdown_connections 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c080a71f0 0x561c08098170 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c0809dad0 0x561c0809deb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2721855931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 0x561c08097c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 -- 172.21.15.28:0/2721855931 >> 172.21.15.28:0/2721855931 conn(0x561c07fcbbe0 msgr2=0x561c080a5570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 -- 172.21.15.28:0/2721855931 shutdown_connections 2024-05-20T20:09:31.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.431+0000 7f04332b7b80 1 -- 172.21.15.28:0/2721855931 wait complete. 2024-05-20T20:09:31.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.432+0000 7f04332b7b80 1 -- start start 2024-05-20T20:09:31.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.432+0000 7f04332b7b80 1 Processor -- start 2024-05-20T20:09:31.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c0809dad0 0x561c080af790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 0x561c080afcd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f04332b7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080ac760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f04332b7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f04241bdc70 con 0x561c080a3020 2024-05-20T20:09:31.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f04332b7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f04241bd5a0 con 0x561c080a71f0 2024-05-20T20:09:31.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f04332b7b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f04241b0d70 con 0x561c0809dad0 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f04332b7b80 10 client.0 _ll_register_callbacks cb 0x561c07f95c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f0421e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c0809dad0 0x561c080af790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f0421e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c0809dad0 0x561c080af790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55896/0 (socket says 172.21.15.28:55896) 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f0421e00700 1 -- 172.21.15.28:0/2578068048 learned_addr learned my addr 172.21.15.28:0/2578068048 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f0422800700 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080ac760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f0421e00700 1 -- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 msgr2=0x561c080ac760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.433+0000 7f0421e00700 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080ac760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.434+0000 7f0421e00700 1 -- 172.21.15.28:0/2578068048 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 msgr2=0x561c080afcd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.434+0000 7f0421e00700 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 0x561c080afcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.434+0000 7f0421e00700 1 -- 172.21.15.28:0/2578068048 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561c080acd20 con 0x561c0809dad0 2024-05-20T20:09:31.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.434+0000 7f0421e00700 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c0809dad0 0x561c080af790 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f0410000c00 tx=0x7f0410009390 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b6e6c0dff369298d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:31.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.434+0000 7f0403e00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:09:31.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.434+0000 7f0403e00700 1 -- 172.21.15.28:0/2578068048 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04100a5eb0 con 0x561c0809dad0 2024-05-20T20:09:31.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.434+0000 7f0403e00700 1 -- 172.21.15.28:0/2578068048 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f04100a5bc0 con 0x561c0809dad0 2024-05-20T20:09:31.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.435+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f041c026070 con 0x561c0809dad0 2024-05-20T20:09:31.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.435+0000 7f0403e00700 1 -- 172.21.15.28:0/2578068048 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f04100a9b00 con 0x561c0809dad0 2024-05-20T20:09:31.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.437+0000 7f04332b7b80 10 client.26245 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:31.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.437+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561c080acd20 con 0x561c0809dad0 2024-05-20T20:09:31.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.438+0000 7f0403e00700 1 -- 172.21.15.28:0/2578068048 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f041c026070 con 0x561c0809dad0 2024-05-20T20:09:31.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.438+0000 7f0403e00700 1 -- 172.21.15.28:0/2578068048 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f04100a8cc0 con 0x561c0809dad0 2024-05-20T20:09:31.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.439+0000 7f04332b7b80 10 client.26245 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:31.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.439+0000 7f04332b7b80 10 client.26245 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:31.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.439+0000 7f04332b7b80 20 client.26245 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:31.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.439+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c08133e80 0x561c08136270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:31.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.439+0000 7f04332b7b80 4 client.26245 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:09:31.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.439+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x561c07ff7570 con 0x561c08133e80 2024-05-20T20:09:31.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.440+0000 7f0421400700 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c08133e80 0x561c08136270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:31.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.442+0000 7f0421400700 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c08133e80 0x561c08136270 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=597713bbcb864ab6 server_cookie=3a18e1a506a96796 in_seq=0 out_seq=0 2024-05-20T20:09:31.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.442+0000 7f0403e00700 10 client.26245 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.445+0000 7f0403e00700 1 -- 172.21.15.28:0/2578068048 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x561c07ff7570 con 0x561c08133e80 2024-05-20T20:09:31.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.445+0000 7f0403e00700 10 client.26245 handle_command_reply: tid=0 2024-05-20T20:09:31.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.445+0000 7f04332b7b80 1 client.26245 shutdown 2024-05-20T20:09:31.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.445+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c0809dad0 msgr2=0x561c080af790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.445+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c0809dad0 0x561c080af790 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f0410000c00 tx=0x7f0410009390 comp rx=0 tx=0).stop 2024-05-20T20:09:31.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.446+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 shutdown_connections 2024-05-20T20:09:31.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.446+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561c08133e80 0x561c08136270 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.446+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561c080a71f0 0x561c080ac760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.446+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561c0809dad0 0x561c080af790 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.446+0000 7f04332b7b80 1 --2- 172.21.15.28:0/2578068048 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561c080a3020 0x561c080afcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.447+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 >> 172.21.15.28:0/2578068048 conn(0x561c07fcbbe0 msgr2=0x561c080a54a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.447+0000 7f0403e00700 0 client.26245 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:31.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.447+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 shutdown_connections 2024-05-20T20:09:31.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.447+0000 7f04332b7b80 1 -- 172.21.15.28:0/2578068048 wait complete. 2024-05-20T20:09:31.447 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.447+0000 7f04332b7b80 20 client.26245 trim_cache size 0 max 16384 2024-05-20T20:09:31.448 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:09:31.448 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:09:31.448 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:09:31.448 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04080b2af0 msgr2=0x7f04080b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04080b2af0 0x7f04080b4fb0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f041c009690 tx=0x7f041c0091a0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410a540 msgr2=0x7f04241a22e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410a540 0x7f04241a22e0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f0410005910 tx=0x7f0410004690 comp rx=0 tx=0).stop 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 shutdown_connections 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0424116690 0x7f04241a6b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04080b2af0 0x7f04080b4fb0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f042410a540 0x7f04241a22e0 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 --2- 172.21.15.28:0/3748347303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f042410ae60 0x7f04241a2820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:31.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.449+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 >> 172.21.15.28:0/3748347303 conn(0x7f042406e520 msgr2=0x7f04240720c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:31.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.450+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 shutdown_connections 2024-05-20T20:09:31.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:31.450+0000 7f0423200700 1 -- 172.21.15.28:0/3748347303 wait complete. 2024-05-20T20:09:33.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:32 smithi028 ceph-mon[31178]: pgmap v120: 97 pgs: 97 active+clean; 12 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 197 MiB/s wr, 1.53k op/s 2024-05-20T20:09:33.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:32 smithi154 ceph-mon[36314]: pgmap v120: 97 pgs: 97 active+clean; 12 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 197 MiB/s wr, 1.53k op/s 2024-05-20T20:09:33.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:32 smithi033 ceph-mon[36179]: pgmap v120: 97 pgs: 97 active+clean; 12 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 197 MiB/s wr, 1.53k op/s 2024-05-20T20:09:33.804 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:09:33.804 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:09:34.679 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:34 smithi154 ceph-mon[36314]: pgmap v121: 97 pgs: 97 active+clean; 13 GiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 290 MiB/s wr, 2.01k op/s 2024-05-20T20:09:35.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:34 smithi033 ceph-mon[36179]: pgmap v121: 97 pgs: 97 active+clean; 13 GiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 290 MiB/s wr, 2.01k op/s 2024-05-20T20:09:35.163 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:34 smithi028 ceph-mon[31178]: pgmap v121: 97 pgs: 97 active+clean; 13 GiB data, 2.9 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 290 MiB/s wr, 2.01k op/s 2024-05-20T20:09:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:36 smithi154 ceph-mon[36314]: pgmap v122: 97 pgs: 97 active+clean; 14 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 255 MiB/s wr, 1.54k op/s 2024-05-20T20:09:37.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:36 smithi033 ceph-mon[36179]: pgmap v122: 97 pgs: 97 active+clean; 14 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 255 MiB/s wr, 1.54k op/s 2024-05-20T20:09:37.149 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:36 smithi028 ceph-mon[31178]: pgmap v122: 97 pgs: 97 active+clean; 14 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 255 MiB/s wr, 1.54k op/s 2024-05-20T20:09:39.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:38 smithi028 ceph-mon[31178]: pgmap v123: 97 pgs: 97 active+clean; 14 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 237 MiB/s wr, 1.39k op/s 2024-05-20T20:09:39.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:38 smithi154 ceph-mon[36314]: pgmap v123: 97 pgs: 97 active+clean; 14 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 237 MiB/s wr, 1.39k op/s 2024-05-20T20:09:39.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:38 smithi033 ceph-mon[36179]: pgmap v123: 97 pgs: 97 active+clean; 14 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 237 MiB/s wr, 1.39k op/s 2024-05-20T20:09:39.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.928+0000 7f7d11600700 1 -- 172.21.15.28:0/2859273186 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 msgr2=0x7f7d0c06bbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:39.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.928+0000 7f7d11600700 1 --2- 172.21.15.28:0/2859273186 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c06bbf0 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc0098f0 tx=0x7f7cfc009c00 comp rx=0 tx=0).stop 2024-05-20T20:09:39.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.929+0000 7f7d11600700 1 -- 172.21.15.28:0/2859273186 shutdown_connections 2024-05-20T20:09:39.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.929+0000 7f7d11600700 1 --2- 172.21.15.28:0/2859273186 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d0c105f80 0x7f7d0c106410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:39.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.929+0000 7f7d11600700 1 --2- 172.21.15.28:0/2859273186 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d0c108470 0x7f7d0c06b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:39.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.929+0000 7f7d11600700 1 --2- 172.21.15.28:0/2859273186 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c06bbf0 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:39.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.929+0000 7f7d11600700 1 -- 172.21.15.28:0/2859273186 >> 172.21.15.28:0/2859273186 conn(0x7f7d0c0793f0 msgr2=0x7f7d0c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:39.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.930+0000 7f7d11600700 1 -- 172.21.15.28:0/2859273186 shutdown_connections 2024-05-20T20:09:39.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.930+0000 7f7d11600700 1 -- 172.21.15.28:0/2859273186 wait complete. 2024-05-20T20:09:39.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.932+0000 7f7d11600700 1 Processor -- start 2024-05-20T20:09:39.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.932+0000 7f7d11600700 1 -- start start 2024-05-20T20:09:39.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.933+0000 7f7d11600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c1a02a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:39.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.933+0000 7f7d11600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d0c105f80 0x7f7d0c1a07e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:39.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.933+0000 7f7d11600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d0c108470 0x7f7d0c1a4b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:39.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.933+0000 7f7d11600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7d0c119d70 con 0x7f7d0c06b790 2024-05-20T20:09:39.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.933+0000 7f7d11600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7d0c119bf0 con 0x7f7d0c105f80 2024-05-20T20:09:39.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.933+0000 7f7d11600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7d0c119ef0 con 0x7f7d0c108470 2024-05-20T20:09:39.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.934+0000 7f7d0aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c1a02a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:39.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.934+0000 7f7d0aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c1a02a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35882/0 (socket says 172.21.15.28:35882) 2024-05-20T20:09:39.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.934+0000 7f7d0aa00700 1 -- 172.21.15.28:0/2947796392 learned_addr learned my addr 172.21.15.28:0/2947796392 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:39.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.934+0000 7f7d02000700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d0c105f80 0x7f7d0c1a07e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:39.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.935+0000 7f7d0aa00700 1 -- 172.21.15.28:0/2947796392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d0c108470 msgr2=0x7f7d0c1a4b70 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:09:39.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.935+0000 7f7d0aa00700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d0c108470 0x7f7d0c1a4b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.935+0000 7f7d0aa00700 1 -- 172.21.15.28:0/2947796392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d0c105f80 msgr2=0x7f7d0c1a07e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.935+0000 7f7d0aa00700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d0c105f80 0x7f7d0c1a07e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.935+0000 7f7d0aa00700 1 -- 172.21.15.28:0/2947796392 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7d0c1a52b0 con 0x7f7d0c06b790 2024-05-20T20:09:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.936+0000 7f7d0aa00700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c1a02a0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f7cf400e9c0 tx=0x7f7cf400ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.936+0000 7f7d03e00700 1 -- 172.21.15.28:0/2947796392 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7cf400cbd0 con 0x7f7d0c06b790 2024-05-20T20:09:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.936+0000 7f7d03e00700 1 -- 172.21.15.28:0/2947796392 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7cf400cd70 con 0x7f7d0c06b790 2024-05-20T20:09:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.936+0000 7f7d03e00700 1 -- 172.21.15.28:0/2947796392 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7cf40055a0 con 0x7f7d0c06b790 2024-05-20T20:09:39.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.937+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7d0c1a55a0 con 0x7f7d0c06b790 2024-05-20T20:09:39.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.938+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7d0c1ace20 con 0x7f7d0c06b790 2024-05-20T20:09:39.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.939+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7d0c102a60 con 0x7f7d0c06b790 2024-05-20T20:09:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.950+0000 7f7d03e00700 1 -- 172.21.15.28:0/2947796392 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7cf4004750 con 0x7f7d0c06b790 2024-05-20T20:09:39.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.955+0000 7f7d03e00700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7cf00b2aa0 0x7f7cf00b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:39.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.955+0000 7f7d03e00700 1 -- 172.21.15.28:0/2947796392 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7cf40e27f0 con 0x7f7d0c06b790 2024-05-20T20:09:39.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.957+0000 7f7d02000700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7cf00b2aa0 0x7f7cf00b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:39.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.958+0000 7f7d02000700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7cf00b2aa0 0x7f7cf00b4f60 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc0098c0 tx=0x7f7cfc005d00 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:39.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:39.966+0000 7f7d03e00700 1 -- 172.21.15.28:0/2947796392 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7cf40aa1e0 con 0x7f7d0c06b790 2024-05-20T20:09:40.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.518+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f7d0c06a230 con 0x7f7d0c06b790 2024-05-20T20:09:40.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.520+0000 7f7d03e00700 1 -- 172.21.15.28:0/2947796392 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f7cf40a9ae0 con 0x7f7d0c06b790 2024-05-20T20:09:40.521 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:09:40.540 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:09:40.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.533+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7cf00b2aa0 msgr2=0x7f7cf00b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:40.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.533+0000 7f7d11600700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7cf00b2aa0 0x7f7cf00b4f60 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f7cfc0098c0 tx=0x7f7cfc005d00 comp rx=0 tx=0).stop 2024-05-20T20:09:40.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.534+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 msgr2=0x7f7d0c1a02a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:40.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.534+0000 7f7d11600700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c1a02a0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f7cf400e9c0 tx=0x7f7cf400ecd0 comp rx=0 tx=0).stop 2024-05-20T20:09:40.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.541+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 shutdown_connections 2024-05-20T20:09:40.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.541+0000 7f7d11600700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7d0c108470 0x7f7d0c1a4b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:40.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.541+0000 7f7d11600700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7d0c06b790 0x7f7d0c1a02a0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:40.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.541+0000 7f7d11600700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7cf00b2aa0 0x7f7cf00b4f60 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:40.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.541+0000 7f7d11600700 1 --2- 172.21.15.28:0/2947796392 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7d0c105f80 0x7f7d0c1a07e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:40.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.541+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 >> 172.21.15.28:0/2947796392 conn(0x7f7d0c0793f0 msgr2=0x7f7d0c104f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:40.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.547+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 shutdown_connections 2024-05-20T20:09:40.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:40.548+0000 7f7d11600700 1 -- 172.21.15.28:0/2947796392 wait complete. 2024-05-20T20:09:40.561 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:09:41.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:40 smithi154 ceph-mon[36314]: pgmap v124: 97 pgs: 97 active+clean; 15 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 362 MiB/s wr, 1.85k op/s 2024-05-20T20:09:41.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:40 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2947796392' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:09:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:40 smithi033 ceph-mon[36179]: pgmap v124: 97 pgs: 97 active+clean; 15 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 362 MiB/s wr, 1.85k op/s 2024-05-20T20:09:41.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:40 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2947796392' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:09:41.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:40 smithi028 ceph-mon[31178]: pgmap v124: 97 pgs: 97 active+clean; 15 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 362 MiB/s wr, 1.85k op/s 2024-05-20T20:09:41.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:40 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2947796392' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:09:42.640 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:09:43.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:42 smithi028 ceph-mon[31178]: pgmap v125: 97 pgs: 97 active+clean; 15 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 247 MiB/s wr, 1.09k op/s 2024-05-20T20:09:43.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:42 smithi154 ceph-mon[36314]: pgmap v125: 97 pgs: 97 active+clean; 15 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 247 MiB/s wr, 1.09k op/s 2024-05-20T20:09:43.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:42 smithi033 ceph-mon[36179]: pgmap v125: 97 pgs: 97 active+clean; 15 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 247 MiB/s wr, 1.09k op/s 2024-05-20T20:09:43.576 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:44.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 -- 172.21.15.28:0/200055925 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58106df0 msgr2=0x7f6b58107280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 --2- 172.21.15.28:0/200055925 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58106df0 0x7f6b58107280 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f6b50009be0 tx=0x7f6b5002a7e0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 -- 172.21.15.28:0/200055925 shutdown_connections 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 --2- 172.21.15.28:0/200055925 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58106df0 0x7f6b58107280 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 --2- 172.21.15.28:0/200055925 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6b581023f0 0x7f6b581068b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 --2- 172.21.15.28:0/200055925 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6b58101ad0 0x7f6b58101eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 -- 172.21.15.28:0/200055925 >> 172.21.15.28:0/200055925 conn(0x7f6b58078ad0 msgr2=0x7f6b58078ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 -- 172.21.15.28:0/200055925 shutdown_connections 2024-05-20T20:09:44.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.522+0000 7f6b5f600640 1 -- 172.21.15.28:0/200055925 wait complete. 2024-05-20T20:09:44.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.524+0000 7f6b5f600640 1 Processor -- start 2024-05-20T20:09:44.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.524+0000 7f6b5f600640 1 -- start start 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5f600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58101ad0 0x7f6b581a53e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5f600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6b581023f0 0x7f6b581a5920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5f600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6b58106df0 0x7f6b5819f5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5f600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6b581192a0 con 0x7f6b581023f0 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5f600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6b58119120 con 0x7f6b58101ad0 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5f600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6b58119420 con 0x7f6b58106df0 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5cc00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58101ad0 0x7f6b581a53e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5d600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6b58106df0 0x7f6b5819f5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5cc00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58101ad0 0x7f6b581a53e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36360/0 (socket says 172.21.15.28:36360) 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5d600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6b58106df0 0x7f6b5819f5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47910/0 (socket says 172.21.15.28:47910) 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.525+0000 7f6b5cc00640 1 -- 172.21.15.28:0/333804341 learned_addr learned my addr 172.21.15.28:0/333804341 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:44.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b57e00640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6b581023f0 0x7f6b581a5920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b5cc00640 1 -- 172.21.15.28:0/333804341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6b58106df0 msgr2=0x7f6b5819f5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6b58106df0 0x7f6b5819f5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b5cc00640 1 -- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6b581023f0 msgr2=0x7f6b581a5920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6b581023f0 0x7f6b581a5920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b5cc00640 1 -- 172.21.15.28:0/333804341 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6b5819fd60 con 0x7f6b58101ad0 2024-05-20T20:09:44.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b57e00640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6b581023f0 0x7f6b581a5920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:09:44.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.526+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58101ad0 0x7f6b581a53e0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f6b4c00e9f0 tx=0x7f6b4c00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:44.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.527+0000 7f6b55600640 1 -- 172.21.15.28:0/333804341 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b4c00cd80 con 0x7f6b58101ad0 2024-05-20T20:09:44.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.527+0000 7f6b55600640 1 -- 172.21.15.28:0/333804341 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6b4c0108e0 con 0x7f6b58101ad0 2024-05-20T20:09:44.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.527+0000 7f6b55600640 1 -- 172.21.15.28:0/333804341 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b4c018980 con 0x7f6b58101ad0 2024-05-20T20:09:44.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.527+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6b581a0050 con 0x7f6b58101ad0 2024-05-20T20:09:44.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.528+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6b581ac2a0 con 0x7f6b58101ad0 2024-05-20T20:09:44.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.535+0000 7f6b55600640 1 -- 172.21.15.28:0/333804341 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f6b4c010430 con 0x7f6b58101ad0 2024-05-20T20:09:44.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.538+0000 7f6b623ac740 1 Processor -- start 2024-05-20T20:09:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.538+0000 7f6b623ac740 1 -- start start 2024-05-20T20:09:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.539+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf44070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.539+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf445b0 0x564dcbf4e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.539+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf51260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.539+0000 7f6b623ac740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564dcbeb4b90 con 0x564dcbf43c90 2024-05-20T20:09:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.539+0000 7f6b623ac740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564dcbea4730 con 0x564dcbf445b0 2024-05-20T20:09:44.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.539+0000 7f6b623ac740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564dcbea8770 con 0x564dcbf4ee70 2024-05-20T20:09:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.540+0000 7f6b5cc00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf445b0 0x564dcbf4e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.540+0000 7f6b5cc00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf445b0 0x564dcbf4e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36368/0 (socket says 172.21.15.28:36368) 2024-05-20T20:09:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.540+0000 7f6b57e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf51260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.540+0000 7f6b57e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf51260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47926/0 (socket says 172.21.15.28:47926) 2024-05-20T20:09:44.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.540+0000 7f6b57e00640 1 -- 172.21.15.28:0/3781102862 learned_addr learned my addr 172.21.15.28:0/3781102862 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:44.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.541+0000 7f6b57e00640 1 -- 172.21.15.28:0/3781102862 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf445b0 msgr2=0x564dcbf4e930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:44.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.541+0000 7f6b57e00640 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf445b0 0x564dcbf4e930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.541+0000 7f6b57e00640 1 -- 172.21.15.28:0/3781102862 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 msgr2=0x564dcbf44070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:44.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.541+0000 7f6b5d600640 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf44070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.541+0000 7f6b57e00640 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf44070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.541+0000 7f6b57e00640 1 -- 172.21.15.28:0/3781102862 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564dcbe3d8b0 con 0x564dcbf4ee70 2024-05-20T20:09:44.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.541+0000 7f6b5d600640 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf44070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:44.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.545+0000 7f6b55600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6b400b2a90 0x7f6b400b4f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.545+0000 7f6b57e00640 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf51260 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f6b4400bc60 tx=0x7f6b4400f730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5d5306c4f6782a45 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:44.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.546+0000 7f6b49c00640 1 -- 172.21.15.28:0/3781102862 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b44015070 con 0x564dcbf4ee70 2024-05-20T20:09:44.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.546+0000 7f6b49c00640 1 -- 172.21.15.28:0/3781102862 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6b440035f0 con 0x564dcbf4ee70 2024-05-20T20:09:44.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.546+0000 7f6b49c00640 1 -- 172.21.15.28:0/3781102862 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b440116a0 con 0x564dcbf4ee70 2024-05-20T20:09:44.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.548+0000 7f6b55600640 1 -- 172.21.15.28:0/333804341 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f6b4c0e2c00 con 0x7f6b58101ad0 2024-05-20T20:09:44.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.548+0000 7f6b5d600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6b400b2a90 0x7f6b400b4f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.549+0000 7f6b623ac740 1 -- 172.21.15.28:0/3781102862 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 msgr2=0x564dcbf51260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.550+0000 7f6b5d600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6b400b2a90 0x7f6b400b4f50 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f6b50004af0 tx=0x7f6b50005fd0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:44.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.550+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf51260 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f6b4400bc60 tx=0x7f6b4400f730 comp rx=0 tx=0).stop 2024-05-20T20:09:44.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.551+0000 7f6b623ac740 1 -- 172.21.15.28:0/3781102862 shutdown_connections 2024-05-20T20:09:44.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.551+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf445b0 0x564dcbf4e930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.551+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf51260 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.551+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3781102862 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf44070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.551+0000 7f6b623ac740 1 -- 172.21.15.28:0/3781102862 >> 172.21.15.28:0/3781102862 conn(0x564dcbf401a0 msgr2=0x564dcbf40580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:44.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.557+0000 7f6b623ac740 1 -- 172.21.15.28:0/3781102862 shutdown_connections 2024-05-20T20:09:44.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.558+0000 7f6b623ac740 1 -- 172.21.15.28:0/3781102862 wait complete. 2024-05-20T20:09:44.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.563+0000 7f6b623ac740 1 -- start start 2024-05-20T20:09:44.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.564+0000 7f6b623ac740 1 Processor -- start 2024-05-20T20:09:44.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.569+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf4e7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.569+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf445b0 0x564dcbf4bab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.569+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf4ee70 0x564dcbf4c010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.570+0000 7f6b623ac740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564dcbb8a790 con 0x564dcbf43c90 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.570+0000 7f6b623ac740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564dcbe4cbc0 con 0x564dcbf4ee70 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.570+0000 7f6b623ac740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564dcbf62ab0 con 0x564dcbf445b0 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.570+0000 7f6b5d600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf4ee70 0x564dcbf4c010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.570+0000 7f6b5d600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf4ee70 0x564dcbf4c010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36378/0 (socket says 172.21.15.28:36378) 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.570+0000 7f6b5d600640 1 -- 172.21.15.28:0/2066701949 learned_addr learned my addr 172.21.15.28:0/2066701949 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.570+0000 7f6b623ac740 10 client.0 _ll_register_callbacks cb 0x564dcbe2a400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.571+0000 7f6b5d600640 1 -- 172.21.15.28:0/2066701949 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf445b0 msgr2=0x564dcbf4bab0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.571+0000 7f6b5d600640 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf445b0 0x564dcbf4bab0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.571+0000 7f6b5d600640 1 -- 172.21.15.28:0/2066701949 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 msgr2=0x564dcbf4e7c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.571+0000 7f6b5d600640 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf4e7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.571+0000 7f6b5d600640 1 -- 172.21.15.28:0/2066701949 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564dcbbb9210 con 0x564dcbf4ee70 2024-05-20T20:09:44.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.572+0000 7f6b5d600640 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf4ee70 0x564dcbf4c010 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f6b50033000 tx=0x7f6b5002a7e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d42997f81ccd3fcd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:44.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.573+0000 7f6b3e000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:44.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.573+0000 7f6b3e000640 1 -- 172.21.15.28:0/2066701949 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b50044070 con 0x564dcbf4ee70 2024-05-20T20:09:44.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.573+0000 7f6b3e000640 1 -- 172.21.15.28:0/2066701949 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f6b50004250 con 0x564dcbf4ee70 2024-05-20T20:09:44.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.573+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6b440035f0 con 0x564dcbf4ee70 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.574+0000 7f6b3e000640 1 -- 172.21.15.28:0/2066701949 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6b50036040 con 0x564dcbf4ee70 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.574+0000 7f6b623ac740 10 client.26330 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.574+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564dcbc85620 con 0x564dcbf4ee70 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.574+0000 7f6b3e000640 1 -- 172.21.15.28:0/2066701949 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6b440035f0 con 0x564dcbf4ee70 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.575+0000 7f6b3e000640 1 -- 172.21.15.28:0/2066701949 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f6b5003c040 con 0x564dcbf4ee70 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.575+0000 7f6b623ac740 10 client.26330 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.575+0000 7f6b623ac740 10 client.26330 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.575+0000 7f6b623ac740 20 client.26330 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.575+0000 7f6b623ac740 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbf6f680 0x564dcbf71b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.575+0000 7f6b623ac740 4 client.26330 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:44.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.575+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564dcbeb6ae0 con 0x564dcbf6f680 2024-05-20T20:09:44.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.581+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbf6f680 0x564dcbf71b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.583+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbf6f680 0x564dcbf71b40 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d142e18536bfa695 server_cookie=b042d4f4264d2c85 in_seq=0 out_seq=0 2024-05-20T20:09:44.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.583+0000 7f6b3e000640 10 client.26330 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:44.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.585+0000 7f6b3e000640 1 -- 172.21.15.28:0/2066701949 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x564dcbeb6ae0 con 0x564dcbf6f680 2024-05-20T20:09:44.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.585+0000 7f6b3e000640 10 client.26330 handle_command_reply: tid=0 2024-05-20T20:09:44.587 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.585+0000 7f6b623ac740 1 client.26330 shutdown 2024-05-20T20:09:44.587 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.585+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf4ee70 msgr2=0x564dcbf4c010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.587 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.585+0000 7f6b623ac740 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf4ee70 0x564dcbf4c010 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f6b50033000 tx=0x7f6b5002a7e0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 shutdown_connections 2024-05-20T20:09:44.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbf6f680 0x564dcbf71b40 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf4ee70 0x564dcbf4c010 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf445b0 0x564dcbf4bab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 --2- 172.21.15.28:0/2066701949 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf43c90 0x564dcbf4e7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 >> 172.21.15.28:0/2066701949 conn(0x564dcbf401a0 msgr2=0x564dcbf45f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:44.589 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b3e000640 0 client.26330 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:44.589 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 shutdown_connections 2024-05-20T20:09:44.589 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 1 -- 172.21.15.28:0/2066701949 wait complete. 2024-05-20T20:09:44.589 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.587+0000 7f6b623ac740 20 client.26330 trim_cache size 0 max 16384 2024-05-20T20:09:44.623 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.622+0000 7f6b623ac740 1 Processor -- start 2024-05-20T20:09:44.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.622+0000 7f6b623ac740 1 -- start start 2024-05-20T20:09:44.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.623+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf43c90 0x564dcbf6bb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.623+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbf3f190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.623+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf4e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.623+0000 7f6b623ac740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6b581ac4f0 con 0x564dcbf445b0 2024-05-20T20:09:44.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.623+0000 7f6b623ac740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6b581ae010 con 0x564dcbf43c90 2024-05-20T20:09:44.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.623+0000 7f6b623ac740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6b581acba0 con 0x564dcbf4ee70 2024-05-20T20:09:44.625 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf4e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.625 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf4e2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47934/0 (socket says 172.21.15.28:47934) 2024-05-20T20:09:44.625 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 -- 172.21.15.28:0/1693145190 learned_addr learned my addr 172.21.15.28:0/1693145190 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:44.625 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b57e00640 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbf3f190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.625 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 -- 172.21.15.28:0/1693145190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf43c90 msgr2=0x564dcbf6bb40 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:44.626 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf43c90 0x564dcbf6bb40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.626 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 -- 172.21.15.28:0/1693145190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 msgr2=0x564dcbf3f190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:44.626 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbf3f190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.626 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.624+0000 7f6b5d600640 1 -- 172.21.15.28:0/1693145190 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564dcbf3e6b0 con 0x564dcbf4ee70 2024-05-20T20:09:44.626 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.625+0000 7f6b5d600640 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf4e2f0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f6b5004bc90 tx=0x7f6b500308b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3af5471d8e4b66cd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:44.626 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.625+0000 7f6b49200640 1 -- 172.21.15.28:0/1693145190 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b50063070 con 0x564dcbf4ee70 2024-05-20T20:09:44.627 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.625+0000 7f6b49200640 1 -- 172.21.15.28:0/1693145190 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6b50050070 con 0x564dcbf4ee70 2024-05-20T20:09:44.627 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.625+0000 7f6b49200640 1 -- 172.21.15.28:0/1693145190 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b50044070 con 0x564dcbf4ee70 2024-05-20T20:09:44.627 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.626+0000 7f6b623ac740 1 -- 172.21.15.28:0/1693145190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 msgr2=0x564dcbf4e2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.627 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.626+0000 7f6b623ac740 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf4e2f0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f6b5004bc90 tx=0x7f6b500308b0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.628 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.627+0000 7f6b623ac740 1 -- 172.21.15.28:0/1693145190 shutdown_connections 2024-05-20T20:09:44.628 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.627+0000 7f6b623ac740 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbf4e2f0 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.628 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.627+0000 7f6b623ac740 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbf3f190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.628 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.627+0000 7f6b623ac740 1 --2- 172.21.15.28:0/1693145190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbf43c90 0x564dcbf6bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.628 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.627+0000 7f6b623ac740 1 -- 172.21.15.28:0/1693145190 >> 172.21.15.28:0/1693145190 conn(0x564dcbf401a0 msgr2=0x564dcbf3ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:44.629 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.628+0000 7f6b623ac740 1 -- 172.21.15.28:0/1693145190 shutdown_connections 2024-05-20T20:09:44.629 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.629+0000 7f6b623ac740 1 -- 172.21.15.28:0/1693145190 wait complete. 2024-05-20T20:09:44.630 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.629+0000 7f6b623ac740 1 -- start start 2024-05-20T20:09:44.630 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 1 Processor -- start 2024-05-20T20:09:44.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbfd2460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbfd29a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbfd2ee0 0x564dcbf66d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6b581ad940 con 0x564dcbf445b0 2024-05-20T20:09:44.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6b581ad270 con 0x564dcbfd2ee0 2024-05-20T20:09:44.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6b581a0ac0 con 0x564dcbf4ee70 2024-05-20T20:09:44.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.630+0000 7f6b623ac740 10 client.0 _ll_register_callbacks cb 0x7f6b4c005e30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:44.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbfd2460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5d600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbfd2ee0 0x564dcbf66d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbfd2460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35916/0 (socket says 172.21.15.28:35916) 2024-05-20T20:09:44.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 -- 172.21.15.28:0/3175786698 learned_addr learned my addr 172.21.15.28:0/3175786698 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:44.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b57e00640 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbfd29a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 -- 172.21.15.28:0/3175786698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 msgr2=0x564dcbfd29a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:44.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbfd29a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 -- 172.21.15.28:0/3175786698 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbfd2ee0 msgr2=0x564dcbf66d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:44.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbfd2ee0 0x564dcbf66d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.631+0000 7f6b5cc00640 1 -- 172.21.15.28:0/3175786698 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564dcbf673c0 con 0x564dcbf445b0 2024-05-20T20:09:44.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.632+0000 7f6b5cc00640 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbfd2460 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f6b4c0a7100 tx=0x7f6b4c0a98b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7f5d8df230afea7d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:44.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.632+0000 7f6b3e000640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:44.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.632+0000 7f6b3e000640 1 -- 172.21.15.28:0/3175786698 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b4c0a99a0 con 0x564dcbf445b0 2024-05-20T20:09:44.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.632+0000 7f6b3e000640 1 -- 172.21.15.28:0/3175786698 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f6b4c0e6050 con 0x564dcbf445b0 2024-05-20T20:09:44.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.633+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6b50050070 con 0x564dcbf445b0 2024-05-20T20:09:44.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.633+0000 7f6b3e000640 1 -- 172.21.15.28:0/3175786698 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6b4c0aa590 con 0x564dcbf445b0 2024-05-20T20:09:44.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.633+0000 7f6b623ac740 10 client.16140 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:44.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.633+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564dcbf686f0 con 0x564dcbf445b0 2024-05-20T20:09:44.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b3e000640 1 -- 172.21.15.28:0/3175786698 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6b50050070 con 0x564dcbf445b0 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b3e000640 1 -- 172.21.15.28:0/3175786698 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f6b4c0e9030 con 0x564dcbf445b0 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b623ac740 10 client.16140 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b623ac740 10 client.16140 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b623ac740 20 client.16140 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbfeb600 0x564dcbfeda20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b623ac740 4 client.16140 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.634+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564dcbeb02c0 con 0x564dcbfeb600 2024-05-20T20:09:44.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.636+0000 7f6b57e00640 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbfeb600 0x564dcbfeda20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:44.639 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.638+0000 7f6b57e00640 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbfeb600 0x564dcbfeda20 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f34365afce88408a server_cookie=d234337d797382e2 in_seq=0 out_seq=0 2024-05-20T20:09:44.639 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.639+0000 7f6b3e000640 10 client.16140 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:44.663 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.662+0000 7f6b3e000640 1 -- 172.21.15.28:0/3175786698 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x564dcbeb02c0 con 0x564dcbfeb600 2024-05-20T20:09:44.664 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.662+0000 7f6b3e000640 10 client.16140 handle_command_reply: tid=0 2024-05-20T20:09:44.664 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.662+0000 7f6b623ac740 1 client.16140 shutdown 2024-05-20T20:09:44.664 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.663+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 msgr2=0x564dcbfd2460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.664 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.663+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbfd2460 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f6b4c0a7100 tx=0x7f6b4c0a98b0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 shutdown_connections 2024-05-20T20:09:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564dcbfeb600 0x564dcbfeda20 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564dcbfd2ee0 0x564dcbf66d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564dcbf4ee70 0x564dcbfd29a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 --2- 172.21.15.28:0/3175786698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564dcbf445b0 0x564dcbfd2460 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 >> 172.21.15.28:0/3175786698 conn(0x564dcbf401a0 msgr2=0x564dcbf3f310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:44.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b3e000640 0 client.16140 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 shutdown_connections 2024-05-20T20:09:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 1 -- 172.21.15.28:0/3175786698 wait complete. 2024-05-20T20:09:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.664+0000 7f6b623ac740 20 client.16140 trim_cache size 0 max 16384 2024-05-20T20:09:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:09:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:09:44.666 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:09:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.666+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6b400b2a90 msgr2=0x7f6b400b4f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.666+0000 7f6b5f600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6b400b2a90 0x7f6b400b4f50 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f6b50004af0 tx=0x7f6b50005fd0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.666+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58101ad0 msgr2=0x7f6b581a53e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:44.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.666+0000 7f6b5f600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58101ad0 0x7f6b581a53e0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f6b4c00e9f0 tx=0x7f6b4c00eeb0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.668+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 shutdown_connections 2024-05-20T20:09:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.668+0000 7f6b5f600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6b400b2a90 0x7f6b400b4f50 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.668+0000 7f6b5f600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6b58106df0 0x7f6b5819f5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.668+0000 7f6b5f600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6b581023f0 0x7f6b581a5920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.668+0000 7f6b5f600640 1 --2- 172.21.15.28:0/333804341 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6b58101ad0 0x7f6b581a53e0 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:44.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.668+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 >> 172.21.15.28:0/333804341 conn(0x7f6b58078ad0 msgr2=0x7f6b58104880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:44.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.669+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 shutdown_connections 2024-05-20T20:09:44.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:44.669+0000 7f6b5f600640 1 -- 172.21.15.28:0/333804341 wait complete. 2024-05-20T20:09:45.060 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:44 smithi028 ceph-mon[31178]: pgmap v126: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 339 MiB/s wr, 1.40k op/s 2024-05-20T20:09:45.177 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:44 smithi154 ceph-mon[36314]: pgmap v126: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 339 MiB/s wr, 1.40k op/s 2024-05-20T20:09:45.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:44 smithi033 ceph-mon[36179]: pgmap v126: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 339 MiB/s wr, 1.40k op/s 2024-05-20T20:09:46.309 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:45 smithi028 ceph-mon[31178]: pgmap v127: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 277 MiB/s wr, 1.01k op/s 2024-05-20T20:09:46.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:45 smithi154 ceph-mon[36314]: pgmap v127: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 277 MiB/s wr, 1.01k op/s 2024-05-20T20:09:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:45 smithi033 ceph-mon[36179]: pgmap v127: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 277 MiB/s wr, 1.01k op/s 2024-05-20T20:09:48.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.433+0000 7fa6f0800700 1 -- 172.21.15.28:0/1787288408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e811cd20 msgr2=0x7fa6e811f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.434+0000 7fa6f0800700 1 --2- 172.21.15.28:0/1787288408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e811cd20 0x7fa6e811f110 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fa6dc0098f0 tx=0x7fa6dc009c00 comp rx=0 tx=0).stop 2024-05-20T20:09:48.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.434+0000 7fa6f0800700 1 -- 172.21.15.28:0/1787288408 shutdown_connections 2024-05-20T20:09:48.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.434+0000 7fa6f0800700 1 --2- 172.21.15.28:0/1787288408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e811cd20 0x7fa6e811f110 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.435+0000 7fa6f0800700 1 --2- 172.21.15.28:0/1787288408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e8111d20 0x7fa6e810a540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.435+0000 7fa6f0800700 1 --2- 172.21.15.28:0/1787288408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e810aa80 0x7fa6e810af00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.435+0000 7fa6f0800700 1 -- 172.21.15.28:0/1787288408 >> 172.21.15.28:0/1787288408 conn(0x7fa6e806e6b0 msgr2=0x7fa6e806eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:48.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.436+0000 7fa6f0800700 1 -- 172.21.15.28:0/1787288408 shutdown_connections 2024-05-20T20:09:48.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.438+0000 7fa6f0800700 1 -- 172.21.15.28:0/1787288408 wait complete. 2024-05-20T20:09:48.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.441+0000 7fa6f0800700 1 Processor -- start 2024-05-20T20:09:48.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.441+0000 7fa6f0800700 1 -- start start 2024-05-20T20:09:48.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.442+0000 7fa6f0800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e810aa80 0x7fa6e81ae6e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.442+0000 7fa6f0800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e8111d20 0x7fa6e81aec20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.442+0000 7fa6f0800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e811cd20 0x7fa6e81a88a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.442+0000 7fa6f0800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e8122600 con 0x7fa6e811cd20 2024-05-20T20:09:48.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.442+0000 7fa6f0800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e8122480 con 0x7fa6e810aa80 2024-05-20T20:09:48.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.442+0000 7fa6f0800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e8122780 con 0x7fa6e8111d20 2024-05-20T20:09:48.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.444+0000 7fa6ede00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e810aa80 0x7fa6e81ae6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.444+0000 7fa6ede00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e810aa80 0x7fa6e81ae6e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37966/0 (socket says 172.21.15.28:37966) 2024-05-20T20:09:48.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.444+0000 7fa6ede00700 1 -- 172.21.15.28:0/103526266 learned_addr learned my addr 172.21.15.28:0/103526266 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:48.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.445+0000 7fa6ed400700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e8111d20 0x7fa6e81aec20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.445+0000 7fa6ee800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e811cd20 0x7fa6e81a88a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.448+0000 7fa6ee800700 1 -- 172.21.15.28:0/103526266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e8111d20 msgr2=0x7fa6e81aec20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.448+0000 7fa6ee800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e8111d20 0x7fa6e81aec20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.448+0000 7fa6ee800700 1 -- 172.21.15.28:0/103526266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e810aa80 msgr2=0x7fa6e81ae6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.448+0000 7fa6ee800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e810aa80 0x7fa6e81ae6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.448+0000 7fa6ee800700 1 -- 172.21.15.28:0/103526266 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa6e81a9000 con 0x7fa6e811cd20 2024-05-20T20:09:48.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.449+0000 7fa6ee800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e811cd20 0x7fa6e81a88a0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7fa6dc009810 tx=0x7fa6dc00fa00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:48.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.450+0000 7fa6e2a00700 1 -- 172.21.15.28:0/103526266 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6dc027070 con 0x7fa6e811cd20 2024-05-20T20:09:48.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.450+0000 7fa6e2a00700 1 -- 172.21.15.28:0/103526266 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6dc0056b0 con 0x7fa6e811cd20 2024-05-20T20:09:48.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.450+0000 7fa6e2a00700 1 -- 172.21.15.28:0/103526266 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6dc004950 con 0x7fa6e811cd20 2024-05-20T20:09:48.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.451+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa6e81a9290 con 0x7fa6e811cd20 2024-05-20T20:09:48.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.452+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa6e81b55b0 con 0x7fa6e811cd20 2024-05-20T20:09:48.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.466+0000 7fa6e2a00700 1 -- 172.21.15.28:0/103526266 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa6dc00b680 con 0x7fa6e811cd20 2024-05-20T20:09:48.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.469+0000 7fa6fb0adb80 1 Processor -- start 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.469+0000 7fa6fb0adb80 1 -- start start 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2d9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6e2a00700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6d00b2bc0 0x7fa6d00b5080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2df30 0x56315df27b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df285c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56315de85e40 con 0x56315df2d610 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56315de759e0 con 0x56315df2df30 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56315de79a20 con 0x56315df28140 2024-05-20T20:09:48.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.470+0000 7fa6e2a00700 1 -- 172.21.15.28:0/103526266 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa6dc0e31f0 con 0x7fa6e811cd20 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.472+0000 7fa6ee800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df285c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.473+0000 7fa6ee800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df285c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52574/0 (socket says 172.21.15.28:52574) 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.473+0000 7fa6ee800700 1 -- 172.21.15.28:0/3132037038 learned_addr learned my addr 172.21.15.28:0/3132037038 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.474+0000 7fa6ed400700 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2df30 0x56315df27b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.474+0000 7fa6ed400700 1 -- 172.21.15.28:0/3132037038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 msgr2=0x56315df285c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.474+0000 7fa6ed400700 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df285c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.474+0000 7fa6ed400700 1 -- 172.21.15.28:0/3132037038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 msgr2=0x56315df2d9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.474+0000 7fa6ed400700 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2d9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.474+0000 7fa6ed400700 1 -- 172.21.15.28:0/3132037038 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56315de3e420 con 0x56315df2df30 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.475+0000 7fa6ed400700 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2df30 0x56315df27b20 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8009cd0 tx=0x7fa6d8012450 comp rx=0 tx=0).ready entity=mon.2 client_cookie=97ea3ebd47d382a6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.475+0000 7fa6d7400700 1 -- 172.21.15.28:0/3132037038 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d8012d80 con 0x56315df2df30 2024-05-20T20:09:48.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.476+0000 7fa6d7400700 1 -- 172.21.15.28:0/3132037038 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6d800f8e0 con 0x56315df2df30 2024-05-20T20:09:48.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.476+0000 7fa6d7400700 1 -- 172.21.15.28:0/3132037038 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d801a940 con 0x56315df2df30 2024-05-20T20:09:48.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.476+0000 7fa6ed400700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6d00b2bc0 0x7fa6d00b5080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.477+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/3132037038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2df30 msgr2=0x56315df27b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.477+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2df30 0x56315df27b20 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8009cd0 tx=0x7fa6d8012450 comp rx=0 tx=0).stop 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.480+0000 7fa6ed400700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6d00b2bc0 0x7fa6d00b5080 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8009cd0 tx=0x7fa6d8012450 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.480+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/3132037038 shutdown_connections 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.480+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df285c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.480+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2d9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.480+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/3132037038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2df30 0x56315df27b20 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.480+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/3132037038 >> 172.21.15.28:0/3132037038 conn(0x56315de55be0 msgr2=0x56315df22690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.481+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/3132037038 shutdown_connections 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.481+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/3132037038 wait complete. 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.481+0000 7fa6fb0adb80 1 -- start start 2024-05-20T20:09:48.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.482+0000 7fa6fb0adb80 1 Processor -- start 2024-05-20T20:09:48.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.488+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df33c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.488+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2d610 0x56315df31030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.488+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df31570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.488+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56315dd7bfa0 con 0x56315df2df30 2024-05-20T20:09:48.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.488+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56315de16760 con 0x56315df28140 2024-05-20T20:09:48.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.488+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56315df459f0 con 0x56315df2d610 2024-05-20T20:09:48.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.488+0000 7fa6fb0adb80 10 client.0 _ll_register_callbacks cb 0x56315de1fc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:48.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.489+0000 7fa6ee800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df31570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.490+0000 7fa6ee800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df31570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42336/0 (socket says 172.21.15.28:42336) 2024-05-20T20:09:48.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.490+0000 7fa6ee800700 1 -- 172.21.15.28:0/2009035975 learned_addr learned my addr 172.21.15.28:0/2009035975 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:48.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.491+0000 7fa6ee800700 1 -- 172.21.15.28:0/2009035975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2d610 msgr2=0x56315df31030 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:09:48.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.491+0000 7fa6ee800700 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2d610 0x56315df31030 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.491+0000 7fa6ee800700 1 -- 172.21.15.28:0/2009035975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 msgr2=0x56315df33c90 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:48.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.491+0000 7fa6ee800700 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df33c90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.491+0000 7fa6ee800700 1 -- 172.21.15.28:0/2009035975 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56315ddc58d0 con 0x56315df2df30 2024-05-20T20:09:48.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.493+0000 7fa6ee800700 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df31570 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fa6dc00c010 tx=0x7fa6dc0aa390 comp rx=0 tx=0).ready entity=mon.0 client_cookie=293fccaf922fa6d7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:48.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.493+0000 7fa6cbe00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:09:48.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.494+0000 7fa6cbe00700 1 -- 172.21.15.28:0/2009035975 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6dc0aa020 con 0x56315df2df30 2024-05-20T20:09:48.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.494+0000 7fa6cbe00700 1 -- 172.21.15.28:0/2009035975 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa6dc0e0b70 con 0x56315df2df30 2024-05-20T20:09:48.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.494+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa6d800f8e0 con 0x56315df2df30 2024-05-20T20:09:48.498 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.497+0000 7fa6cbe00700 1 -- 172.21.15.28:0/2009035975 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa6dc0e0d70 con 0x56315df2df30 2024-05-20T20:09:48.498 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.497+0000 7fa6fb0adb80 10 client.16149 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:48.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.498+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56315de46060 con 0x56315df2df30 2024-05-20T20:09:48.501 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.500+0000 7fa6cbe00700 1 -- 172.21.15.28:0/2009035975 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6dc0e0d70 con 0x56315df2df30 2024-05-20T20:09:48.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.501+0000 7fa6cbe00700 1 -- 172.21.15.28:0/2009035975 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa6dc02a020 con 0x56315df2df30 2024-05-20T20:09:48.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.504+0000 7fa6fb0adb80 10 client.16149 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:48.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.505+0000 7fa6fb0adb80 10 client.16149 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:09:48.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.505+0000 7fa6fb0adb80 20 client.16149 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:48.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.505+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315df52490 0x56315df548d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.505+0000 7fa6fb0adb80 4 client.16149 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:48.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.505+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56315de87d90 con 0x56315df52490 2024-05-20T20:09:48.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.506+0000 7fa6ede00700 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315df52490 0x56315df548d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.507+0000 7fa6ede00700 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315df52490 0x56315df548d0 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=8a680bf8c035f8d8 server_cookie=953a0f7cd8b7d5bf in_seq=0 out_seq=0 2024-05-20T20:09:48.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.508+0000 7fa6cbe00700 10 client.16149 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:09:48.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.508+0000 7fa6cbe00700 1 -- 172.21.15.28:0/2009035975 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56315de87d90 con 0x56315df52490 2024-05-20T20:09:48.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.508+0000 7fa6cbe00700 10 client.16149 handle_command_reply: tid=0 2024-05-20T20:09:48.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 client.16149 shutdown 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 msgr2=0x56315df31570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df31570 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fa6dc00c010 tx=0x7fa6dc0aa390 comp rx=0 tx=0).stop 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 shutdown_connections 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df31570 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df33c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315df52490 0x56315df548d0 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2009035975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2d610 0x56315df31030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.509+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 >> 172.21.15.28:0/2009035975 conn(0x56315de55be0 msgr2=0x56315df21d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:48.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.510+0000 7fa6cbe00700 0 client.16149 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:09:48.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.510+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 shutdown_connections 2024-05-20T20:09:48.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.510+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2009035975 wait complete. 2024-05-20T20:09:48.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.511+0000 7fa6fb0adb80 20 client.16149 trim_cache size 0 max 16384 2024-05-20T20:09:48.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.544+0000 7fa6fb0adb80 1 Processor -- start 2024-05-20T20:09:48.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.544+0000 7fa6fb0adb80 1 -- start start 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df2b790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2ec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2df30 0x56315df224f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e8119840 con 0x56315df2d610 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e81b5e00 con 0x56315df28140 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e8119f10 con 0x56315df2df30 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6ede00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df2b790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6ede00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df2b790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37990/0 (socket says 172.21.15.28:37990) 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.545+0000 7fa6ede00700 1 -- 172.21.15.28:0/2729248363 learned_addr learned my addr 172.21.15.28:0/2729248363 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ed400700 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ee800700 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2df30 0x56315df224f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ede00700 1 -- 172.21.15.28:0/2729248363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2df30 msgr2=0x56315df224f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:48.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ede00700 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2df30 0x56315df224f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ede00700 1 -- 172.21.15.28:0/2729248363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 msgr2=0x56315df2ec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:48.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ede00700 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2ec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ede00700 1 -- 172.21.15.28:0/2729248363 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56315df23890 con 0x56315df28140 2024-05-20T20:09:48.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ee800700 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2df30 0x56315df224f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:48.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ed400700 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:48.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6ede00700 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df2b790 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fa6e4009fd0 tx=0x7fa6e4004210 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fa0951020b294330 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.546+0000 7fa6d6a00700 1 -- 172.21.15.28:0/2729248363 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6e40090d0 con 0x56315df28140 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6d6a00700 1 -- 172.21.15.28:0/2729248363 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6e4009270 con 0x56315df28140 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6d6a00700 1 -- 172.21.15.28:0/2729248363 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6e4005340 con 0x56315df28140 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2729248363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 msgr2=0x56315df2b790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df2b790 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fa6e4009fd0 tx=0x7fa6e4004210 comp rx=0 tx=0).stop 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2729248363 shutdown_connections 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df2df30 0x56315df224f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df28140 0x56315df2b790 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/2729248363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2d610 0x56315df2ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.547+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2729248363 >> 172.21.15.28:0/2729248363 conn(0x56315de55be0 msgr2=0x56315df2f920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:48.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.548+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2729248363 shutdown_connections 2024-05-20T20:09:48.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.548+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/2729248363 wait complete. 2024-05-20T20:09:48.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.548+0000 7fa6fb0adb80 1 -- start start 2024-05-20T20:09:48.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.548+0000 7fa6fb0adb80 1 Processor -- start 2024-05-20T20:09:48.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.549+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df4b4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.549+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2d610 0x56315df59bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.549+0000 7fa6fb0adb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df5a2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.550+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e81b5780 con 0x56315df2df30 2024-05-20T20:09:48.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.550+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e811a5e0 con 0x56315df2d610 2024-05-20T20:09:48.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.550+0000 7fa6fb0adb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e81177e0 con 0x56315df28140 2024-05-20T20:09:48.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.550+0000 7fa6fb0adb80 10 client.0 _ll_register_callbacks cb 0x56315de1fc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:48.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.550+0000 7fa6ed400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2d610 0x56315df59bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.550+0000 7fa6ed400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2d610 0x56315df59bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37996/0 (socket says 172.21.15.28:37996) 2024-05-20T20:09:48.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.550+0000 7fa6ed400700 1 -- 172.21.15.28:0/1077308218 learned_addr learned my addr 172.21.15.28:0/1077308218 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:48.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.551+0000 7fa6ed400700 1 -- 172.21.15.28:0/1077308218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 msgr2=0x56315df4b4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:48.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.551+0000 7fa6ed400700 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df4b4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.552 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.551+0000 7fa6ed400700 1 -- 172.21.15.28:0/1077308218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 msgr2=0x56315df5a2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:48.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.551+0000 7fa6ee800700 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df5a2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.551+0000 7fa6ed400700 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df5a2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.551+0000 7fa6ed400700 1 -- 172.21.15.28:0/1077308218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56315dfbc0b0 con 0x56315df2d610 2024-05-20T20:09:48.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.551+0000 7fa6ee800700 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df5a2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:48.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.552+0000 7fa6ed400700 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2d610 0x56315df59bd0 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7fa6d80059d0 tx=0x7fa6d8004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=de3ac11ecdd22467 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:48.553 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.552+0000 7fa6cbe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:48.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.553+0000 7fa6cbe00700 1 -- 172.21.15.28:0/1077308218 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d8020040 con 0x56315df2d610 2024-05-20T20:09:48.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.553+0000 7fa6cbe00700 1 -- 172.21.15.28:0/1077308218 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa6d802d070 con 0x56315df2d610 2024-05-20T20:09:48.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.553+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa6e4009270 con 0x56315df2d610 2024-05-20T20:09:48.554 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.553+0000 7fa6cbe00700 1 -- 172.21.15.28:0/1077308218 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa6d803d050 con 0x56315df2d610 2024-05-20T20:09:48.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.553+0000 7fa6fb0adb80 10 client.26348 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:48.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.553+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56315dfbd210 con 0x56315df2d610 2024-05-20T20:09:48.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.554+0000 7fa6cbe00700 1 -- 172.21.15.28:0/1077308218 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6e4009270 con 0x56315df2d610 2024-05-20T20:09:48.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6cbe00700 1 -- 172.21.15.28:0/1077308218 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa6d8019070 con 0x56315df2d610 2024-05-20T20:09:48.555 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6fb0adb80 10 client.26348 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:48.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6fb0adb80 10 client.26348 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:09:48.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6fb0adb80 20 client.26348 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:48.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315dfbd800 0x56315dfbfc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:48.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6fb0adb80 4 client.26348 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:09:48.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56315de81570 con 0x56315dfbd800 2024-05-20T20:09:48.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.555+0000 7fa6ede00700 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315dfbd800 0x56315dfbfc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:48.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.556+0000 7fa6ede00700 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315dfbd800 0x56315dfbfc20 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=6da10115941a0969 server_cookie=31aa0926ddd65084 in_seq=0 out_seq=0 2024-05-20T20:09:48.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.556+0000 7fa6cbe00700 10 client.26348 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:09:48.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.556+0000 7fa6cbe00700 1 -- 172.21.15.28:0/1077308218 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56315de81570 con 0x56315dfbd800 2024-05-20T20:09:48.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.556+0000 7fa6cbe00700 10 client.26348 handle_command_reply: tid=0 2024-05-20T20:09:48.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.556+0000 7fa6fb0adb80 1 client.26348 shutdown 2024-05-20T20:09:48.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.556+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2d610 msgr2=0x56315df59bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.556+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2d610 0x56315df59bd0 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7fa6d80059d0 tx=0x7fa6d8004750 comp rx=0 tx=0).stop 2024-05-20T20:09:48.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.557+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 shutdown_connections 2024-05-20T20:09:48.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.557+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56315df2df30 0x56315df5a2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.557+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x56315dfbd800 0x56315dfbfc20 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.557+0000 7fa6cbe00700 0 client.26348 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:09:48.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.557+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56315df28140 0x56315df4b4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.558+0000 7fa6fb0adb80 1 --2- 172.21.15.28:0/1077308218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56315df2d610 0x56315df59bd0 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.558+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 >> 172.21.15.28:0/1077308218 conn(0x56315de55be0 msgr2=0x56315df301b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:48.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.559+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 shutdown_connections 2024-05-20T20:09:48.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.559+0000 7fa6fb0adb80 1 -- 172.21.15.28:0/1077308218 wait complete. 2024-05-20T20:09:48.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.559+0000 7fa6fb0adb80 20 client.26348 trim_cache size 0 max 16384 2024-05-20T20:09:48.560 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:09:48.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.560+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6d00b2bc0 msgr2=0x7fa6d00b5080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.560+0000 7fa6f0800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6d00b2bc0 0x7fa6d00b5080 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8009cd0 tx=0x7fa6d8012450 comp rx=0 tx=0).stop 2024-05-20T20:09:48.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.560+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e811cd20 msgr2=0x7fa6e81a88a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:48.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.560+0000 7fa6f0800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e811cd20 0x7fa6e81a88a0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7fa6dc009810 tx=0x7fa6dc00fa00 comp rx=0 tx=0).stop 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 shutdown_connections 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e811cd20 0x7fa6e81a88a0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6d00b2bc0 0x7fa6d00b5080 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e810aa80 0x7fa6e81ae6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 --2- 172.21.15.28:0/103526266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e8111d20 0x7fa6e81aec20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 >> 172.21.15.28:0/103526266 conn(0x7fa6e806e6b0 msgr2=0x7fa6e811ad10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 shutdown_connections 2024-05-20T20:09:48.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:48.561+0000 7fa6f0800700 1 -- 172.21.15.28:0/103526266 wait complete. 2024-05-20T20:09:48.787 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:48 smithi028 ceph-mon[31178]: pgmap v128: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 248 MiB/s wr, 865 op/s 2024-05-20T20:09:49.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:48 smithi154 ceph-mon[36314]: pgmap v128: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 248 MiB/s wr, 865 op/s 2024-05-20T20:09:49.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:48 smithi033 ceph-mon[36179]: pgmap v128: 97 pgs: 97 active+clean; 16 GiB data, 3.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 248 MiB/s wr, 865 op/s 2024-05-20T20:09:50.632 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:09:51.057 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:50 smithi028 ceph-mon[31178]: pgmap v129: 97 pgs: 97 active+clean; 17 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 380 MiB/s wr, 1.33k op/s 2024-05-20T20:09:51.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:50 smithi154 ceph-mon[36314]: pgmap v129: 97 pgs: 97 active+clean; 17 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 380 MiB/s wr, 1.33k op/s 2024-05-20T20:09:51.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:50 smithi033 ceph-mon[36179]: pgmap v129: 97 pgs: 97 active+clean; 17 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 380 MiB/s wr, 1.33k op/s 2024-05-20T20:09:51.430 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:53.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:52 smithi028 ceph-mon[31178]: pgmap v130: 97 pgs: 97 active+clean; 17 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 255 MiB/s wr, 869 op/s 2024-05-20T20:09:53.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:52 smithi154 ceph-mon[36314]: pgmap v130: 97 pgs: 97 active+clean; 17 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 255 MiB/s wr, 869 op/s 2024-05-20T20:09:53.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:52 smithi033 ceph-mon[36179]: pgmap v130: 97 pgs: 97 active+clean; 17 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 255 MiB/s wr, 869 op/s 2024-05-20T20:09:55.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:54 smithi154 ceph-mon[36314]: pgmap v131: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 354 MiB/s wr, 1.23k op/s 2024-05-20T20:09:55.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:54 smithi033 ceph-mon[36179]: pgmap v131: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 354 MiB/s wr, 1.23k op/s 2024-05-20T20:09:55.254 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:54 smithi028 ceph-mon[31178]: pgmap v131: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 354 MiB/s wr, 1.23k op/s 2024-05-20T20:09:56.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 -- 172.21.15.28:0/3067032114 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 msgr2=0x7fa6141081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 --2- 172.21.15.28:0/3067032114 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa6141081c0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7fa6000099c0 tx=0x7fa600009cd0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 -- 172.21.15.28:0/3067032114 shutdown_connections 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 --2- 172.21.15.28:0/3067032114 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6141088f0 0x7fa614110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 --2- 172.21.15.28:0/3067032114 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6141037f0 0x7fa614103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 --2- 172.21.15.28:0/3067032114 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa6141081c0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 -- 172.21.15.28:0/3067032114 >> 172.21.15.28:0/3067032114 conn(0x7fa61407a6a0 msgr2=0x7fa61407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 -- 172.21.15.28:0/3067032114 shutdown_connections 2024-05-20T20:09:56.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.092+0000 7fa619600700 1 -- 172.21.15.28:0/3067032114 wait complete. 2024-05-20T20:09:56.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.093+0000 7fa619600700 1 Processor -- start 2024-05-20T20:09:56.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.093+0000 7fa619600700 1 -- start start 2024-05-20T20:09:56.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.093+0000 7fa619600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6141037f0 0x7fa6140770a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa619600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa614072180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa619600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6141088f0 0x7fa6140726c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa619600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa614113700 con 0x7fa6141088f0 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa619600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa614113580 con 0x7fa6141041a0 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa619600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa614113880 con 0x7fa6141037f0 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa614072180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa614072180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38010/0 (socket says 172.21.15.28:38010) 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 -- 172.21.15.28:0/2513142649 learned_addr learned my addr 172.21.15.28:0/2513142649 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612a00700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6141037f0 0x7fa6140770a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa613400700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6141088f0 0x7fa6140726c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 -- 172.21.15.28:0/2513142649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6141037f0 msgr2=0x7fa6140770a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6141037f0 0x7fa6140770a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 -- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6141088f0 msgr2=0x7fa6140726c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6141088f0 0x7fa6140726c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa612000700 1 -- 172.21.15.28:0/2513142649 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa614072d90 con 0x7fa6141041a0 2024-05-20T20:09:56.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.094+0000 7fa613400700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6141088f0 0x7fa6140726c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:09:56.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.095+0000 7fa612000700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa614072180 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7fa600006010 tx=0x7fa600005280 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:56.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.095+0000 7fa60b400700 1 -- 172.21.15.28:0/2513142649 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa600026070 con 0x7fa6141041a0 2024-05-20T20:09:56.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.095+0000 7fa60b400700 1 -- 172.21.15.28:0/2513142649 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa600003c70 con 0x7fa6141041a0 2024-05-20T20:09:56.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.095+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa6141b0c20 con 0x7fa6141041a0 2024-05-20T20:09:56.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.095+0000 7fa60b400700 1 -- 172.21.15.28:0/2513142649 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60000fb40 con 0x7fa6141041a0 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.095+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa6141b0fc0 con 0x7fa6141041a0 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.099+0000 7fa60b400700 1 -- 172.21.15.28:0/2513142649 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa600029070 con 0x7fa6141041a0 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 Processor -- start 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 -- start start 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa60b400700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa5f80b2a70 0x7fa5f80b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a569509f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694ab20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5694b5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557a568a8e40 con 0x557a56950610 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557a568989e0 con 0x557a56950f30 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.102+0000 7fa624086b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557a5689ca20 con 0x557a5694b140 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.103+0000 7fa60b400700 1 -- 172.21.15.28:0/2513142649 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa6000e2e70 con 0x7fa6141041a0 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.103+0000 7fa612000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694ab20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.103+0000 7fa612000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694ab20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38022/0 (socket says 172.21.15.28:38022) 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.103+0000 7fa612000700 1 -- 172.21.15.28:0/1813890926 learned_addr learned my addr 172.21.15.28:0/1813890926 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.103+0000 7fa612000700 1 -- 172.21.15.28:0/1813890926 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 msgr2=0x557a5694b5c0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.103+0000 7fa612000700 1 --2- 172.21.15.28:0/1813890926 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5694b5c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.104+0000 7fa612000700 1 -- 172.21.15.28:0/1813890926 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 msgr2=0x557a569509f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.104+0000 7fa612000700 1 --2- 172.21.15.28:0/1813890926 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a569509f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.104+0000 7fa612000700 1 -- 172.21.15.28:0/1813890926 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557a56861420 con 0x557a56950f30 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.105+0000 7fa613400700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa5f80b2a70 0x7fa5f80b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.105+0000 7fa612000700 1 --2- 172.21.15.28:0/1813890926 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694ab20 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7fa600009350 tx=0x7fa600005870 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6acdb6dda40bea53 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.106+0000 7fa5ffe00700 1 -- 172.21.15.28:0/1813890926 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6000ae320 con 0x557a56950f30 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.106+0000 7fa5ffe00700 1 -- 172.21.15.28:0/1813890926 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6000a6090 con 0x557a56950f30 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.106+0000 7fa5ffe00700 1 -- 172.21.15.28:0/1813890926 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6000aa650 con 0x557a56950f30 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.107+0000 7fa624086b80 1 -- 172.21.15.28:0/1813890926 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 msgr2=0x557a5694ab20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.107+0000 7fa624086b80 1 --2- 172.21.15.28:0/1813890926 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694ab20 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7fa600009350 tx=0x7fa600005870 comp rx=0 tx=0).stop 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.107+0000 7fa624086b80 1 -- 172.21.15.28:0/1813890926 shutdown_connections 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.107+0000 7fa624086b80 1 --2- 172.21.15.28:0/1813890926 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5694b5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.107+0000 7fa624086b80 1 --2- 172.21.15.28:0/1813890926 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a569509f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.107+0000 7fa624086b80 1 --2- 172.21.15.28:0/1813890926 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694ab20 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.107+0000 7fa624086b80 1 -- 172.21.15.28:0/1813890926 >> 172.21.15.28:0/1813890926 conn(0x557a56878be0 msgr2=0x557a56945690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.108+0000 7fa624086b80 1 -- 172.21.15.28:0/1813890926 shutdown_connections 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.108+0000 7fa624086b80 1 -- 172.21.15.28:0/1813890926 wait complete. 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.109+0000 7fa624086b80 1 -- start start 2024-05-20T20:09:56.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.109+0000 7fa624086b80 1 Processor -- start 2024-05-20T20:09:56.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.112+0000 7fa613400700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa5f80b2a70 0x7fa5f80b4f30 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x557a5694bb00 tx=0x7fa60c00b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.112+0000 7fa624086b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a5694b140 0x557a56956e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa624086b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950610 0x557a569541b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa624086b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a56950f30 0x557a569546f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa624086b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557a5679efa0 con 0x557a5694b140 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa624086b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557a56839760 con 0x557a56950610 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa624086b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557a569577a0 con 0x557a56950f30 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa624086b80 10 client.0 _ll_register_callbacks cb 0x557a56842c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa612000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950610 0x557a569541b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa612000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950610 0x557a569541b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38038/0 (socket says 172.21.15.28:38038) 2024-05-20T20:09:56.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa612000700 1 -- 172.21.15.28:0/119542966 learned_addr learned my addr 172.21.15.28:0/119542966 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:56.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.113+0000 7fa612a00700 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a5694b140 0x557a56956e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.115+0000 7fa612000700 1 -- 172.21.15.28:0/119542966 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a56950f30 msgr2=0x557a569546f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:56.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.115+0000 7fa612000700 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a56950f30 0x557a569546f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.115+0000 7fa612000700 1 -- 172.21.15.28:0/119542966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a5694b140 msgr2=0x557a56956e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:56.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.115+0000 7fa612000700 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a5694b140 0x557a56956e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.115+0000 7fa612000700 1 -- 172.21.15.28:0/119542966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557a567e88d0 con 0x557a56950610 2024-05-20T20:09:56.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.119+0000 7fa612000700 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950610 0x557a569541b0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fa600009990 tx=0x7fa6000aa6d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3f14573124804cd0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:56.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.119+0000 7fa5fcc00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:56.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.119+0000 7fa5fcc00700 1 -- 172.21.15.28:0/119542966 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6000ae540 con 0x557a56950610 2024-05-20T20:09:56.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.120+0000 7fa5fcc00700 1 -- 172.21.15.28:0/119542966 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa6000b2ce0 con 0x557a56950610 2024-05-20T20:09:56.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.120+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa6000a6090 con 0x557a56950610 2024-05-20T20:09:56.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.120+0000 7fa5fcc00700 1 -- 172.21.15.28:0/119542966 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa6000b2f60 con 0x557a56950610 2024-05-20T20:09:56.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.121+0000 7fa624086b80 10 client.26360 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:56.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.121+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557a56869060 con 0x557a56950610 2024-05-20T20:09:56.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.122+0000 7fa5fcc00700 1 -- 172.21.15.28:0/119542966 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6000a6090 con 0x557a56950610 2024-05-20T20:09:56.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.122+0000 7fa5fcc00700 1 -- 172.21.15.28:0/119542966 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa6000a5b40 con 0x557a56950610 2024-05-20T20:09:56.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.124+0000 7fa624086b80 10 client.26360 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:56.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.124+0000 7fa624086b80 10 client.26360 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:09:56.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.124+0000 7fa624086b80 20 client.26360 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:56.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.125+0000 7fa624086b80 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a56964270 0x557a56966660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.125+0000 7fa624086b80 4 client.26360 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:56.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.125+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557a568aad90 con 0x557a56964270 2024-05-20T20:09:56.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.125+0000 7fa612a00700 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a56964270 0x557a56966660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.126+0000 7fa612a00700 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a56964270 0x557a56966660 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=24bbd9b3beb4715d server_cookie=47ec59563c0a253b in_seq=0 out_seq=0 2024-05-20T20:09:56.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.127+0000 7fa5fcc00700 10 client.26360 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:09:56.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.128+0000 7fa5fcc00700 1 -- 172.21.15.28:0/119542966 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x557a568aad90 con 0x557a56964270 2024-05-20T20:09:56.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.128+0000 7fa5fcc00700 10 client.26360 handle_command_reply: tid=0 2024-05-20T20:09:56.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.128+0000 7fa624086b80 1 client.26360 shutdown 2024-05-20T20:09:56.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.129+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950610 msgr2=0x557a569541b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.129+0000 7fa624086b80 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950610 0x557a569541b0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7fa600009990 tx=0x7fa6000aa6d0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.130+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 shutdown_connections 2024-05-20T20:09:56.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.130+0000 7fa624086b80 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a56964270 0x557a56966660 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.131+0000 7fa624086b80 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a56950f30 0x557a569546f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.131+0000 7fa624086b80 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a5694b140 0x557a56956e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.131+0000 7fa624086b80 1 --2- 172.21.15.28:0/119542966 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950610 0x557a569541b0 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.131+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 >> 172.21.15.28:0/119542966 conn(0x557a56878be0 msgr2=0x557a56944d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:56.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.131+0000 7fa5fcc00700 0 client.26360 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:09:56.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.132+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 shutdown_connections 2024-05-20T20:09:56.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.132+0000 7fa624086b80 1 -- 172.21.15.28:0/119542966 wait complete. 2024-05-20T20:09:56.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.133+0000 7fa624086b80 20 client.26360 trim_cache size 0 max 16384 2024-05-20T20:09:56.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 Processor -- start 2024-05-20T20:09:56.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 -- start start 2024-05-20T20:09:56.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5694ab20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a56960800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694f9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6141b8770 con 0x557a56950610 2024-05-20T20:09:56.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6141ba2b0 con 0x557a56950f30 2024-05-20T20:09:56.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.168+0000 7fa624086b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6141b8e40 con 0x557a5694b140 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.169+0000 7fa613400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694f9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.169+0000 7fa612000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a56960800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.169+0000 7fa613400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694f9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38054/0 (socket says 172.21.15.28:38054) 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.169+0000 7fa613400700 1 -- 172.21.15.28:0/3880391219 learned_addr learned my addr 172.21.15.28:0/3880391219 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.170+0000 7fa612a00700 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5694ab20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.170+0000 7fa613400700 1 -- 172.21.15.28:0/3880391219 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 msgr2=0x557a5694ab20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.170+0000 7fa613400700 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5694ab20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.170+0000 7fa613400700 1 -- 172.21.15.28:0/3880391219 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 msgr2=0x557a56960800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:56.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.170+0000 7fa613400700 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a56960800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.170+0000 7fa613400700 1 -- 172.21.15.28:0/3880391219 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557a56954a20 con 0x557a56950f30 2024-05-20T20:09:56.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.171+0000 7fa612000700 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a56960800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:56.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.171+0000 7fa613400700 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694f9d0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7fa60c008010 tx=0x7fa60c013000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=17f6506778588ea6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:56.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.171+0000 7fa5ff400700 1 -- 172.21.15.28:0/3880391219 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60c01c050 con 0x557a56950f30 2024-05-20T20:09:56.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.171+0000 7fa5ff400700 1 -- 172.21.15.28:0/3880391219 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa60c026070 con 0x557a56950f30 2024-05-20T20:09:56.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.171+0000 7fa5ff400700 1 -- 172.21.15.28:0/3880391219 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60c0167d0 con 0x557a56950f30 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.172+0000 7fa624086b80 1 -- 172.21.15.28:0/3880391219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 msgr2=0x557a5694f9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.172+0000 7fa624086b80 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694f9d0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7fa60c008010 tx=0x7fa60c013000 comp rx=0 tx=0).stop 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.172+0000 7fa624086b80 1 -- 172.21.15.28:0/3880391219 shutdown_connections 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.172+0000 7fa624086b80 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5694f9d0 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.172+0000 7fa624086b80 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5694ab20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.172+0000 7fa624086b80 1 --2- 172.21.15.28:0/3880391219 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a56960800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.172+0000 7fa624086b80 1 -- 172.21.15.28:0/3880391219 >> 172.21.15.28:0/3880391219 conn(0x557a56878be0 msgr2=0x557a5694f2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.173+0000 7fa624086b80 1 -- 172.21.15.28:0/3880391219 shutdown_connections 2024-05-20T20:09:56.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.173+0000 7fa624086b80 1 -- 172.21.15.28:0/3880391219 wait complete. 2024-05-20T20:09:56.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.173+0000 7fa624086b80 1 -- start start 2024-05-20T20:09:56.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 1 Processor -- start 2024-05-20T20:09:56.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5696bac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a5696c000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5696c540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6141b9be0 con 0x557a56950610 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6141b9510 con 0x557a56950f30 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6141b6710 con 0x557a5694b140 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.174+0000 7fa624086b80 10 client.0 _ll_register_callbacks cb 0x557a56842c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5696c540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5696c540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38058/0 (socket says 172.21.15.28:38058) 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 -- 172.21.15.28:0/2690478019 learned_addr learned my addr 172.21.15.28:0/2690478019 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa612a00700 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5696bac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 -- 172.21.15.28:0/2690478019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 msgr2=0x557a5696bac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5696bac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 -- 172.21.15.28:0/2690478019 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 msgr2=0x557a5696c000 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a5696c000 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.175+0000 7fa613400700 1 -- 172.21.15.28:0/2690478019 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557a569d6680 con 0x557a56950f30 2024-05-20T20:09:56.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa613400700 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5696c540 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7fa60c009960 tx=0x7fa60c004f00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=40968b5534d20e6b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:56.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa5fcc00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:09:56.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa5fcc00700 1 -- 172.21.15.28:0/2690478019 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa60c021850 con 0x557a56950f30 2024-05-20T20:09:56.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa60c026070 con 0x557a56950f30 2024-05-20T20:09:56.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa5fcc00700 1 -- 172.21.15.28:0/2690478019 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fa60c0219f0 con 0x557a56950f30 2024-05-20T20:09:56.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa5fcc00700 1 -- 172.21.15.28:0/2690478019 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa60c01c020 con 0x557a56950f30 2024-05-20T20:09:56.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa624086b80 10 client.26366 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:56.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.176+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557a569d7960 con 0x557a56950f30 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa5fcc00700 1 -- 172.21.15.28:0/2690478019 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa60c026070 con 0x557a56950f30 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa5fcc00700 1 -- 172.21.15.28:0/2690478019 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa60c03e470 con 0x557a56950f30 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa624086b80 10 client.26366 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa624086b80 10 client.26366 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa624086b80 20 client.26366 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa624086b80 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a569d7fb0 0x557a569da3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa624086b80 4 client.26366 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:09:56.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.177+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x557a568a4570 con 0x557a569d7fb0 2024-05-20T20:09:56.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.178+0000 7fa612a00700 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a569d7fb0 0x557a569da3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.179+0000 7fa612a00700 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a569d7fb0 0x557a569da3d0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=8a6bd936b35606c8 server_cookie=9a0cc5b990e17dae in_seq=0 out_seq=0 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.179+0000 7fa5fcc00700 10 client.26366 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.179+0000 7fa5fcc00700 1 -- 172.21.15.28:0/2690478019 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x557a568a4570 con 0x557a569d7fb0 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.179+0000 7fa5fcc00700 10 client.26366 handle_command_reply: tid=0 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 client.26366 shutdown 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 msgr2=0x557a5696c540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5696c540 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7fa60c009960 tx=0x7fa60c004f00 comp rx=0 tx=0).stop 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 shutdown_connections 2024-05-20T20:09:56.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x557a569d7fb0 0x557a569da3d0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557a56950f30 0x557a5696c540 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557a5694b140 0x557a5696bac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 --2- 172.21.15.28:0/2690478019 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557a56950610 0x557a5696c000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 >> 172.21.15.28:0/2690478019 conn(0x557a56878be0 msgr2=0x557a56951350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:56.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.180+0000 7fa5fcc00700 0 client.26366 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:09:56.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.181+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 shutdown_connections 2024-05-20T20:09:56.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.181+0000 7fa624086b80 1 -- 172.21.15.28:0/2690478019 wait complete. 2024-05-20T20:09:56.184 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:09:56.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.181+0000 7fa624086b80 20 client.26366 trim_cache size 0 max 16384 2024-05-20T20:09:56.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.182+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa5f80b2a70 msgr2=0x7fa5f80b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.182+0000 7fa619600700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa5f80b2a70 0x7fa5f80b4f30 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x557a5694bb00 tx=0x7fa60c00b040 comp rx=0 tx=0).stop 2024-05-20T20:09:56.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.182+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 msgr2=0x7fa614072180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:56.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.182+0000 7fa619600700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa614072180 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7fa600006010 tx=0x7fa600005280 comp rx=0 tx=0).stop 2024-05-20T20:09:56.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 shutdown_connections 2024-05-20T20:09:56.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6141088f0 0x7fa6140726c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6141037f0 0x7fa6140770a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa5f80b2a70 0x7fa5f80b4f30 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 --2- 172.21.15.28:0/2513142649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6141041a0 0x7fa614072180 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:56.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 >> 172.21.15.28:0/2513142649 conn(0x7fa61407a6a0 msgr2=0x7fa614101820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:56.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 shutdown_connections 2024-05-20T20:09:56.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:09:56.184+0000 7fa619600700 1 -- 172.21.15.28:0/2513142649 wait complete. 2024-05-20T20:09:57.119 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:56 smithi028 ceph-mon[31178]: pgmap v132: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 291 MiB/s wr, 1.03k op/s 2024-05-20T20:09:57.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:56 smithi154 ceph-mon[36314]: pgmap v132: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 291 MiB/s wr, 1.03k op/s 2024-05-20T20:09:57.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:56 smithi033 ceph-mon[36179]: pgmap v132: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 291 MiB/s wr, 1.03k op/s 2024-05-20T20:09:58.089 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:09:58.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.088+0000 7feb33200640 1 -- 172.21.15.28:0/277176523 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb34109b00 msgr2=0x7feb34109ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.088+0000 7feb33200640 1 --2- 172.21.15.28:0/277176523 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb34109b00 0x7feb34109ee0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7feb20009900 tx=0x7feb2002a410 comp rx=0 tx=0).stop 2024-05-20T20:09:58.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.088+0000 7feb33200640 1 -- 172.21.15.28:0/277176523 shutdown_connections 2024-05-20T20:09:58.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.088+0000 7feb33200640 1 --2- 172.21.15.28:0/277176523 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3410e160 0x7feb34115dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.088+0000 7feb33200640 1 --2- 172.21.15.28:0/277176523 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb3410a4b0 0x7feb3410da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.088+0000 7feb33200640 1 --2- 172.21.15.28:0/277176523 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb34109b00 0x7feb34109ee0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.088+0000 7feb33200640 1 -- 172.21.15.28:0/277176523 >> 172.21.15.28:0/277176523 conn(0x7feb340784a0 msgr2=0x7feb3407a8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.090+0000 7feb33200640 1 -- 172.21.15.28:0/277176523 shutdown_connections 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.090+0000 7feb33200640 1 -- 172.21.15.28:0/277176523 wait complete. 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.092+0000 7feb33200640 1 Processor -- start 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.093+0000 7feb33200640 1 -- start start 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.093+0000 7feb33200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb34109b00 0x7feb341a5410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.093+0000 7feb33200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb3410e160 0x7feb341a5950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.094+0000 7feb33200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3419f5c0 0x7feb3419fa70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.094+0000 7feb33200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7feb34119200 con 0x7feb34109b00 2024-05-20T20:09:58.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.094+0000 7feb33200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7feb34119080 con 0x7feb3419f5c0 2024-05-20T20:09:58.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.094+0000 7feb33200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7feb34119380 con 0x7feb3410e160 2024-05-20T20:09:58.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.094+0000 7feb32800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3419f5c0 0x7feb3419fa70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.094+0000 7feb32800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3419f5c0 0x7feb3419fa70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49898/0 (socket says 172.21.15.28:49898) 2024-05-20T20:09:58.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.094+0000 7feb32800640 1 -- 172.21.15.28:0/2455529122 learned_addr learned my addr 172.21.15.28:0/2455529122 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:58.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.095+0000 7feb31400640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb3410e160 0x7feb341a5950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.096+0000 7feb31e00640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb34109b00 0x7feb341a5410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.097+0000 7feb32800640 1 -- 172.21.15.28:0/2455529122 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb3410e160 msgr2=0x7feb341a5950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.097+0000 7feb32800640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb3410e160 0x7feb341a5950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.097+0000 7feb32800640 1 -- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb34109b00 msgr2=0x7feb341a5410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.097+0000 7feb32800640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb34109b00 0x7feb341a5410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.097+0000 7feb32800640 1 -- 172.21.15.28:0/2455529122 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feb341a00e0 con 0x7feb3419f5c0 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.098+0000 7feb31400640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb3410e160 0x7feb341a5950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.098+0000 7feb31e00640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb34109b00 0x7feb341a5410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:58.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.098+0000 7feb32800640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3419f5c0 0x7feb3419fa70 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7feb2c00b970 tx=0x7feb2c00be30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.099+0000 7feb2aa00640 1 -- 172.21.15.28:0/2455529122 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb2c019070 con 0x7feb3419f5c0 2024-05-20T20:09:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.099+0000 7feb2aa00640 1 -- 172.21.15.28:0/2455529122 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb2c014440 con 0x7feb3419f5c0 2024-05-20T20:09:58.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.099+0000 7feb2aa00640 1 -- 172.21.15.28:0/2455529122 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb2c0075d0 con 0x7feb3419f5c0 2024-05-20T20:09:58.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.102+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feb341a03d0 con 0x7feb3419f5c0 2024-05-20T20:09:58.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.102+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7feb341ac5d0 con 0x7feb3419f5c0 2024-05-20T20:09:58.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.111+0000 7feb3b964740 1 Processor -- start 2024-05-20T20:09:58.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.112+0000 7feb2aa00640 1 -- 172.21.15.28:0/2455529122 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7feb2c00cb40 con 0x7feb3419f5c0 2024-05-20T20:09:58.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.112+0000 7feb3b964740 1 -- start start 2024-05-20T20:09:58.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.113+0000 7feb3b964740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6366e70 0x555af6367250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.113+0000 7feb3b964740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6367790 0x555af6371b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.113+0000 7feb3b964740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af6374440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.113+0000 7feb3b964740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555af62d7b90 con 0x555af6366e70 2024-05-20T20:09:58.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.113+0000 7feb3b964740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555af62c7730 con 0x555af6372050 2024-05-20T20:09:58.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.113+0000 7feb3b964740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555af62cb770 con 0x555af6367790 2024-05-20T20:09:58.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.114+0000 7feb31400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6366e70 0x555af6367250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.114+0000 7feb31400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6366e70 0x555af6367250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45230/0 (socket says 172.21.15.28:45230) 2024-05-20T20:09:58.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.114+0000 7feb31400640 1 -- 172.21.15.28:0/3742547778 learned_addr learned my addr 172.21.15.28:0/3742547778 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:58.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.114+0000 7feb31400640 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af6374440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.115+0000 7feb31400640 1 -- 172.21.15.28:0/3742547778 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6367790 msgr2=0x555af6371b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.115+0000 7feb31400640 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6367790 0x555af6371b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.115+0000 7feb31400640 1 -- 172.21.15.28:0/3742547778 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6366e70 msgr2=0x555af6367250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.115+0000 7feb31400640 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6366e70 0x555af6367250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.115+0000 7feb31400640 1 -- 172.21.15.28:0/3742547778 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555af62608b0 con 0x555af6372050 2024-05-20T20:09:58.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.116+0000 7feb2aa00640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feb180b2b70 0x7feb180b5030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.116+0000 7feb2aa00640 1 -- 172.21.15.28:0/2455529122 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7feb2c0e1970 con 0x7feb3419f5c0 2024-05-20T20:09:58.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.117+0000 7feb31400640 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af6374440 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7feb24011ab0 tx=0x7feb2400a2d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3966585a76cc97d6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:58.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.117+0000 7feb1f000640 1 -- 172.21.15.28:0/3742547778 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb2400dad0 con 0x555af6372050 2024-05-20T20:09:58.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.118+0000 7feb31e00640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feb180b2b70 0x7feb180b5030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.118+0000 7feb1f000640 1 -- 172.21.15.28:0/3742547778 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb2400dc70 con 0x555af6372050 2024-05-20T20:09:58.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.118+0000 7feb1f000640 1 -- 172.21.15.28:0/3742547778 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb24003cd0 con 0x555af6372050 2024-05-20T20:09:58.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.123+0000 7feb3b964740 1 -- 172.21.15.28:0/3742547778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 msgr2=0x555af6374440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.123+0000 7feb3b964740 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af6374440 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7feb24011ab0 tx=0x7feb2400a2d0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.125+0000 7feb31e00640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feb180b2b70 0x7feb180b5030 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7feb20000c00 tx=0x7feb20031400 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:58.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.125+0000 7feb3b964740 1 -- 172.21.15.28:0/3742547778 shutdown_connections 2024-05-20T20:09:58.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.125+0000 7feb3b964740 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af6374440 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.125+0000 7feb3b964740 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6367790 0x555af6371b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.125+0000 7feb3b964740 1 --2- 172.21.15.28:0/3742547778 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6366e70 0x555af6367250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.126 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.126+0000 7feb3b964740 1 -- 172.21.15.28:0/3742547778 >> 172.21.15.28:0/3742547778 conn(0x555af6363380 msgr2=0x555af6363760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:58.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.126+0000 7feb3b964740 1 -- 172.21.15.28:0/3742547778 shutdown_connections 2024-05-20T20:09:58.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.126+0000 7feb3b964740 1 -- 172.21.15.28:0/3742547778 wait complete. 2024-05-20T20:09:58.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.127+0000 7feb3b964740 1 -- start start 2024-05-20T20:09:58.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.128+0000 7feb3b964740 1 Processor -- start 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.133+0000 7feb3b964740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af6371a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.133+0000 7feb3b964740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6367790 0x555af636ed40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.133+0000 7feb3b964740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6372050 0x555af636f350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.133+0000 7feb3b964740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555af5fad790 con 0x555af6372050 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.133+0000 7feb3b964740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555af626fbc0 con 0x555af6367790 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.133+0000 7feb3b964740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555af6385c90 con 0x555af6366e70 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.133+0000 7feb3b964740 10 client.0 _ll_register_callbacks cb 0x555af624d400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.134+0000 7feb32800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6372050 0x555af636f350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.135+0000 7feb31e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af6371a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.135+0000 7feb31e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af6371a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:46982/0 (socket says 172.21.15.28:46982) 2024-05-20T20:09:58.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.135+0000 7feb31e00640 1 -- 172.21.15.28:0/3395500452 learned_addr learned my addr 172.21.15.28:0/3395500452 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:58.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.135+0000 7feb31400640 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6367790 0x555af636ed40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.136+0000 7feb31e00640 1 -- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6367790 msgr2=0x555af636ed40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.136+0000 7feb31e00640 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6367790 0x555af636ed40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.136+0000 7feb31e00640 1 -- 172.21.15.28:0/3395500452 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6372050 msgr2=0x555af636f350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.136+0000 7feb31e00640 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6372050 0x555af636f350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.136+0000 7feb31e00640 1 -- 172.21.15.28:0/3395500452 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555af5fdc210 con 0x555af6366e70 2024-05-20T20:09:58.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.139+0000 7feb31e00640 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af6371a50 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7feb20009900 tx=0x7feb2002a410 comp rx=0 tx=0).ready entity=mon.1 client_cookie=eaaddf324f7cdd4f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:58.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.139+0000 7feb13400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:09:58.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.140+0000 7feb13400640 1 -- 172.21.15.28:0/3395500452 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb2004d020 con 0x555af6366e70 2024-05-20T20:09:58.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.140+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7feb2400dc70 con 0x555af6366e70 2024-05-20T20:09:58.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.142+0000 7feb13400640 1 -- 172.21.15.28:0/3395500452 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7feb200314a0 con 0x555af6366e70 2024-05-20T20:09:58.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.144+0000 7feb13400640 1 -- 172.21.15.28:0/3395500452 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7feb20058050 con 0x555af6366e70 2024-05-20T20:09:58.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.144+0000 7feb3b964740 10 client.26290 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:58.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.144+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555af60a8620 con 0x555af6366e70 2024-05-20T20:09:58.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.147+0000 7feb13400640 1 -- 172.21.15.28:0/3395500452 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb2400dc70 con 0x555af6366e70 2024-05-20T20:09:58.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.148+0000 7feb13400640 1 -- 172.21.15.28:0/3395500452 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7feb20049910 con 0x555af6366e70 2024-05-20T20:09:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.149+0000 7feb3b964740 10 client.26290 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.149+0000 7feb3b964740 10 client.26290 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.149+0000 7feb3b964740 20 client.26290 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.149+0000 7feb3b964740 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af6394e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.149+0000 7feb3b964740 4 client.26290 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:09:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.149+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555af62d9ae0 con 0x555af6392a90 2024-05-20T20:09:58.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.150+0000 7feb31400640 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af6394e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.152+0000 7feb13400640 10 client.26290 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:58.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.152+0000 7feb31400640 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af6394e80 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9fe968b32a0e8df6 server_cookie=21f9a0bc8f699223 in_seq=0 out_seq=0 2024-05-20T20:09:58.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.153+0000 7feb13400640 1 -- 172.21.15.28:0/3395500452 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x555af62d9ae0 con 0x555af6392a90 2024-05-20T20:09:58.154 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.153+0000 7feb13400640 10 client.26290 handle_command_reply: tid=0 2024-05-20T20:09:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.154+0000 7feb3b964740 1 client.26290 shutdown 2024-05-20T20:09:58.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 msgr2=0x555af6371a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af6371a50 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7feb20009900 tx=0x7feb2002a410 comp rx=0 tx=0).stop 2024-05-20T20:09:58.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 shutdown_connections 2024-05-20T20:09:58.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af6394e80 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb13400640 0 client.26290 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:58.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6367790 0x555af636ed40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6372050 0x555af636f350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 --2- 172.21.15.28:0/3395500452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af6371a50 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.155+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 >> 172.21.15.28:0/3395500452 conn(0x555af6363380 msgr2=0x555af6369100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:58.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.156+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 shutdown_connections 2024-05-20T20:09:58.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.156+0000 7feb3b964740 1 -- 172.21.15.28:0/3395500452 wait complete. 2024-05-20T20:09:58.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.156+0000 7feb3b964740 20 client.26290 trim_cache size 0 max 16384 2024-05-20T20:09:58.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 Processor -- start 2024-05-20T20:09:58.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 -- start start 2024-05-20T20:09:58.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af636ece0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af638ecf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af639a790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7feb340746b0 con 0x555af6367790 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7feb341ade30 con 0x555af6372050 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.175+0000 7feb3b964740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7feb341ac9e0 con 0x555af6366e70 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.176+0000 7feb31e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af636ece0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.176+0000 7feb31e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af636ece0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:46996/0 (socket says 172.21.15.28:46996) 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.176+0000 7feb31e00640 1 -- 172.21.15.28:0/1102107984 learned_addr learned my addr 172.21.15.28:0/1102107984 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.176+0000 7feb31400640 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af638ecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.176+0000 7feb32800640 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af639a790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.177+0000 7feb31e00640 1 -- 172.21.15.28:0/1102107984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 msgr2=0x555af639a790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.177+0000 7feb31e00640 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af639a790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.177+0000 7feb31e00640 1 -- 172.21.15.28:0/1102107984 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 msgr2=0x555af638ecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.177+0000 7feb31e00640 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af638ecf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.177+0000 7feb31e00640 1 -- 172.21.15.28:0/1102107984 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555af6366c30 con 0x555af6366e70 2024-05-20T20:09:58.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.177+0000 7feb32800640 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af639a790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:09:58.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.178+0000 7feb31e00640 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af636ece0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7feb20049d30 tx=0x7feb20009830 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e89b406eaedcec2c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:58.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.178+0000 7feb1e600640 1 -- 172.21.15.28:0/1102107984 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb2004d040 con 0x555af6366e70 2024-05-20T20:09:58.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.178+0000 7feb1e600640 1 -- 172.21.15.28:0/1102107984 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb20061070 con 0x555af6366e70 2024-05-20T20:09:58.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.178+0000 7feb1e600640 1 -- 172.21.15.28:0/1102107984 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb20035040 con 0x555af6366e70 2024-05-20T20:09:58.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.179+0000 7feb3b964740 1 -- 172.21.15.28:0/1102107984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 msgr2=0x555af636ece0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.179+0000 7feb3b964740 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af636ece0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7feb20049d30 tx=0x7feb20009830 comp rx=0 tx=0).stop 2024-05-20T20:09:58.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 -- 172.21.15.28:0/1102107984 shutdown_connections 2024-05-20T20:09:58.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6372050 0x555af639a790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af638ecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 --2- 172.21.15.28:0/1102107984 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6366e70 0x555af636ece0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 -- 172.21.15.28:0/1102107984 >> 172.21.15.28:0/1102107984 conn(0x555af6363380 msgr2=0x555af636e5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:58.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 -- 172.21.15.28:0/1102107984 shutdown_connections 2024-05-20T20:09:58.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 -- 172.21.15.28:0/1102107984 wait complete. 2024-05-20T20:09:58.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.180+0000 7feb3b964740 1 -- start start 2024-05-20T20:09:58.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.181+0000 7feb3b964740 1 Processor -- start 2024-05-20T20:09:58.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.181+0000 7feb3b964740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6366e70 0x555af6385f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb3b964740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af63864a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb3b964740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6372050 0x555af63fb650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb32800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6372050 0x555af63fb650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb32800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6372050 0x555af63fb650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:46998/0 (socket says 172.21.15.28:46998) 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb32800640 1 -- 172.21.15.28:0/3935967909 learned_addr learned my addr 172.21.15.28:0/3935967909 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7feb341ad760 con 0x555af6367790 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7feb341ad090 con 0x555af6366e70 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7feb34072690 con 0x555af6372050 2024-05-20T20:09:58.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.182+0000 7feb3b964740 10 client.0 _ll_register_callbacks cb 0x7feb20040070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:09:58.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.183+0000 7feb31e00640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6366e70 0x555af6385f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.183+0000 7feb31400640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af63864a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.186+0000 7feb32800640 1 -- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6366e70 msgr2=0x555af6385f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.186+0000 7feb32800640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6366e70 0x555af6385f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.186+0000 7feb32800640 1 -- 172.21.15.28:0/3935967909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 msgr2=0x555af63864a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:09:58.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.186+0000 7feb32800640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af63864a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.186+0000 7feb32800640 1 -- 172.21.15.28:0/3935967909 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555af6386d60 con 0x555af6372050 2024-05-20T20:09:58.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.187+0000 7feb31e00640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6366e70 0x555af6385f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:09:58.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.190+0000 7feb32800640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6372050 0x555af63fb650 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7feb2c0a8d50 tx=0x7feb2c00cf70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c114a37edfa4549b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:09:58.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.190+0000 7feb13400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:09:58.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.191+0000 7feb13400640 1 -- 172.21.15.28:0/3935967909 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb2c0df500 con 0x555af6372050 2024-05-20T20:09:58.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.191+0000 7feb13400640 1 -- 172.21.15.28:0/3935967909 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7feb2c0c33a0 con 0x555af6372050 2024-05-20T20:09:58.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.194+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7feb20061070 con 0x555af6372050 2024-05-20T20:09:58.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.195+0000 7feb13400640 1 -- 172.21.15.28:0/3935967909 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7feb2c0c9050 con 0x555af6372050 2024-05-20T20:09:58.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.195+0000 7feb3b964740 10 client.26296 fetch_fsmap learned FSMap version 29 2024-05-20T20:09:58.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.195+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555af63fcc70 con 0x555af6372050 2024-05-20T20:09:58.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb13400640 1 -- 172.21.15.28:0/3935967909 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb20061070 con 0x555af6372050 2024-05-20T20:09:58.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb13400640 1 -- 172.21.15.28:0/3935967909 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7feb2c0a48b0 con 0x555af6372050 2024-05-20T20:09:58.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb3b964740 10 client.26296 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:09:58.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb3b964740 10 client.26296 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:09:58.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb3b964740 20 client.26296 populate_metadata read hostname 'smithi028' 2024-05-20T20:09:58.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb3b964740 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af63ff380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:09:58.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb3b964740 4 client.26296 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:09:58.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.198+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x555af62d32c0 con 0x555af6392a90 2024-05-20T20:09:58.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.199+0000 7feb31e00640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af63ff380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:09:58.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.200+0000 7feb31e00640 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af63ff380 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bcfa8e4b5740935c server_cookie=15bb0bf3ff2fa879 in_seq=0 out_seq=0 2024-05-20T20:09:58.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.200+0000 7feb13400640 10 client.26296 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb13400640 1 -- 172.21.15.28:0/3935967909 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x555af62d32c0 con 0x555af6392a90 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb13400640 10 client.26296 handle_command_reply: tid=0 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 client.26296 shutdown 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6372050 msgr2=0x555af63fb650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6372050 0x555af63fb650 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7feb2c0a8d50 tx=0x7feb2c00cf70 comp rx=0 tx=0).stop 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 shutdown_connections 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x555af6392a90 0x555af63ff380 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555af6372050 0x555af63fb650 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555af6367790 0x555af63864a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 --2- 172.21.15.28:0/3935967909 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555af6366e70 0x555af6385f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 >> 172.21.15.28:0/3935967909 conn(0x555af6363380 msgr2=0x555af63623b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:58.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.230+0000 7feb13400640 0 client.26296 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:09:58.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.231+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 shutdown_connections 2024-05-20T20:09:58.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.231+0000 7feb3b964740 1 -- 172.21.15.28:0/3935967909 wait complete. 2024-05-20T20:09:58.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.231+0000 7feb3b964740 20 client.26296 trim_cache size 0 max 16384 2024-05-20T20:09:58.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:09:58.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:09:58.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:09:58.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.233+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feb180b2b70 msgr2=0x7feb180b5030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.233+0000 7feb33200640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feb180b2b70 0x7feb180b5030 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7feb20000c00 tx=0x7feb20031400 comp rx=0 tx=0).stop 2024-05-20T20:09:58.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.233+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3419f5c0 msgr2=0x7feb3419fa70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:09:58.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.233+0000 7feb33200640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3419f5c0 0x7feb3419fa70 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7feb2c00b970 tx=0x7feb2c00be30 comp rx=0 tx=0).stop 2024-05-20T20:09:58.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.234+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 shutdown_connections 2024-05-20T20:09:58.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.234+0000 7feb33200640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb3419f5c0 0x7feb3419fa70 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.234+0000 7feb33200640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feb180b2b70 0x7feb180b5030 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.234+0000 7feb33200640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb3410e160 0x7feb341a5950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.234+0000 7feb33200640 1 --2- 172.21.15.28:0/2455529122 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb34109b00 0x7feb341a5410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:09:58.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.234+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 >> 172.21.15.28:0/2455529122 conn(0x7feb340784a0 msgr2=0x7feb3407a890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:09:58.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.235+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 shutdown_connections 2024-05-20T20:09:58.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:09:58.235+0000 7feb33200640 1 -- 172.21.15.28:0/2455529122 wait complete. 2024-05-20T20:09:58.962 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:09:59.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:09:58 smithi154 ceph-mon[36314]: pgmap v133: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 260 MiB/s wr, 937 op/s 2024-05-20T20:09:59.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:09:58 smithi033 ceph-mon[36179]: pgmap v133: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 260 MiB/s wr, 937 op/s 2024-05-20T20:09:59.159 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:09:58 smithi028 ceph-mon[31178]: pgmap v133: 97 pgs: 97 active+clean; 18 GiB data, 3.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 260 MiB/s wr, 937 op/s 2024-05-20T20:10:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:00 smithi033 ceph-mon[36179]: pgmap v134: 97 pgs: 97 active+clean; 19 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 391 MiB/s wr, 1.40k op/s 2024-05-20T20:10:01.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:00 smithi033 ceph-mon[36179]: overall HEALTH_OK 2024-05-20T20:10:01.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:00 smithi028 ceph-mon[31178]: pgmap v134: 97 pgs: 97 active+clean; 19 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 391 MiB/s wr, 1.40k op/s 2024-05-20T20:10:01.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:00 smithi028 ceph-mon[31178]: overall HEALTH_OK 2024-05-20T20:10:01.319 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:00 smithi154 ceph-mon[36314]: pgmap v134: 97 pgs: 97 active+clean; 19 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 391 MiB/s wr, 1.40k op/s 2024-05-20T20:10:01.320 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:00 smithi154 ceph-mon[36314]: overall HEALTH_OK 2024-05-20T20:10:03.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:02 smithi028 ceph-mon[31178]: pgmap v135: 97 pgs: 97 active+clean; 19 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 259 MiB/s wr, 935 op/s 2024-05-20T20:10:03.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:02 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:10:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:02 smithi154 ceph-mon[36314]: pgmap v135: 97 pgs: 97 active+clean; 19 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 259 MiB/s wr, 935 op/s 2024-05-20T20:10:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:02 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:10:03.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:02 smithi033 ceph-mon[36179]: pgmap v135: 97 pgs: 97 active+clean; 19 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 259 MiB/s wr, 935 op/s 2024-05-20T20:10:03.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:02 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:10:04.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.011+0000 7fb9f8e00700 1 -- 172.21.15.28:0/3806918494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f410ffa0 msgr2=0x7fb9f4112390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.011+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/3806918494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f410ffa0 0x7fb9f4112390 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fb9ec0098f0 tx=0x7fb9ec009c00 comp rx=0 tx=0).stop 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.012+0000 7fb9f8e00700 1 -- 172.21.15.28:0/3806918494 shutdown_connections 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.012+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/3806918494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f410ffa0 0x7fb9f4112390 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.012+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/3806918494 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9f4075770 0x7fb9f4075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.012+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/3806918494 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f4076090 0x7fb9f410fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.012+0000 7fb9f8e00700 1 -- 172.21.15.28:0/3806918494 >> 172.21.15.28:0/3806918494 conn(0x7fb9f407a6a0 msgr2=0x7fb9f407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.012+0000 7fb9f8e00700 1 -- 172.21.15.28:0/3806918494 shutdown_connections 2024-05-20T20:10:04.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.012+0000 7fb9f8e00700 1 -- 172.21.15.28:0/3806918494 wait complete. 2024-05-20T20:10:04.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.013+0000 7fb9f8e00700 1 Processor -- start 2024-05-20T20:10:04.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.013+0000 7fb9f8e00700 1 -- start start 2024-05-20T20:10:04.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f8e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9f4075770 0x7fb9f419be90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f8e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f4076090 0x7fb9f419c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.014 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f8e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f410ffa0 0x7fb9f41a0760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f8e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb9f4115a00 con 0x7fb9f4076090 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f8e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb9f4115880 con 0x7fb9f410ffa0 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f8e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb9f4115b80 con 0x7fb9f4075770 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f410ffa0 0x7fb9f41a0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f1400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f4076090 0x7fb9f419c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f410ffa0 0x7fb9f41a0760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49966/0 (socket says 172.21.15.28:49966) 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.014+0000 7fb9f2800700 1 -- 172.21.15.28:0/4165824962 learned_addr learned my addr 172.21.15.28:0/4165824962 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:04.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9f4075770 0x7fb9f419be90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f2800700 1 -- 172.21.15.28:0/4165824962 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9f4075770 msgr2=0x7fb9f419be90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f2800700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9f4075770 0x7fb9f419be90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f2800700 1 -- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f4076090 msgr2=0x7fb9f419c3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f2800700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f4076090 0x7fb9f419c3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f2800700 1 -- 172.21.15.28:0/4165824962 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb9f41a0ea0 con 0x7fb9f410ffa0 2024-05-20T20:10:04.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f2800700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f410ffa0 0x7fb9f41a0760 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fb9ec009810 tx=0x7fb9ec00f9a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:04.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9eaa00700 1 -- 172.21.15.28:0/4165824962 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9ec003880 con 0x7fb9f410ffa0 2024-05-20T20:10:04.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.015+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb9f41a1130 con 0x7fb9f410ffa0 2024-05-20T20:10:04.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.016+0000 7fb9eaa00700 1 -- 172.21.15.28:0/4165824962 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb9ec00b610 con 0x7fb9f410ffa0 2024-05-20T20:10:04.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.016+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb9f41a8a10 con 0x7fb9f410ffa0 2024-05-20T20:10:04.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.016+0000 7fb9eaa00700 1 -- 172.21.15.28:0/4165824962 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9ec017640 con 0x7fb9f410ffa0 2024-05-20T20:10:04.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.026+0000 7fba0385db80 1 Processor -- start 2024-05-20T20:10:04.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.026+0000 7fb9eaa00700 1 -- 172.21.15.28:0/4165824962 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb9ec0177e0 con 0x7fb9f410ffa0 2024-05-20T20:10:04.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.028+0000 7fba0385db80 1 -- start start 2024-05-20T20:10:04.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.029+0000 7fba0385db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6610 0x56039f6d69f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.029+0000 7fba0385db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d6f30 0x56039f6d0b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.029+0000 7fba0385db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d1140 0x56039f6d15c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.029+0000 7fba0385db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56039f62ee40 con 0x56039f6d6f30 2024-05-20T20:10:04.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.029+0000 7fba0385db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56039f61e9e0 con 0x56039f6d6610 2024-05-20T20:10:04.032 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.029+0000 7fba0385db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56039f622a20 con 0x56039f6d1140 2024-05-20T20:10:04.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.034+0000 7fb9eaa00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9d80c3b00 0x7fb9d80c5fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.035+0000 7fb9eaa00700 1 -- 172.21.15.28:0/4165824962 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb9ec0e2a70 con 0x7fb9f410ffa0 2024-05-20T20:10:04.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.035+0000 7fb9f2800700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9d80c3b00 0x7fb9d80c5fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.035+0000 7fb9f1e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d6f30 0x56039f6d0b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.035+0000 7fb9f1e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d6f30 0x56039f6d0b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45276/0 (socket says 172.21.15.28:45276) 2024-05-20T20:10:04.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.035+0000 7fb9f1e00700 1 -- 172.21.15.28:0/987581206 learned_addr learned my addr 172.21.15.28:0/987581206 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:04.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.036+0000 7fb9f1e00700 1 -- 172.21.15.28:0/987581206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d1140 msgr2=0x56039f6d15c0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:10:04.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.036+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/987581206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d1140 0x56039f6d15c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.036+0000 7fb9f1e00700 1 -- 172.21.15.28:0/987581206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6610 msgr2=0x56039f6d69f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:04.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.036+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/987581206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6610 0x56039f6d69f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.037+0000 7fb9f1e00700 1 -- 172.21.15.28:0/987581206 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56039f5e7420 con 0x56039f6d6f30 2024-05-20T20:10:04.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.037+0000 7fb9f2800700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9d80c3b00 0x7fb9d80c5fc0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fb9ec0a74c0 tx=0x7fb9ec00cf60 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:04.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.038+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/987581206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d6f30 0x56039f6d0b20 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7fb9e000bd30 tx=0x7fb9e000f640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=18ef687937aabada server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:04.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.038+0000 7fb9df400700 1 -- 172.21.15.28:0/987581206 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9e0015070 con 0x56039f6d6f30 2024-05-20T20:10:04.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.039+0000 7fb9df400700 1 -- 172.21.15.28:0/987581206 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb9e000fe50 con 0x56039f6d6f30 2024-05-20T20:10:04.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.039+0000 7fb9df400700 1 -- 172.21.15.28:0/987581206 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9e0011710 con 0x56039f6d6f30 2024-05-20T20:10:04.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.040+0000 7fba0385db80 1 -- 172.21.15.28:0/987581206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d6f30 msgr2=0x56039f6d0b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.040+0000 7fba0385db80 1 --2- 172.21.15.28:0/987581206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d6f30 0x56039f6d0b20 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7fb9e000bd30 tx=0x7fb9e000f640 comp rx=0 tx=0).stop 2024-05-20T20:10:04.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.045+0000 7fba0385db80 1 -- 172.21.15.28:0/987581206 shutdown_connections 2024-05-20T20:10:04.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.045+0000 7fba0385db80 1 --2- 172.21.15.28:0/987581206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d1140 0x56039f6d15c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.045+0000 7fba0385db80 1 --2- 172.21.15.28:0/987581206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6610 0x56039f6d69f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.045+0000 7fba0385db80 1 --2- 172.21.15.28:0/987581206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d6f30 0x56039f6d0b20 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.045+0000 7fba0385db80 1 -- 172.21.15.28:0/987581206 >> 172.21.15.28:0/987581206 conn(0x56039f5febe0 msgr2=0x56039f6cb690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:04.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.048+0000 7fba0385db80 1 -- 172.21.15.28:0/987581206 shutdown_connections 2024-05-20T20:10:04.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.048+0000 7fba0385db80 1 -- 172.21.15.28:0/987581206 wait complete. 2024-05-20T20:10:04.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.049+0000 7fba0385db80 1 -- start start 2024-05-20T20:10:04.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.049+0000 7fba0385db80 1 Processor -- start 2024-05-20T20:10:04.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.050+0000 7fba0385db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6dcc50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.054+0000 7fba0385db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d9ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.054+0000 7fba0385db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6da530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.054+0000 7fba0385db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56039f524fa0 con 0x56039f6d1140 2024-05-20T20:10:04.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.054+0000 7fba0385db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56039f5bf760 con 0x56039f6d6f30 2024-05-20T20:10:04.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.054+0000 7fba0385db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56039f6dd9e0 con 0x56039f6d6610 2024-05-20T20:10:04.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.054+0000 7fba0385db80 10 client.0 _ll_register_callbacks cb 0x56039f5c8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:04.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6da530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6da530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49994/0 (socket says 172.21.15.28:49994) 2024-05-20T20:10:04.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 -- 172.21.15.28:0/3811600887 learned_addr learned my addr 172.21.15.28:0/3811600887 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:04.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 -- 172.21.15.28:0/3811600887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 msgr2=0x56039f6d9ff0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d9ff0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 -- 172.21.15.28:0/3811600887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 msgr2=0x56039f6dcc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6dcc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.055+0000 7fb9f1400700 1 -- 172.21.15.28:0/3811600887 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56039f56e8d0 con 0x56039f6d6f30 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.056+0000 7fb9f1400700 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6da530 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7fb9e4009e10 tx=0x7fb9e400bf00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4cc787804c05005a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.056+0000 7fb9d3e00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.057+0000 7fb9d3e00700 1 -- 172.21.15.28:0/3811600887 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9e401b060 con 0x56039f6d6f30 2024-05-20T20:10:04.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.057+0000 7fb9d3e00700 1 -- 172.21.15.28:0/3811600887 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb9e4008b60 con 0x56039f6d6f30 2024-05-20T20:10:04.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.057+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb9e000fe50 con 0x56039f6d6f30 2024-05-20T20:10:04.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.057+0000 7fb9d3e00700 1 -- 172.21.15.28:0/3811600887 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb9e4008d60 con 0x56039f6d6f30 2024-05-20T20:10:04.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.057+0000 7fba0385db80 10 client.26384 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:04.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.057+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56039f5ef060 con 0x56039f6d6f30 2024-05-20T20:10:04.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.058+0000 7fb9d3e00700 1 -- 172.21.15.28:0/3811600887 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb9e000fe50 con 0x56039f6d6f30 2024-05-20T20:10:04.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.062+0000 7fb9d3e00700 1 -- 172.21.15.28:0/3811600887 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb9e40169b0 con 0x56039f6d6f30 2024-05-20T20:10:04.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.063+0000 7fba0385db80 10 client.26384 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:04.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.063+0000 7fba0385db80 10 client.26384 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:10:04.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.063+0000 7fba0385db80 20 client.26384 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:04.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.063+0000 7fba0385db80 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f6f3240 0x56039f6f5660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.063+0000 7fba0385db80 4 client.26384 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:04.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.063+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56039f630d90 con 0x56039f6f3240 2024-05-20T20:10:04.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.068+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f6f3240 0x56039f6f5660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.072+0000 7fb9d3e00700 10 client.26384 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:10:04.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.072+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f6f3240 0x56039f6f5660 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=ff2eddb13f01c1b3 server_cookie=b8cf821f1b9444b8 in_seq=0 out_seq=0 2024-05-20T20:10:04.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.076+0000 7fb9d3e00700 1 -- 172.21.15.28:0/3811600887 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56039f630d90 con 0x56039f6f3240 2024-05-20T20:10:04.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.076+0000 7fb9d3e00700 10 client.26384 handle_command_reply: tid=0 2024-05-20T20:10:04.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.076+0000 7fba0385db80 1 client.26384 shutdown 2024-05-20T20:10:04.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 msgr2=0x56039f6da530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6da530 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7fb9e4009e10 tx=0x7fb9e400bf00 comp rx=0 tx=0).stop 2024-05-20T20:10:04.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 shutdown_connections 2024-05-20T20:10:04.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6da530 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f6f3240 0x56039f6f5660 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fb9d3e00700 0 client.26384 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:10:04.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6dcc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 --2- 172.21.15.28:0/3811600887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d9ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.077+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 >> 172.21.15.28:0/3811600887 conn(0x56039f5febe0 msgr2=0x56039f6cad70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:04.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.078+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 shutdown_connections 2024-05-20T20:10:04.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.078+0000 7fba0385db80 1 -- 172.21.15.28:0/3811600887 wait complete. 2024-05-20T20:10:04.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.078+0000 7fba0385db80 20 client.26384 trim_cache size 0 max 16384 2024-05-20T20:10:04.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.110+0000 7fba0385db80 1 Processor -- start 2024-05-20T20:10:04.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.110+0000 7fba0385db80 1 -- start start 2024-05-20T20:10:04.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fba0385db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6d0b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fba0385db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d8f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fba0385db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6d7a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fba0385db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb9d400d670 con 0x56039f6d1140 2024-05-20T20:10:04.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fba0385db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb9d400f1b0 con 0x56039f6d6f30 2024-05-20T20:10:04.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fba0385db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb9d400dd40 con 0x56039f6d6610 2024-05-20T20:10:04.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fb9f1400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6d0b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fb9f1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d8f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fb9f1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d8f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47040/0 (socket says 172.21.15.28:47040) 2024-05-20T20:10:04.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.111+0000 7fb9f1e00700 1 -- 172.21.15.28:0/2648374699 learned_addr learned my addr 172.21.15.28:0/2648374699 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:04.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.112+0000 7fb9f1e00700 1 -- 172.21.15.28:0/2648374699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 msgr2=0x56039f6d7a20 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:10:04.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.112+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6d7a20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.112+0000 7fb9f1e00700 1 -- 172.21.15.28:0/2648374699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 msgr2=0x56039f6d0b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:04.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.112+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6d0b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.112+0000 7fb9f1e00700 1 -- 172.21.15.28:0/2648374699 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56039f6d6d30 con 0x56039f6d6610 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.112+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d8f90 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fb9e001af20 tx=0x7fb9e0000bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f394f7ce23e1d789 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.112+0000 7fb9f1400700 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6d0b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.113+0000 7fb9dea00700 1 -- 172.21.15.28:0/2648374699 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9e0015040 con 0x56039f6d6610 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.113+0000 7fb9dea00700 1 -- 172.21.15.28:0/2648374699 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb9e000fbf0 con 0x56039f6d6610 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.113+0000 7fb9dea00700 1 -- 172.21.15.28:0/2648374699 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9e001a930 con 0x56039f6d6610 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.113+0000 7fba0385db80 1 -- 172.21.15.28:0/2648374699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 msgr2=0x56039f6d8f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.113+0000 7fba0385db80 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d8f90 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fb9e001af20 tx=0x7fb9e0000bc0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 -- 172.21.15.28:0/2648374699 shutdown_connections 2024-05-20T20:10:04.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6d7a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6d0b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 --2- 172.21.15.28:0/2648374699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6d8f90 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 -- 172.21.15.28:0/2648374699 >> 172.21.15.28:0/2648374699 conn(0x56039f5febe0 msgr2=0x56039f6d7310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 -- 172.21.15.28:0/2648374699 shutdown_connections 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 -- 172.21.15.28:0/2648374699 wait complete. 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.114+0000 7fba0385db80 1 -- start start 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 1 Processor -- start 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6fad10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6fb250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6fb790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb9d400eae0 con 0x56039f6d1140 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb9d400e410 con 0x56039f6d6f30 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb9d400b610 con 0x56039f6d6610 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.115+0000 7fba0385db80 10 client.0 _ll_register_callbacks cb 0x56039f5c8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.116+0000 7fb9f1400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6fad10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.116+0000 7fb9f1400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6fad10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45310/0 (socket says 172.21.15.28:45310) 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.116+0000 7fb9f1400700 1 -- 172.21.15.28:0/423993180 learned_addr learned my addr 172.21.15.28:0/423993180 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.116+0000 7fb9f1400700 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6fb790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.116+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6fb250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.117+0000 7fb9f1400700 1 -- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 msgr2=0x56039f6fb250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.117+0000 7fb9f1400700 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6fb250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.117+0000 7fb9f1400700 1 -- 172.21.15.28:0/423993180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 msgr2=0x56039f6fb790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.117+0000 7fb9f1400700 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6fb790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.117+0000 7fb9f1400700 1 -- 172.21.15.28:0/423993180 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56039f77f100 con 0x56039f6d1140 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.117+0000 7fb9f1400700 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6fad10 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x56039f6cc460 tx=0x7fb9e400bbb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52ac4f66127180e2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.117+0000 7fb9d3e00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.118+0000 7fb9d3e00700 1 -- 172.21.15.28:0/423993180 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb9e4025080 con 0x56039f6d1140 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.118+0000 7fb9d3e00700 1 -- 172.21.15.28:0/423993180 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb9e4020830 con 0x56039f6d1140 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.118+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb9e000fbf0 con 0x56039f6d1140 2024-05-20T20:10:04.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.119+0000 7fb9d3e00700 1 -- 172.21.15.28:0/423993180 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb9e4020a30 con 0x56039f6d1140 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.119+0000 7fba0385db80 10 client.16188 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.119+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56039f755e00 con 0x56039f6d1140 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.120+0000 7fb9d3e00700 1 -- 172.21.15.28:0/423993180 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb9e000fbf0 con 0x56039f6d1140 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.121+0000 7fb9d3e00700 1 -- 172.21.15.28:0/423993180 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb9e4078c10 con 0x56039f6d1140 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.122+0000 7fba0385db80 10 client.16188 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.122+0000 7fba0385db80 10 client.16188 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.123+0000 7fba0385db80 20 client.16188 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.123+0000 7fba0385db80 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f7563b0 0x56039f7587d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.123+0000 7fba0385db80 4 client.16188 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.123+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56039f62a570 con 0x56039f7563b0 2024-05-20T20:10:04.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.125+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f7563b0 0x56039f7587d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:04.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.126+0000 7fb9d3e00700 10 client.16188 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:10:04.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.126+0000 7fb9f1e00700 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f7563b0 0x56039f7587d0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=4db659c246a58e63 server_cookie=45fea7800b543965 in_seq=0 out_seq=0 2024-05-20T20:10:04.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.127+0000 7fb9d3e00700 1 -- 172.21.15.28:0/423993180 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56039f62a570 con 0x56039f7563b0 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.127+0000 7fb9d3e00700 10 client.16188 handle_command_reply: tid=0 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.127+0000 7fba0385db80 1 client.16188 shutdown 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.127+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 msgr2=0x56039f6fad10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.127+0000 7fba0385db80 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6fad10 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x56039f6cc460 tx=0x7fb9e400bbb0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 shutdown_connections 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56039f6d6f30 0x56039f6fb790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x56039f7563b0 0x56039f7587d0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56039f6d1140 0x56039f6fad10 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 --2- 172.21.15.28:0/423993180 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56039f6d6610 0x56039f6fb250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 >> 172.21.15.28:0/423993180 conn(0x56039f5febe0 msgr2=0x56039f6d7ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:04.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fb9d3e00700 0 client.16188 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:10:04.134 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:10:04.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 shutdown_connections 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 1 -- 172.21.15.28:0/423993180 wait complete. 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.128+0000 7fba0385db80 20 client.16188 trim_cache size 0 max 16384 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9d80c3b00 msgr2=0x7fb9d80c5fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9d80c3b00 0x7fb9d80c5fc0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fb9ec0a74c0 tx=0x7fb9ec00cf60 comp rx=0 tx=0).stop 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f410ffa0 msgr2=0x7fb9f41a0760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f410ffa0 0x7fb9f41a0760 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fb9ec009810 tx=0x7fb9ec00f9a0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 shutdown_connections 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb9d80c3b00 0x7fb9d80c5fc0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb9f410ffa0 0x7fb9f41a0760 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb9f4075770 0x7fb9f419be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 --2- 172.21.15.28:0/4165824962 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb9f4076090 0x7fb9f419c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.130+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 >> 172.21.15.28:0/4165824962 conn(0x7fb9f407a6a0 msgr2=0x7fb9f4100810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.132+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 shutdown_connections 2024-05-20T20:10:04.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:04.132+0000 7fb9f8e00700 1 -- 172.21.15.28:0/4165824962 wait complete. 2024-05-20T20:10:05.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:04 smithi028 ceph-mon[31178]: pgmap v136: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 355 MiB/s wr, 1.38k op/s 2024-05-20T20:10:05.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:04 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:10:05.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:04 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:10:05.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:04 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:10:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:04 smithi154 ceph-mon[36314]: pgmap v136: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 355 MiB/s wr, 1.38k op/s 2024-05-20T20:10:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:04 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:10:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:04 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:10:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:04 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:10:05.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:04 smithi033 ceph-mon[36179]: pgmap v136: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 355 MiB/s wr, 1.38k op/s 2024-05-20T20:10:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:04 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:10:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:04 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:10:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:04 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:10:05.875 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:10:06.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:05 smithi028 ceph-mon[31178]: pgmap v137: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 283 MiB/s wr, 1.14k op/s 2024-05-20T20:10:06.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:05 smithi154 ceph-mon[36314]: pgmap v137: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 283 MiB/s wr, 1.14k op/s 2024-05-20T20:10:06.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:05 smithi033 ceph-mon[36179]: pgmap v137: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 283 MiB/s wr, 1.14k op/s 2024-05-20T20:10:06.595 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:09.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:08 smithi154 ceph-mon[36314]: pgmap v138: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 254 MiB/s wr, 1.03k op/s 2024-05-20T20:10:09.129 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:08 smithi033 ceph-mon[36179]: pgmap v138: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 254 MiB/s wr, 1.03k op/s 2024-05-20T20:10:09.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:08 smithi028 ceph-mon[31178]: pgmap v138: 97 pgs: 97 active+clean; 19 GiB data, 4.1 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 254 MiB/s wr, 1.03k op/s 2024-05-20T20:10:10.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.229+0000 7f182c600700 1 -- 172.21.15.28:0/3366950202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182411cd20 msgr2=0x7f182411f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.229+0000 7f182c600700 1 --2- 172.21.15.28:0/3366950202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182411cd20 0x7f182411f110 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f181c00b0a0 tx=0x7f181c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.232+0000 7f182c600700 1 -- 172.21.15.28:0/3366950202 shutdown_connections 2024-05-20T20:10:10.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.232+0000 7f182c600700 1 --2- 172.21.15.28:0/3366950202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182411cd20 0x7f182411f110 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.232+0000 7f182c600700 1 --2- 172.21.15.28:0/3366950202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1824111d20 0x7f182410a540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.232+0000 7f182c600700 1 --2- 172.21.15.28:0/3366950202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f182410aa80 0x7f182410af00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.232+0000 7f182c600700 1 -- 172.21.15.28:0/3366950202 >> 172.21.15.28:0/3366950202 conn(0x7f182406e6b0 msgr2=0x7f182406eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:10.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.233+0000 7f182c600700 1 -- 172.21.15.28:0/3366950202 shutdown_connections 2024-05-20T20:10:10.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.233+0000 7f182c600700 1 -- 172.21.15.28:0/3366950202 wait complete. 2024-05-20T20:10:10.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.235+0000 7f182c600700 1 Processor -- start 2024-05-20T20:10:10.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.235+0000 7f182c600700 1 -- start start 2024-05-20T20:10:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.236+0000 7f182c600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182410aa80 0x7f18241ae770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.236+0000 7f182c600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1824111d20 0x7f18241aecb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.236+0000 7f182c600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f182411cd20 0x7f18241a8930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.236+0000 7f182c600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1824122630 con 0x7f182411cd20 2024-05-20T20:10:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.236+0000 7f182c600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f18241224b0 con 0x7f1824111d20 2024-05-20T20:10:10.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.236+0000 7f182c600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f18241227b0 con 0x7f182410aa80 2024-05-20T20:10:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.237+0000 7f182a600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f182411cd20 0x7f18241a8930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.237+0000 7f182a600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f182411cd20 0x7f18241a8930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57566/0 (socket says 172.21.15.28:57566) 2024-05-20T20:10:10.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.237+0000 7f182a600700 1 -- 172.21.15.28:0/54939939 learned_addr learned my addr 172.21.15.28:0/54939939 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:10.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.238+0000 7f1829c00700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182410aa80 0x7f18241ae770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.239+0000 7f1829c00700 1 -- 172.21.15.28:0/54939939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1824111d20 msgr2=0x7f18241aecb0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:10:10.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.239+0000 7f1829c00700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1824111d20 0x7f18241aecb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.241 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.239+0000 7f1829c00700 1 -- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f182411cd20 msgr2=0x7f18241a8930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.241 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.239+0000 7f1829c00700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f182411cd20 0x7f18241a8930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.241 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.239+0000 7f1829c00700 1 -- 172.21.15.28:0/54939939 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f18241a9090 con 0x7f182410aa80 2024-05-20T20:10:10.241 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.241+0000 7f1829c00700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182410aa80 0x7f18241ae770 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f181400e9f0 tx=0x7f181400ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:10.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.242+0000 7f1822000700 1 -- 172.21.15.28:0/54939939 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f181400cbd0 con 0x7f182410aa80 2024-05-20T20:10:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.242+0000 7f1822000700 1 -- 172.21.15.28:0/54939939 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f181400cd70 con 0x7f182410aa80 2024-05-20T20:10:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.242+0000 7f1822000700 1 -- 172.21.15.28:0/54939939 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1814018680 con 0x7f182410aa80 2024-05-20T20:10:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.243+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f18241a9380 con 0x7f182410aa80 2024-05-20T20:10:10.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.243+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f18241b5640 con 0x7f182410aa80 2024-05-20T20:10:10.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.256+0000 7f1822000700 1 -- 172.21.15.28:0/54939939 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1814018820 con 0x7f182410aa80 2024-05-20T20:10:10.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.258+0000 7f1822000700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f180c0b2af0 0x7f180c0b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.259+0000 7f1822000700 1 -- 172.21.15.28:0/54939939 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f18140e2d00 con 0x7f182410aa80 2024-05-20T20:10:10.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.260+0000 7f1836f06b80 1 Processor -- start 2024-05-20T20:10:10.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.261+0000 7f1836f06b80 1 -- start start 2024-05-20T20:10:10.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.261+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f9218290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.261+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f9212260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.261+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.261+0000 7f1836f06b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c6f9170e40 con 0x55c6f9212880 2024-05-20T20:10:10.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.261+0000 7f1836f06b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c6f91609e0 con 0x55c6f92187d0 2024-05-20T20:10:10.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.261+0000 7f1836f06b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c6f9164a20 con 0x55c6f9217eb0 2024-05-20T20:10:10.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.262+0000 7f1829200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f9212260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.262+0000 7f1829200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f9212260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38946/0 (socket says 172.21.15.28:38946) 2024-05-20T20:10:10.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.262+0000 7f1829200700 1 -- 172.21.15.28:0/2596800879 learned_addr learned my addr 172.21.15.28:0/2596800879 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:10.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.263+0000 7f182a600700 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.263+0000 7f1829c00700 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f9218290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.264+0000 7f1829200700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f180c0b2af0 0x7f180c0b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.264+0000 7f1829c00700 1 -- 172.21.15.28:0/2596800879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 msgr2=0x55c6f9212260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.264+0000 7f1829c00700 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f9212260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.264+0000 7f1829c00700 1 -- 172.21.15.28:0/2596800879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 msgr2=0x55c6f9212ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.264+0000 7f1829c00700 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.264+0000 7f1829c00700 1 -- 172.21.15.28:0/2596800879 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c6f9129420 con 0x55c6f9217eb0 2024-05-20T20:10:10.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.265+0000 7f1829c00700 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f9218290 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f18140e0400 tx=0x7f18140aa1a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=86d1bc0acbdc105e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:10.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.266+0000 7f1812a00700 1 -- 172.21.15.28:0/2596800879 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f18140e0890 con 0x55c6f9217eb0 2024-05-20T20:10:10.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.266+0000 7f1829200700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f180c0b2af0 0x7f180c0b4fb0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f18180123c0 tx=0x7f181800d040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:10.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.267+0000 7f1812a00700 1 -- 172.21.15.28:0/2596800879 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f18140aa540 con 0x55c6f9217eb0 2024-05-20T20:10:10.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.268+0000 7f1836f06b80 1 -- 172.21.15.28:0/2596800879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 msgr2=0x55c6f9218290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.268+0000 7f1836f06b80 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f9218290 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f18140e0400 tx=0x7f18140aa1a0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.269+0000 7f1836f06b80 1 -- 172.21.15.28:0/2596800879 shutdown_connections 2024-05-20T20:10:10.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.269+0000 7f1836f06b80 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.270+0000 7f1836f06b80 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f9218290 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.270+0000 7f1836f06b80 1 --2- 172.21.15.28:0/2596800879 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f9212260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.270+0000 7f1836f06b80 1 -- 172.21.15.28:0/2596800879 >> 172.21.15.28:0/2596800879 conn(0x55c6f9140be0 msgr2=0x55c6f920d010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:10.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.271+0000 7f1836f06b80 1 -- 172.21.15.28:0/2596800879 shutdown_connections 2024-05-20T20:10:10.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.271+0000 7f1836f06b80 1 -- 172.21.15.28:0/2596800879 wait complete. 2024-05-20T20:10:10.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.273+0000 7f1836f06b80 1 -- start start 2024-05-20T20:10:10.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.273+0000 7f1836f06b80 1 Processor -- start 2024-05-20T20:10:10.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.274+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f921e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.275+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f9217eb0 0x55c6f921b890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.275+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f92187d0 0x55c6f921bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.275+0000 7f1836f06b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c6f9066fa0 con 0x55c6f9212880 2024-05-20T20:10:10.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.275+0000 7f1836f06b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c6f9101760 con 0x55c6f9217eb0 2024-05-20T20:10:10.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.275+0000 7f1836f06b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c6f921f220 con 0x55c6f92187d0 2024-05-20T20:10:10.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.275+0000 7f1836f06b80 10 client.0 _ll_register_callbacks cb 0x55c6f910ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:10.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.275+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f921e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.281+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f921e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57594/0 (socket says 172.21.15.28:57594) 2024-05-20T20:10:10.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.281+0000 7f1829c00700 1 -- 172.21.15.28:0/734288765 learned_addr learned my addr 172.21.15.28:0/734288765 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:10.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.281+0000 7f1829200700 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f9217eb0 0x55c6f921b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.282+0000 7f182a600700 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f92187d0 0x55c6f921bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.284+0000 7f1829c00700 1 -- 172.21.15.28:0/734288765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f92187d0 msgr2=0x55c6f921bdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.284+0000 7f1829c00700 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f92187d0 0x55c6f921bdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.284+0000 7f1829c00700 1 -- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f9217eb0 msgr2=0x55c6f921b890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.284+0000 7f1829c00700 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f9217eb0 0x55c6f921b890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.284+0000 7f1829c00700 1 -- 172.21.15.28:0/734288765 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c6f90b08d0 con 0x55c6f9212880 2024-05-20T20:10:10.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.289+0000 7f1829c00700 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f921e4f0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f18140a6ab0 tx=0x7f18140b3630 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bbd7f0524c687d38 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:10.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.290+0000 7f1807400700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:10:10.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.293+0000 7f1807400700 1 -- 172.21.15.28:0/734288765 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f18140aaa30 con 0x55c6f9212880 2024-05-20T20:10:10.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.295+0000 7f1807400700 1 -- 172.21.15.28:0/734288765 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f18140a9c90 con 0x55c6f9212880 2024-05-20T20:10:10.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.295+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f18140aa540 con 0x55c6f9212880 2024-05-20T20:10:10.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.296+0000 7f1807400700 1 -- 172.21.15.28:0/734288765 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f18140aede0 con 0x55c6f9212880 2024-05-20T20:10:10.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.296+0000 7f1836f06b80 10 client.16197 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:10.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.296+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c6f9131060 con 0x55c6f9212880 2024-05-20T20:10:10.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.301+0000 7f1807400700 1 -- 172.21.15.28:0/734288765 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f18140aa540 con 0x55c6f9212880 2024-05-20T20:10:10.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.302+0000 7f1807400700 1 -- 172.21.15.28:0/734288765 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f18140a9700 con 0x55c6f9212880 2024-05-20T20:10:10.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.305+0000 7f1836f06b80 10 client.16197 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:10.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.305+0000 7f1836f06b80 10 client.16197 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:10:10.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.305+0000 7f1836f06b80 20 client.16197 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:10.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.305+0000 7f1836f06b80 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f922e020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.305+0000 7f1836f06b80 4 client.16197 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:10.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.305+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c6f9172d90 con 0x55c6f922bc30 2024-05-20T20:10:10.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.306+0000 7f1829200700 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f922e020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.307+0000 7f1807400700 10 client.16197 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:10:10.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.307+0000 7f1829200700 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f922e020 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=673153bf121ea44e server_cookie=4d41449ebb2d43 in_seq=0 out_seq=0 2024-05-20T20:10:10.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.308+0000 7f1807400700 1 -- 172.21.15.28:0/734288765 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55c6f9172d90 con 0x55c6f922bc30 2024-05-20T20:10:10.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.308+0000 7f1807400700 10 client.16197 handle_command_reply: tid=0 2024-05-20T20:10:10.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.309+0000 7f1836f06b80 1 client.16197 shutdown 2024-05-20T20:10:10.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 msgr2=0x55c6f921e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f921e4f0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f18140a6ab0 tx=0x7f18140b3630 comp rx=0 tx=0).stop 2024-05-20T20:10:10.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 shutdown_connections 2024-05-20T20:10:10.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f922e020 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1807400700 0 client.16197 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:10:10.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f92187d0 0x55c6f921bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f921e4f0 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 --2- 172.21.15.28:0/734288765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f9217eb0 0x55c6f921b890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.310+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 >> 172.21.15.28:0/734288765 conn(0x55c6f9140be0 msgr2=0x55c6f9211250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:10.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.311+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 shutdown_connections 2024-05-20T20:10:10.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.311+0000 7f1836f06b80 1 -- 172.21.15.28:0/734288765 wait complete. 2024-05-20T20:10:10.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.312+0000 7f1836f06b80 20 client.16197 trim_cache size 0 max 16384 2024-05-20T20:10:10.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 Processor -- start 2024-05-20T20:10:10.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 -- start start 2024-05-20T20:10:10.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f921e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f920d0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f18241bce50 con 0x55c6f9212880 2024-05-20T20:10:10.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f18241be990 con 0x55c6f92187d0 2024-05-20T20:10:10.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.348+0000 7f1836f06b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f18241bd520 con 0x55c6f9217eb0 2024-05-20T20:10:10.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f921e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f921e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41238/0 (socket says 172.21.15.28:41238) 2024-05-20T20:10:10.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 -- 172.21.15.28:0/274893687 learned_addr learned my addr 172.21.15.28:0/274893687 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:10.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829c00700 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f182a600700 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f920d0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 -- 172.21.15.28:0/274893687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 msgr2=0x55c6f920d0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f920d0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 -- 172.21.15.28:0/274893687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 msgr2=0x55c6f9212260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 -- 172.21.15.28:0/274893687 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c6f92074c0 con 0x55c6f9217eb0 2024-05-20T20:10:10.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.349+0000 7f1829200700 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f921e4f0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f181800d010 tx=0x7f1818004750 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4a0f4dbabd3439f9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:10.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.350+0000 7f1812000700 1 -- 172.21.15.28:0/274893687 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1818012de0 con 0x55c6f9217eb0 2024-05-20T20:10:10.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.350+0000 7f1812000700 1 -- 172.21.15.28:0/274893687 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f18180044e0 con 0x55c6f9217eb0 2024-05-20T20:10:10.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.350+0000 7f1812000700 1 -- 172.21.15.28:0/274893687 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1818003680 con 0x55c6f9217eb0 2024-05-20T20:10:10.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 -- 172.21.15.28:0/274893687 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 msgr2=0x55c6f921e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f921e4f0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f181800d010 tx=0x7f1818004750 comp rx=0 tx=0).stop 2024-05-20T20:10:10.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 -- 172.21.15.28:0/274893687 shutdown_connections 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f920d0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f9212260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 --2- 172.21.15.28:0/274893687 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f921e4f0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 -- 172.21.15.28:0/274893687 >> 172.21.15.28:0/274893687 conn(0x55c6f9140be0 msgr2=0x55c6f920c9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 -- 172.21.15.28:0/274893687 shutdown_connections 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.351+0000 7f1836f06b80 1 -- 172.21.15.28:0/274893687 wait complete. 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.352+0000 7f1836f06b80 1 -- start start 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.352+0000 7f1836f06b80 1 Processor -- start 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.353+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f922e720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.353+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f922ec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.353+0000 7f1836f06b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f928e4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.353+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f922e720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.353+0000 7f1829c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f922e720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57612/0 (socket says 172.21.15.28:57612) 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.353+0000 7f1829200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f922ec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.353+0000 7f1829200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f922ec60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41246/0 (socket says 172.21.15.28:41246) 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.354+0000 7f182a600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f928e4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.354+0000 7f182a600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f928e4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38966/0 (socket says 172.21.15.28:38966) 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.354+0000 7f1829c00700 1 -- 172.21.15.28:0/337131324 learned_addr learned my addr 172.21.15.28:0/337131324 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.354+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f18241be2c0 con 0x55c6f9212880 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.354+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f18241bdbf0 con 0x55c6f92187d0 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.354+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f18241badf0 con 0x55c6f9217eb0 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1829c00700 1 -- 172.21.15.28:0/337131324 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 msgr2=0x55c6f922ec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1829c00700 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f922ec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1829c00700 1 -- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 msgr2=0x55c6f928e4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:10.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1836f06b80 10 client.0 _ll_register_callbacks cb 0x55c6f910ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1829c00700 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f928e4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1829c00700 1 -- 172.21.15.28:0/337131324 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c6f928ea60 con 0x55c6f9212880 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f182a600700 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f928e4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f18180044e0 con 0x55c6f9212880 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1829c00700 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f922e720 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f18140a9aa0 tx=0x7f18140aa1a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f593eeafaff3535 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.355+0000 7f1807400700 10 client.16203 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.356+0000 7f1807400700 1 -- 172.21.15.28:0/337131324 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f18140b0240 con 0x55c6f9212880 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.356+0000 7f1807400700 1 -- 172.21.15.28:0/337131324 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f18140b0040 con 0x55c6f9212880 2024-05-20T20:10:10.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.356+0000 7f1807400700 1 -- 172.21.15.28:0/337131324 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f18140b0240 con 0x55c6f9212880 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 10 client.16203 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({fsmap=29}) -- 0x55c6f9295c80 con 0x55c6f9212880 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1807400700 1 -- 172.21.15.28:0/337131324 <== mon.0 v2:172.21.15.28:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f18140a63a0 con 0x55c6f9212880 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 10 client.16203 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 10 client.16203 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 20 client.16203 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f9298070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 4 client.16203 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:10:10.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.358+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55c6f916c570 con 0x55c6f922bc30 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.359+0000 7f1829200700 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f9298070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.360+0000 7f1807400700 10 client.16203 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.360+0000 7f1829200700 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f9298070 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=d3d00ee431c6a54f server_cookie=f1e40f451fc08333 in_seq=0 out_seq=0 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.360+0000 7f1807400700 1 -- 172.21.15.28:0/337131324 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55c6f916c570 con 0x55c6f922bc30 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.360+0000 7f1807400700 10 client.16203 handle_command_reply: tid=0 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.360+0000 7f1836f06b80 1 client.16203 shutdown 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.360+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 msgr2=0x55c6f922e720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f922e720 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f18140a9aa0 tx=0x7f18140aa1a0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 shutdown_connections 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55c6f922bc30 0x55c6f9298070 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c6f92187d0 0x55c6f928e4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c6f9212880 0x55c6f922e720 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 --2- 172.21.15.28:0/337131324 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c6f9217eb0 0x55c6f922ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 >> 172.21.15.28:0/337131324 conn(0x55c6f9140be0 msgr2=0x55c6f9218bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:10.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1807400700 0 client.16203 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:10:10.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.361+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 shutdown_connections 2024-05-20T20:10:10.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.362+0000 7f1836f06b80 1 -- 172.21.15.28:0/337131324 wait complete. 2024-05-20T20:10:10.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.362+0000 7f1836f06b80 20 client.16203 trim_cache size 0 max 16384 2024-05-20T20:10:10.364 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:10:10.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.363+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f180c0b2af0 msgr2=0x7f180c0b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.363+0000 7f182c600700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f180c0b2af0 0x7f180c0b4fb0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f18180123c0 tx=0x7f181800d040 comp rx=0 tx=0).stop 2024-05-20T20:10:10.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.363+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182410aa80 msgr2=0x7f18241ae770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:10.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.363+0000 7f182c600700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182410aa80 0x7f18241ae770 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f181400e9f0 tx=0x7f181400ed00 comp rx=0 tx=0).stop 2024-05-20T20:10:10.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.364+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 shutdown_connections 2024-05-20T20:10:10.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.364+0000 7f182c600700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f182411cd20 0x7f18241a8930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.364+0000 7f182c600700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f182410aa80 0x7f18241ae770 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.364+0000 7f182c600700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f180c0b2af0 0x7f180c0b4fb0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.364+0000 7f182c600700 1 --2- 172.21.15.28:0/54939939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1824111d20 0x7f18241aecb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:10.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.364+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 >> 172.21.15.28:0/54939939 conn(0x7f182406e6b0 msgr2=0x7f182411ad10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:10.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.365+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 shutdown_connections 2024-05-20T20:10:10.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:10.365+0000 7f182c600700 1 -- 172.21.15.28:0/54939939 wait complete. 2024-05-20T20:10:11.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:10 smithi028 ceph-mon[31178]: pgmap v139: 97 pgs: 97 active+clean; 19 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 373 MiB/s wr, 1.92k op/s 2024-05-20T20:10:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:10 smithi154 ceph-mon[36314]: pgmap v139: 97 pgs: 97 active+clean; 19 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 373 MiB/s wr, 1.92k op/s 2024-05-20T20:10:11.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:10 smithi033 ceph-mon[36179]: pgmap v139: 97 pgs: 97 active+clean; 19 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 373 MiB/s wr, 1.92k op/s 2024-05-20T20:10:11.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.502+0000 7f7560a00640 1 -- 172.21.15.28:0/924235557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c10a370 msgr2=0x7f755c115660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.502+0000 7f7560a00640 1 --2- 172.21.15.28:0/924235557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c10a370 0x7f755c115660 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f754c009890 tx=0x7f754c02a3f0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.503+0000 7f7560a00640 1 -- 172.21.15.28:0/924235557 shutdown_connections 2024-05-20T20:10:11.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.503+0000 7f7560a00640 1 --2- 172.21.15.28:0/924235557 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c10a370 0x7f755c115660 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.503+0000 7f7560a00640 1 --2- 172.21.15.28:0/924235557 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f755c115ba0 0x7f755c117f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.503+0000 7f7560a00640 1 --2- 172.21.15.28:0/924235557 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f755c109a50 0x7f755c109e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.503+0000 7f7560a00640 1 -- 172.21.15.28:0/924235557 >> 172.21.15.28:0/924235557 conn(0x7f755c06db20 msgr2=0x7f755c06df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:11.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.507+0000 7f7560a00640 1 -- 172.21.15.28:0/924235557 shutdown_connections 2024-05-20T20:10:11.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.511+0000 7f7560a00640 1 -- 172.21.15.28:0/924235557 wait complete. 2024-05-20T20:10:11.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.512+0000 7f7560a00640 1 Processor -- start 2024-05-20T20:10:11.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.512+0000 7f7560a00640 1 -- start start 2024-05-20T20:10:11.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f7560a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f755c109a50 0x7f755c1a7e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f7560a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f755c10a370 0x7f755c1a8390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f7560a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c115ba0 0x7f755c1ac710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f7560a00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f755c121af0 con 0x7f755c10a370 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f7560a00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f755c121970 con 0x7f755c115ba0 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f7560a00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f755c121c70 con 0x7f755c109a50 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f755a800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c115ba0 0x7f755c1ac710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f755a800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c115ba0 0x7f755c1ac710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38982/0 (socket says 172.21.15.28:38982) 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f7559e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f755c109a50 0x7f755c1a7e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.513+0000 7f755a800640 1 -- 172.21.15.28:0/686937833 learned_addr learned my addr 172.21.15.28:0/686937833 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.514+0000 7f7559e00640 1 -- 172.21.15.28:0/686937833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c115ba0 msgr2=0x7f755c1ac710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.514+0000 7f7559e00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c115ba0 0x7f755c1ac710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.514+0000 7f7559e00640 1 -- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f755c10a370 msgr2=0x7f755c1a8390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.514+0000 7f7559e00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f755c10a370 0x7f755c1a8390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.514+0000 7f7559e00640 1 -- 172.21.15.28:0/686937833 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f755c1ace40 con 0x7f755c109a50 2024-05-20T20:10:11.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.514+0000 7f755a800640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c115ba0 0x7f755c1ac710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:11.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.515+0000 7f7559e00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f755c109a50 0x7f755c1a7e50 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f755400bea0 tx=0x7f75540095b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:11.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.516+0000 7f7552a00640 1 -- 172.21.15.28:0/686937833 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7554009c00 con 0x7f755c109a50 2024-05-20T20:10:11.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.516+0000 7f7552a00640 1 -- 172.21.15.28:0/686937833 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7554009da0 con 0x7f755c109a50 2024-05-20T20:10:11.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.516+0000 7f7552a00640 1 -- 172.21.15.28:0/686937833 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7554015720 con 0x7f755c109a50 2024-05-20T20:10:11.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.518+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f755c1ad130 con 0x7f755c109a50 2024-05-20T20:10:11.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.518+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f755c07acb0 con 0x7f755c109a50 2024-05-20T20:10:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.525+0000 7f7552a00640 1 -- 172.21.15.28:0/686937833 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7554016a60 con 0x7f755c109a50 2024-05-20T20:10:11.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.525+0000 7f7563870740 1 Processor -- start 2024-05-20T20:10:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.525+0000 7f7563870740 1 -- start start 2024-05-20T20:10:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.526+0000 7f7563870740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b51070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.526+0000 7f7563870740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b5b930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.526+0000 7f7563870740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b5e260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.526+0000 7f7563870740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561888ac1b90 con 0x561888b515b0 2024-05-20T20:10:11.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.526+0000 7f7563870740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561888ab1730 con 0x561888b5be70 2024-05-20T20:10:11.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.526+0000 7f7563870740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561888ab5770 con 0x561888b50c90 2024-05-20T20:10:11.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.527+0000 7f755a800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b51070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.527+0000 7f755a800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b51070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41266/0 (socket says 172.21.15.28:41266) 2024-05-20T20:10:11.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.527+0000 7f755a800640 1 -- 172.21.15.28:0/713416210 learned_addr learned my addr 172.21.15.28:0/713416210 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:11.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.527+0000 7f7559400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b5e260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.527+0000 7f7559e00640 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b5b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.528+0000 7f7559400640 1 -- 172.21.15.28:0/713416210 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 msgr2=0x561888b51070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.528+0000 7f7559400640 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b51070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.528+0000 7f7559400640 1 -- 172.21.15.28:0/713416210 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 msgr2=0x561888b5b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.528+0000 7f7559400640 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b5b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.528+0000 7f7559400640 1 -- 172.21.15.28:0/713416210 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561888a4a8b0 con 0x561888b5be70 2024-05-20T20:10:11.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.528+0000 7f755a800640 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b51070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.528+0000 7f7559400640 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b5e260 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f754c009780 tx=0x7f754c009490 comp rx=0 tx=0).ready entity=mon.2 client_cookie=763794679d039dec server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.529+0000 7f7547000640 1 -- 172.21.15.28:0/713416210 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f754c037040 con 0x561888b5be70 2024-05-20T20:10:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.529+0000 7f7547000640 1 -- 172.21.15.28:0/713416210 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f754c02ad70 con 0x561888b5be70 2024-05-20T20:10:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.529+0000 7f7547000640 1 -- 172.21.15.28:0/713416210 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f754c004920 con 0x561888b5be70 2024-05-20T20:10:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.530+0000 7f7563870740 1 -- 172.21.15.28:0/713416210 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 msgr2=0x561888b5e260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.530+0000 7f7563870740 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b5e260 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f754c009780 tx=0x7f754c009490 comp rx=0 tx=0).stop 2024-05-20T20:10:11.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.530+0000 7f7563870740 1 -- 172.21.15.28:0/713416210 shutdown_connections 2024-05-20T20:10:11.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.530+0000 7f7563870740 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b5e260 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.530+0000 7f7563870740 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b5b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.530+0000 7f7563870740 1 --2- 172.21.15.28:0/713416210 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b51070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.530+0000 7f7563870740 1 -- 172.21.15.28:0/713416210 >> 172.21.15.28:0/713416210 conn(0x561888b4d1a0 msgr2=0x561888b4d580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:11.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.531+0000 7f7563870740 1 -- 172.21.15.28:0/713416210 shutdown_connections 2024-05-20T20:10:11.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.531+0000 7f7563870740 1 -- 172.21.15.28:0/713416210 wait complete. 2024-05-20T20:10:11.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.531+0000 7f7552a00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75400b2ae0 0x7f75400b4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.532+0000 7f7563870740 1 -- start start 2024-05-20T20:10:11.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.532+0000 7f7563870740 1 Processor -- start 2024-05-20T20:10:11.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.532+0000 7f7552a00640 1 -- 172.21.15.28:0/686937833 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f75540e2160 con 0x7f755c109a50 2024-05-20T20:10:11.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.533+0000 7f7559400640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75400b2ae0 0x7f75400b4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.533+0000 7f7563870740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b58c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.533+0000 7f7563870740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b59190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.533+0000 7f7563870740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b596d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.534+0000 7f7559e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b58c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.534+0000 7f7559e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b58c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41282/0 (socket says 172.21.15.28:41282) 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.534+0000 7f7559e00640 1 -- 172.21.15.28:0/1465824142 learned_addr learned my addr 172.21.15.28:0/1465824142 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.534+0000 7f7559400640 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b59190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.534+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561888797790 con 0x561888b515b0 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.534+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561888a59bc0 con 0x561888b5be70 2024-05-20T20:10:11.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.534+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561888b5ea40 con 0x561888b50c90 2024-05-20T20:10:11.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.535+0000 7f755a800640 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b596d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.535+0000 7f7563870740 10 client.0 _ll_register_callbacks cb 0x561888a37400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:11.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.535+0000 7f7559400640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75400b2ae0 0x7f75400b4fa0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f754c035040 tx=0x7f754c000bc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:11.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.536+0000 7f755a800640 1 -- 172.21.15.28:0/1465824142 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 msgr2=0x561888b58c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:11.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.536+0000 7f755a800640 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b58c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.536+0000 7f755a800640 1 -- 172.21.15.28:0/1465824142 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 msgr2=0x561888b59190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:11.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.536+0000 7f755a800640 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b59190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.536+0000 7f755a800640 1 -- 172.21.15.28:0/1465824142 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5618887c6210 con 0x561888b5be70 2024-05-20T20:10:11.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.536+0000 7f755a800640 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b596d0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f75480096c0 tx=0x7f754800f790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eff01b2eb9546f31 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:11.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.536+0000 7f753b400640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:11.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.537+0000 7f753b400640 1 -- 172.21.15.28:0/1465824142 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f754801e020 con 0x561888b5be70 2024-05-20T20:10:11.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.537+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f754c02ad70 con 0x561888b5be70 2024-05-20T20:10:11.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.538+0000 7f753b400640 1 -- 172.21.15.28:0/1465824142 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f7548005020 con 0x561888b5be70 2024-05-20T20:10:11.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.538+0000 7f753b400640 1 -- 172.21.15.28:0/1465824142 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f7548005220 con 0x561888b5be70 2024-05-20T20:10:11.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.538+0000 7f7563870740 10 client.26414 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:11.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.538+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561888892620 con 0x561888b5be70 2024-05-20T20:10:11.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.538+0000 7f753b400640 1 -- 172.21.15.28:0/1465824142 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f754c02ad70 con 0x561888b5be70 2024-05-20T20:10:11.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.539+0000 7f753b400640 1 -- 172.21.15.28:0/1465824142 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7548019070 con 0x561888b5be70 2024-05-20T20:10:11.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.539+0000 7f7563870740 10 client.26414 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:11.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.539+0000 7f7563870740 10 client.26414 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:11.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.539+0000 7f7563870740 20 client.26414 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:11.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.539+0000 7f7563870740 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888b6b9c0 0x561888b6de50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.540+0000 7f7563870740 4 client.26414 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:11.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.540+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561888ac3ae0 con 0x561888b6b9c0 2024-05-20T20:10:11.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.540+0000 7f7559e00640 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888b6b9c0 0x561888b6de50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.543+0000 7f7559e00640 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888b6b9c0 0x561888b6de50 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=af754b523d757cde server_cookie=a3c85ff66109544c in_seq=0 out_seq=0 2024-05-20T20:10:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.543+0000 7f753b400640 10 client.26414 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.544+0000 7f753b400640 1 -- 172.21.15.28:0/1465824142 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561888ac3ae0 con 0x561888b6b9c0 2024-05-20T20:10:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.544+0000 7f753b400640 10 client.26414 handle_command_reply: tid=0 2024-05-20T20:10:11.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.544+0000 7f7563870740 1 client.26414 shutdown 2024-05-20T20:10:11.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.546+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 msgr2=0x561888b596d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.546+0000 7f7563870740 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b596d0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f75480096c0 tx=0x7f754800f790 comp rx=0 tx=0).stop 2024-05-20T20:10:11.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.547+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 shutdown_connections 2024-05-20T20:10:11.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.547+0000 7f7563870740 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888b6b9c0 0x561888b6de50 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.547+0000 7f7563870740 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b5be70 0x561888b596d0 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.547+0000 7f7563870740 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b59190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.547+0000 7f7563870740 1 --2- 172.21.15.28:0/1465824142 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b50c90 0x561888b58c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.547+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 >> 172.21.15.28:0/1465824142 conn(0x561888b4d1a0 msgr2=0x561888b52f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:11.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.548+0000 7f753b400640 0 client.26414 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:11.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.548+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 shutdown_connections 2024-05-20T20:10:11.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.548+0000 7f7563870740 1 -- 172.21.15.28:0/1465824142 wait complete. 2024-05-20T20:10:11.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.548+0000 7f7563870740 20 client.26414 trim_cache size 0 max 16384 2024-05-20T20:10:11.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.579+0000 7f7563870740 1 Processor -- start 2024-05-20T20:10:11.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.580+0000 7f7563870740 1 -- start start 2024-05-20T20:10:11.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.580+0000 7f7563870740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b50c90 0x561888b53160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.580+0000 7f7563870740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b515b0 0x561888b67aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.580+0000 7f7563870740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b73730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.580+0000 7f7563870740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f755c083600 con 0x561888b50c90 2024-05-20T20:10:11.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.580+0000 7f7563870740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f755c085140 con 0x561888b515b0 2024-05-20T20:10:11.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.580+0000 7f7563870740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f755c083cd0 con 0x561888b5be70 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b515b0 0x561888b67aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b515b0 0x561888b67aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:39024/0 (socket says 172.21.15.28:39024) 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 -- 172.21.15.28:0/3309519080 learned_addr learned my addr 172.21.15.28:0/3309519080 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559e00640 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b50c90 0x561888b53160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f755a800640 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b73730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 -- 172.21.15.28:0/3309519080 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 msgr2=0x561888b73730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b73730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 -- 172.21.15.28:0/3309519080 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b50c90 msgr2=0x561888b53160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:11.582 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b50c90 0x561888b53160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.582 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.581+0000 7f7559400640 1 -- 172.21.15.28:0/3309519080 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561888b4db20 con 0x561888b515b0 2024-05-20T20:10:11.582 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.582+0000 7f7559400640 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b515b0 0x561888b67aa0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f754c009230 tx=0x7f754c005d50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c2cb3ef0500b53b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:11.582 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.582+0000 7f7546600640 1 -- 172.21.15.28:0/3309519080 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f754c04b040 con 0x561888b515b0 2024-05-20T20:10:11.582 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.582+0000 7f7546600640 1 -- 172.21.15.28:0/3309519080 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f754c005250 con 0x561888b515b0 2024-05-20T20:10:11.583 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.582+0000 7f7546600640 1 -- 172.21.15.28:0/3309519080 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f754c037050 con 0x561888b515b0 2024-05-20T20:10:11.585 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.585+0000 7f7563870740 1 -- 172.21.15.28:0/3309519080 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b515b0 msgr2=0x561888b67aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.585 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.585+0000 7f7563870740 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b515b0 0x561888b67aa0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f754c009230 tx=0x7f754c005d50 comp rx=0 tx=0).stop 2024-05-20T20:10:11.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.586+0000 7f7563870740 1 -- 172.21.15.28:0/3309519080 shutdown_connections 2024-05-20T20:10:11.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.586+0000 7f7563870740 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b515b0 0x561888b67aa0 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.586+0000 7f7563870740 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b73730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.586+0000 7f7563870740 1 --2- 172.21.15.28:0/3309519080 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b50c90 0x561888b53160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.586+0000 7f7563870740 1 -- 172.21.15.28:0/3309519080 >> 172.21.15.28:0/3309519080 conn(0x561888b4d1a0 msgr2=0x561888b594a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:11.586 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.587+0000 7f7563870740 1 -- 172.21.15.28:0/3309519080 shutdown_connections 2024-05-20T20:10:11.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.587+0000 7f7563870740 1 -- 172.21.15.28:0/3309519080 wait complete. 2024-05-20T20:10:11.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.588+0000 7f7563870740 1 -- start start 2024-05-20T20:10:11.589 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.588+0000 7f7563870740 1 Processor -- start 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.589+0000 7f7563870740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b50c90 0x561888b64060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.589+0000 7f7563870740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b55600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.589+0000 7f7563870740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b55b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.589+0000 7f7563870740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f755c084a70 con 0x561888b515b0 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.589+0000 7f7563870740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f755c0843a0 con 0x561888b50c90 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.589+0000 7f7563870740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f755c0507e0 con 0x561888b5be70 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.589+0000 7f7563870740 10 client.0 _ll_register_callbacks cb 0x7f7548009de0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:11.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.590+0000 7f7559e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b50c90 0x561888b64060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.590+0000 7f755a800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b55b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.590+0000 7f755a800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b55b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41300/0 (socket says 172.21.15.28:41300) 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.590+0000 7f755a800640 1 -- 172.21.15.28:0/2668891181 learned_addr learned my addr 172.21.15.28:0/2668891181 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.590+0000 7f755a800640 1 -- 172.21.15.28:0/2668891181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b50c90 msgr2=0x561888b64060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.590+0000 7f755a800640 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b50c90 0x561888b64060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.590+0000 7f755a800640 1 -- 172.21.15.28:0/2668891181 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 msgr2=0x561888b55600 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.591+0000 7f755a800640 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b55600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.591+0000 7f755a800640 1 -- 172.21.15.28:0/2668891181 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561888b56080 con 0x561888b5be70 2024-05-20T20:10:11.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.592+0000 7f755a800640 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b55b40 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f75480051e0 tx=0x7f7548009cc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3280e7ef8ff350d5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:11.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.592+0000 7f753b400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:10:11.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.592+0000 7f753b400640 1 -- 172.21.15.28:0/2668891181 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f754802f020 con 0x561888b5be70 2024-05-20T20:10:11.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.593+0000 7f753b400640 1 -- 172.21.15.28:0/2668891181 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f754800e9e0 con 0x561888b5be70 2024-05-20T20:10:11.593 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.593+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f754c005250 con 0x561888b5be70 2024-05-20T20:10:11.593 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.594+0000 7f753b400640 1 -- 172.21.15.28:0/2668891181 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f754801e070 con 0x561888b5be70 2024-05-20T20:10:11.594 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.594+0000 7f7563870740 10 client.26344 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:11.594 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.594+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561888bd6340 con 0x561888b5be70 2024-05-20T20:10:11.594 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.594+0000 7f753b400640 1 -- 172.21.15.28:0/2668891181 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f754c005250 con 0x561888b5be70 2024-05-20T20:10:11.594 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.595+0000 7f753b400640 1 -- 172.21.15.28:0/2668891181 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f7548019040 con 0x561888b5be70 2024-05-20T20:10:11.595 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.596+0000 7f7563870740 10 client.26344 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:11.596 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.596+0000 7f7563870740 10 client.26344 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:11.596 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.596+0000 7f7563870740 20 client.26344 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:11.596 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.596+0000 7f7563870740 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888bd6a70 0x561888bd8eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:11.596 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.596+0000 7f7563870740 4 client.26344 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:10:11.596 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.596+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x561888b4cff0 con 0x561888bd6a70 2024-05-20T20:10:11.596 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.597+0000 7f7559e00640 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888bd6a70 0x561888bd8eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:11.597 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.597+0000 7f7559e00640 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888bd6a70 0x561888bd8eb0 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4839250e1e7c30ba server_cookie=ccca089e94d54ac in_seq=0 out_seq=0 2024-05-20T20:10:11.597 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.597+0000 7f753b400640 10 client.26344 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:11.624 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.624+0000 7f753b400640 1 -- 172.21.15.28:0/2668891181 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x561888b4cff0 con 0x561888bd6a70 2024-05-20T20:10:11.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.624+0000 7f753b400640 10 client.26344 handle_command_reply: tid=0 2024-05-20T20:10:11.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.625+0000 7f7563870740 1 client.26344 shutdown 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.625+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 msgr2=0x561888b55b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.625+0000 7f7563870740 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b55b40 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f75480051e0 tx=0x7f7548009cc0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 shutdown_connections 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561888bd6a70 0x561888bd8eb0 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561888b5be70 0x561888b55b40 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561888b515b0 0x561888b55600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 --2- 172.21.15.28:0/2668891181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561888b50c90 0x561888b64060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 >> 172.21.15.28:0/2668891181 conn(0x561888b4d1a0 msgr2=0x561888b58a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f753b400640 0 client.26344 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 shutdown_connections 2024-05-20T20:10:11.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.626+0000 7f7563870740 1 -- 172.21.15.28:0/2668891181 wait complete. 2024-05-20T20:10:11.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.627+0000 7f7563870740 20 client.26344 trim_cache size 0 max 16384 2024-05-20T20:10:11.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:10:11.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:10:11.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.628+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75400b2ae0 msgr2=0x7f75400b4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.628+0000 7f7560a00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75400b2ae0 0x7f75400b4fa0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f754c035040 tx=0x7f754c000bc0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.628+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f755c109a50 msgr2=0x7f755c1a7e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.628+0000 7f7560a00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f755c109a50 0x7f755c1a7e50 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f755400bea0 tx=0x7f75540095b0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.629+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 shutdown_connections 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.629+0000 7f7560a00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f755c115ba0 0x7f755c1ac710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.629+0000 7f7560a00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75400b2ae0 0x7f75400b4fa0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.629+0000 7f7560a00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f755c10a370 0x7f755c1a8390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.629+0000 7f7560a00640 1 --2- 172.21.15.28:0/686937833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f755c109a50 0x7f755c1a7e50 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.629+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 >> 172.21.15.28:0/686937833 conn(0x7f755c06db20 msgr2=0x7f755c0715d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:11.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.630+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 shutdown_connections 2024-05-20T20:10:11.663 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:11.630+0000 7f7560a00640 1 -- 172.21.15.28:0/686937833 wait complete. 2024-05-20T20:10:11.971 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:10:12.821 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:12 smithi154 ceph-mon[36314]: pgmap v140: 97 pgs: 97 active+clean; 19 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 242 MiB/s wr, 1.45k op/s 2024-05-20T20:10:13.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:12 smithi033 ceph-mon[36179]: pgmap v140: 97 pgs: 97 active+clean; 19 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 242 MiB/s wr, 1.45k op/s 2024-05-20T20:10:13.238 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:12 smithi028 ceph-mon[31178]: pgmap v140: 97 pgs: 97 active+clean; 19 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 242 MiB/s wr, 1.45k op/s 2024-05-20T20:10:14.543 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 24.7 2024-05-20T20:10:14.543 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:10:15.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:14 smithi028 ceph-mon[31178]: pgmap v141: 97 pgs: 97 active+clean; 19 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 337 MiB/s wr, 2.21k op/s 2024-05-20T20:10:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:14 smithi154 ceph-mon[36314]: pgmap v141: 97 pgs: 97 active+clean; 19 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 337 MiB/s wr, 2.21k op/s 2024-05-20T20:10:15.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:14 smithi033 ceph-mon[36179]: pgmap v141: 97 pgs: 97 active+clean; 19 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 337 MiB/s wr, 2.21k op/s 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.642+0000 7f4e06a00700 1 -- 172.21.15.28:0/1935531030 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df809cf90 msgr2=0x7f4df80a0f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.642+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1935531030 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df809cf90 0x7f4df80a0f90 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f4dec009990 tx=0x7f4dec009ca0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.644+0000 7f4e06a00700 1 -- 172.21.15.28:0/1935531030 shutdown_connections 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.644+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1935531030 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4df80a1670 0x7f4df80a9240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.644+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1935531030 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4df809c5e0 0x7f4df809c9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.644+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1935531030 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df809cf90 0x7f4df80a0f90 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.644+0000 7f4e06a00700 1 -- 172.21.15.28:0/1935531030 >> 172.21.15.28:0/1935531030 conn(0x7f4df800c880 msgr2=0x7f4df809a800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.645+0000 7f4e06a00700 1 -- 172.21.15.28:0/1935531030 shutdown_connections 2024-05-20T20:10:15.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.646+0000 7f4e06a00700 1 -- 172.21.15.28:0/1935531030 wait complete. 2024-05-20T20:10:15.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.648+0000 7f4e06a00700 1 Processor -- start 2024-05-20T20:10:15.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.649+0000 7f4e06a00700 1 -- start start 2024-05-20T20:10:15.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.650+0000 7f4e06a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4df809c5e0 0x7f4df8138710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.650+0000 7f4e06a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4df809cf90 0x7f4df8138c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.650+0000 7f4e06a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df80a1670 0x7f4df81328d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.650+0000 7f4e06a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4df80ac590 con 0x7f4df809c5e0 2024-05-20T20:10:15.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.650+0000 7f4e06a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4df80ac410 con 0x7f4df80a1670 2024-05-20T20:10:15.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.650+0000 7f4e06a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4df80ac710 con 0x7f4df809cf90 2024-05-20T20:10:15.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.651+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4df809cf90 0x7f4df8138c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.651+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4df809cf90 0x7f4df8138c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41310/0 (socket says 172.21.15.28:41310) 2024-05-20T20:10:15.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.651+0000 7f4e04c00700 1 -- 172.21.15.28:0/1690331340 learned_addr learned my addr 172.21.15.28:0/1690331340 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:15.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.652+0000 7f4e08000700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df80a1670 0x7f4df81328d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.653 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:15.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.653+0000 7f4e08000700 1 -- 172.21.15.28:0/1690331340 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4df809cf90 msgr2=0x7f4df8138c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.653+0000 7f4e08000700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4df809cf90 0x7f4df8138c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.653+0000 7f4e08000700 1 -- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4df809c5e0 msgr2=0x7f4df8138710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.653+0000 7f4e08000700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4df809c5e0 0x7f4df8138710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.653+0000 7f4e08000700 1 -- 172.21.15.28:0/1690331340 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4df8133030 con 0x7f4df80a1670 2024-05-20T20:10:15.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.654+0000 7f4e08000700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df80a1670 0x7f4df81328d0 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f4e00000eb0 tx=0x7f4e0006dc70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.654+0000 7f4dfe000700 1 -- 172.21.15.28:0/1690331340 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e000725e0 con 0x7f4df80a1670 2024-05-20T20:10:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.654+0000 7f4dfe000700 1 -- 172.21.15.28:0/1690331340 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e00072c30 con 0x7f4df80a1670 2024-05-20T20:10:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.654+0000 7f4dfe000700 1 -- 172.21.15.28:0/1690331340 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e00078e10 con 0x7f4df80a1670 2024-05-20T20:10:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.655+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4df8133320 con 0x7f4df80a1670 2024-05-20T20:10:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.656+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4df81505f0 con 0x7f4df80a1670 2024-05-20T20:10:15.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.664+0000 7f4dfe000700 1 -- 172.21.15.28:0/1690331340 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4e00078540 con 0x7f4df80a1670 2024-05-20T20:10:15.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.668+0000 7f4e128d9b80 1 Processor -- start 2024-05-20T20:10:15.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.679+0000 7f4dfe000700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4de80b2b40 0x7f4de80b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.679+0000 7f4e128d9b80 1 -- start start 2024-05-20T20:10:15.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.684+0000 7f4e04c00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4de80b2b40 0x7f4de80b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.686+0000 7f4e04c00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4de80b2b40 0x7f4de80b5000 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f4df80a29f0 tx=0x7f4dec00b580 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:15.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.686+0000 7f4dfe000700 1 -- 172.21.15.28:0/1690331340 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f4e00145ed0 con 0x7f4df80a1670 2024-05-20T20:10:15.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.700+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a5507c0 0x55e77a550ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.700+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5510e0 0x55e77a54ad70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.700+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a54b390 0x55e77a54b7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.701+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e77a4a8e40 con 0x55e77a5507c0 2024-05-20T20:10:15.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.701+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e77a4989e0 con 0x55e77a54b390 2024-05-20T20:10:15.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.701+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e77a49ca20 con 0x55e77a5510e0 2024-05-20T20:10:15.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.703+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5510e0 0x55e77a54ad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.705+0000 7f4e05600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a5507c0 0x55e77a550ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.705+0000 7f4e05600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a5507c0 0x55e77a550ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57724/0 (socket says 172.21.15.28:57724) 2024-05-20T20:10:15.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.705+0000 7f4e05600700 1 -- 172.21.15.28:0/1492703421 learned_addr learned my addr 172.21.15.28:0/1492703421 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:15.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.705+0000 7f4e05600700 1 -- 172.21.15.28:0/1492703421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5510e0 msgr2=0x55e77a54ad70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:15.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.705+0000 7f4e05600700 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5510e0 0x55e77a54ad70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.706+0000 7f4e05600700 1 -- 172.21.15.28:0/1492703421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a54b390 msgr2=0x55e77a54b7f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:15.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.707+0000 7f4e08000700 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a54b390 0x55e77a54b7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.707+0000 7f4e05600700 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a54b390 0x55e77a54b7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.707+0000 7f4e05600700 1 -- 172.21.15.28:0/1492703421 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e77a461420 con 0x55e77a5507c0 2024-05-20T20:10:15.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.707+0000 7f4e08000700 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a54b390 0x55e77a54b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:15.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.712+0000 7f4e05600700 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a5507c0 0x55e77a550ba0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f4df400aa50 tx=0x7f4df400ad60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2755b6d1bb54b8b4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:15.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.730+0000 7f4df2a00700 1 -- 172.21.15.28:0/1492703421 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4df400cc10 con 0x55e77a5507c0 2024-05-20T20:10:15.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.730+0000 7f4df2a00700 1 -- 172.21.15.28:0/1492703421 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4df40044d0 con 0x55e77a5507c0 2024-05-20T20:10:15.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.731+0000 7f4df2a00700 1 -- 172.21.15.28:0/1492703421 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4df400f450 con 0x55e77a5507c0 2024-05-20T20:10:15.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.732+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1492703421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a5507c0 msgr2=0x55e77a550ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.732+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a5507c0 0x55e77a550ba0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f4df400aa50 tx=0x7f4df400ad60 comp rx=0 tx=0).stop 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.732+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1492703421 shutdown_connections 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.732+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a54b390 0x55e77a54b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.732+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a5507c0 0x55e77a550ba0 secure :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f4df400aa50 tx=0x7f4df400ad60 comp rx=0 tx=0).stop 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.732+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1492703421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5510e0 0x55e77a54ad70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.732+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1492703421 >> 172.21.15.28:0/1492703421 conn(0x55e77a478be0 msgr2=0x55e77a545810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.733+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1492703421 shutdown_connections 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.733+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1492703421 wait complete. 2024-05-20T20:10:15.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.734+0000 7f4e128d9b80 1 -- start start 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.735+0000 7f4e128d9b80 1 Processor -- start 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a557080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a554380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a554d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e77a39efa0 con 0x55e77a54b390 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e77a439760 con 0x55e77a5510e0 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e77a557c00 con 0x55e77a5548c0 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e128d9b80 10 client.0 _ll_register_callbacks cb 0x55e77a442c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a554380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a554380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:39084/0 (socket says 172.21.15.28:39084) 2024-05-20T20:10:15.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.736+0000 7f4e04c00700 1 -- 172.21.15.28:0/1691913938 learned_addr learned my addr 172.21.15.28:0/1691913938 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:15.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.738+0000 7f4e04c00700 1 -- 172.21.15.28:0/1691913938 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 msgr2=0x55e77a554d20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:15.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.738+0000 7f4e04c00700 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a554d20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.738+0000 7f4e04c00700 1 -- 172.21.15.28:0/1691913938 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 msgr2=0x55e77a557080 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:15.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.738+0000 7f4e04c00700 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a557080 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.738+0000 7f4e04c00700 1 -- 172.21.15.28:0/1691913938 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e77a3e88d0 con 0x55e77a5510e0 2024-05-20T20:10:15.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.740+0000 7f4e04c00700 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a554380 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f4dec025710 tx=0x7f4dec009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b8d620208058b5ae server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:15.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.740+0000 7f4de7400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:15.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.742+0000 7f4de7400700 1 -- 172.21.15.28:0/1691913938 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4dec035060 con 0x55e77a5510e0 2024-05-20T20:10:15.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.742+0000 7f4de7400700 1 -- 172.21.15.28:0/1691913938 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f4dec025cb0 con 0x55e77a5510e0 2024-05-20T20:10:15.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.743+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4df40044d0 con 0x55e77a5510e0 2024-05-20T20:10:15.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.744+0000 7f4de7400700 1 -- 172.21.15.28:0/1691913938 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4dec00be20 con 0x55e77a5510e0 2024-05-20T20:10:15.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.744+0000 7f4e128d9b80 10 client.26429 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:15.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.745+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e77a469060 con 0x55e77a5510e0 2024-05-20T20:10:15.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.745+0000 7f4de7400700 1 -- 172.21.15.28:0/1691913938 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4df40044d0 con 0x55e77a5510e0 2024-05-20T20:10:15.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.745+0000 7f4de7400700 1 -- 172.21.15.28:0/1691913938 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4dec030490 con 0x55e77a5510e0 2024-05-20T20:10:15.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.746+0000 7f4e128d9b80 10 client.26429 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.746+0000 7f4e128d9b80 10 client.26429 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.746+0000 7f4e128d9b80 20 client.26429 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.746+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a575cd0 0x55e77a5780e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.746+0000 7f4e128d9b80 4 client.26429 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.746+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e77a4aad90 con 0x55e77a575cd0 2024-05-20T20:10:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.748+0000 7f4e05600700 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a575cd0 0x55e77a5780e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.750+0000 7f4de7400700 10 client.26429 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:15.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.750+0000 7f4e05600700 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a575cd0 0x55e77a5780e0 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7eb13e68d22d8f8d server_cookie=11fe79a3ebce6984 in_seq=0 out_seq=0 2024-05-20T20:10:15.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.751+0000 7f4de7400700 1 -- 172.21.15.28:0/1691913938 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55e77a4aad90 con 0x55e77a575cd0 2024-05-20T20:10:15.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.751+0000 7f4de7400700 10 client.26429 handle_command_reply: tid=0 2024-05-20T20:10:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.752+0000 7f4e128d9b80 1 client.26429 shutdown 2024-05-20T20:10:15.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 msgr2=0x55e77a554380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a554380 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f4dec025710 tx=0x7f4dec009310 comp rx=0 tx=0).stop 2024-05-20T20:10:15.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 shutdown_connections 2024-05-20T20:10:15.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a575cd0 0x55e77a5780e0 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a554d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a557080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/1691913938 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a554380 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.757+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 >> 172.21.15.28:0/1691913938 conn(0x55e77a478be0 msgr2=0x55e77a544fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.758+0000 7f4de7400700 0 client.26429 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.758+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 shutdown_connections 2024-05-20T20:10:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.759+0000 7f4e128d9b80 1 -- 172.21.15.28:0/1691913938 wait complete. 2024-05-20T20:10:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.759+0000 7f4e128d9b80 20 client.26429 trim_cache size 0 max 16384 2024-05-20T20:10:15.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.810+0000 7f4e128d9b80 1 Processor -- start 2024-05-20T20:10:15.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.810+0000 7f4e128d9b80 1 -- start start 2024-05-20T20:10:15.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.811+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a54ad70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.811+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a54fc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.811+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a545740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.811+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4df8015780 con 0x55e77a54b390 2024-05-20T20:10:15.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.811+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4df8151050 con 0x55e77a5510e0 2024-05-20T20:10:15.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.811+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4df8015e50 con 0x55e77a5548c0 2024-05-20T20:10:15.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.812+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a54fc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.812+0000 7f4e08000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a545740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.813+0000 7f4e08000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a545740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41320/0 (socket says 172.21.15.28:41320) 2024-05-20T20:10:15.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.813+0000 7f4e08000700 1 -- 172.21.15.28:0/914588125 learned_addr learned my addr 172.21.15.28:0/914588125 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:15.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.815+0000 7f4e05600700 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a54ad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.815+0000 7f4e08000700 1 -- 172.21.15.28:0/914588125 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 msgr2=0x55e77a54fc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:15.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.815+0000 7f4e08000700 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a54fc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.815+0000 7f4e08000700 1 -- 172.21.15.28:0/914588125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 msgr2=0x55e77a54ad70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:15.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.819+0000 7f4e08000700 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a54ad70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.819+0000 7f4e08000700 1 -- 172.21.15.28:0/914588125 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e77a556cc0 con 0x55e77a5548c0 2024-05-20T20:10:15.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.819+0000 7f4e04c00700 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a54fc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:15.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.819+0000 7f4e05600700 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a54ad70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:15.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.820+0000 7f4e08000700 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a545740 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f4e00052bd0 tx=0x7f4e000696c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8471ab617390c215 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:15.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.820+0000 7f4df2000700 1 -- 172.21.15.28:0/914588125 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e0010cde0 con 0x55e77a5548c0 2024-05-20T20:10:15.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.821+0000 7f4df2000700 1 -- 172.21.15.28:0/914588125 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e0010d9d0 con 0x55e77a5548c0 2024-05-20T20:10:15.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.821+0000 7f4df2000700 1 -- 172.21.15.28:0/914588125 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4e00115f00 con 0x55e77a5548c0 2024-05-20T20:10:15.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.824+0000 7f4e128d9b80 1 -- 172.21.15.28:0/914588125 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 msgr2=0x55e77a545740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.824+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a545740 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f4e00052bd0 tx=0x7f4e000696c0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.825+0000 7f4e128d9b80 1 -- 172.21.15.28:0/914588125 shutdown_connections 2024-05-20T20:10:15.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.825+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a545740 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.825+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a54ad70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.825+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/914588125 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a54fc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.825+0000 7f4e128d9b80 1 -- 172.21.15.28:0/914588125 >> 172.21.15.28:0/914588125 conn(0x55e77a478be0 msgr2=0x55e77a54f520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:15.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.833+0000 7f4e128d9b80 1 -- 172.21.15.28:0/914588125 shutdown_connections 2024-05-20T20:10:15.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.833+0000 7f4e128d9b80 1 -- 172.21.15.28:0/914588125 wait complete. 2024-05-20T20:10:15.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.834+0000 7f4e128d9b80 1 -- start start 2024-05-20T20:10:15.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.834+0000 7f4e128d9b80 1 Processor -- start 2024-05-20T20:10:15.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.836+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a56e370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.836+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a57cfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.836+0000 7f4e128d9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a57d520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.836+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4df8150980 con 0x55e77a54b390 2024-05-20T20:10:15.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.836+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4df8016520 con 0x55e77a5510e0 2024-05-20T20:10:15.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.836+0000 7f4e128d9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4df8013720 con 0x55e77a5548c0 2024-05-20T20:10:15.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.836+0000 7f4e128d9b80 10 client.0 _ll_register_callbacks cb 0x55e77a572760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:15.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.837+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a57cfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.837+0000 7f4e04c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a57cfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:39102/0 (socket says 172.21.15.28:39102) 2024-05-20T20:10:15.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.837+0000 7f4e04c00700 1 -- 172.21.15.28:0/762712501 learned_addr learned my addr 172.21.15.28:0/762712501 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:15.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.838+0000 7f4e08000700 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a57d520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.838+0000 7f4e04c00700 1 -- 172.21.15.28:0/762712501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 msgr2=0x55e77a57d520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:15.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.838+0000 7f4e04c00700 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a57d520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.838+0000 7f4e04c00700 1 -- 172.21.15.28:0/762712501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 msgr2=0x55e77a56e370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:15.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.838+0000 7f4e04c00700 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a56e370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.838+0000 7f4e04c00700 1 -- 172.21.15.28:0/762712501 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e77a57da60 con 0x55e77a5510e0 2024-05-20T20:10:15.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.841+0000 7f4e04c00700 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a57cfe0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f4dec025710 tx=0x7f4dec030780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f325a5600c85b94b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:15.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.841+0000 7f4de7400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:15.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.844+0000 7f4de7400700 1 -- 172.21.15.28:0/762712501 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4dec035040 con 0x55e77a5510e0 2024-05-20T20:10:15.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.844+0000 7f4de7400700 1 -- 172.21.15.28:0/762712501 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f4dec00f730 con 0x55e77a5510e0 2024-05-20T20:10:15.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.845+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4e0010d9d0 con 0x55e77a5510e0 2024-05-20T20:10:15.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.871+0000 7f4de7400700 1 -- 172.21.15.28:0/762712501 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4dec03f080 con 0x55e77a5510e0 2024-05-20T20:10:15.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.871+0000 7f4e128d9b80 10 client.26435 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:15.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.872+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e77a56dca0 con 0x55e77a5510e0 2024-05-20T20:10:15.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.872+0000 7f4de7400700 1 -- 172.21.15.28:0/762712501 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4e0010d9d0 con 0x55e77a5510e0 2024-05-20T20:10:15.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.876+0000 7f4de7400700 1 -- 172.21.15.28:0/762712501 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4dec00f970 con 0x55e77a5510e0 2024-05-20T20:10:15.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.876+0000 7f4e128d9b80 10 client.26435 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:15.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.876+0000 7f4e128d9b80 10 client.26435 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:15.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.876+0000 7f4e128d9b80 20 client.26435 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:15.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.876+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a5e0d00 0x55e77a5e3120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:15.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.876+0000 7f4e128d9b80 4 client.26435 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:10:15.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.876+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55e77a4a4570 con 0x55e77a5e0d00 2024-05-20T20:10:15.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.877+0000 7f4e05600700 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a5e0d00 0x55e77a5e3120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:15.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.879+0000 7f4de7400700 10 client.26435 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:15.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.880+0000 7f4e05600700 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a5e0d00 0x55e77a5e3120 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=88c15e687b371b87 server_cookie=6c9fcfc72d869c30 in_seq=0 out_seq=0 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.880+0000 7f4de7400700 1 -- 172.21.15.28:0/762712501 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55e77a4a4570 con 0x55e77a5e0d00 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.880+0000 7f4de7400700 10 client.26435 handle_command_reply: tid=0 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.881+0000 7f4e128d9b80 1 client.26435 shutdown 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 msgr2=0x55e77a57cfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a57cfe0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f4dec025710 tx=0x7f4dec030780 comp rx=0 tx=0).stop 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 shutdown_connections 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e77a5e0d00 0x55e77a5e3120 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e77a5548c0 0x55e77a57d520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e77a54b390 0x55e77a56e370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 --2- 172.21.15.28:0/762712501 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e77a5510e0 0x55e77a57cfe0 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 >> 172.21.15.28:0/762712501 conn(0x55e77a478be0 msgr2=0x55e77a553110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:15.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.882+0000 7f4de7400700 0 client.26435 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:15.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.883+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 shutdown_connections 2024-05-20T20:10:15.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.883+0000 7f4e128d9b80 1 -- 172.21.15.28:0/762712501 wait complete. 2024-05-20T20:10:15.886 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:10:15.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.885+0000 7f4e128d9b80 20 client.26435 trim_cache size 0 max 16384 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4de80b2b40 msgr2=0x7f4de80b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4de80b2b40 0x7f4de80b5000 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f4df80a29f0 tx=0x7f4dec00b580 comp rx=0 tx=0).stop 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df80a1670 msgr2=0x7f4df81328d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df80a1670 0x7f4df81328d0 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f4e00000eb0 tx=0x7f4e0006dc70 comp rx=0 tx=0).stop 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 shutdown_connections 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4de80b2b40 0x7f4de80b5000 secure :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f4df80a29f0 tx=0x7f4dec00b580 comp rx=0 tx=0).stop 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4df80a1670 0x7f4df81328d0 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4df809c5e0 0x7f4df8138710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 --2- 172.21.15.28:0/1690331340 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4df809cf90 0x7f4df8138c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:15.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.886+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 >> 172.21.15.28:0/1690331340 conn(0x7f4df800c880 msgr2=0x7f4df80097c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:15.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.887+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 shutdown_connections 2024-05-20T20:10:15.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:15.887+0000 7f4e06a00700 1 -- 172.21.15.28:0/1690331340 wait complete. 2024-05-20T20:10:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:16 smithi154 ceph-mon[36314]: pgmap v142: 97 pgs: 97 active+clean; 20 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 262 MiB/s wr, 1.95k op/s 2024-05-20T20:10:17.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:16 smithi033 ceph-mon[36179]: pgmap v142: 97 pgs: 97 active+clean; 20 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 262 MiB/s wr, 1.95k op/s 2024-05-20T20:10:17.146 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:16 smithi028 ceph-mon[31178]: pgmap v142: 97 pgs: 97 active+clean; 20 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 262 MiB/s wr, 1.95k op/s 2024-05-20T20:10:19.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:18 smithi154 ceph-mon[36314]: pgmap v143: 97 pgs: 97 active+clean; 20 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 235 MiB/s wr, 1.83k op/s 2024-05-20T20:10:19.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:18 smithi033 ceph-mon[36179]: pgmap v143: 97 pgs: 97 active+clean; 20 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 235 MiB/s wr, 1.83k op/s 2024-05-20T20:10:19.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:18 smithi028 ceph-mon[31178]: pgmap v143: 97 pgs: 97 active+clean; 20 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 235 MiB/s wr, 1.83k op/s 2024-05-20T20:10:20.716 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:10:20.717 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:10:21.146 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:20 smithi028 ceph-mon[31178]: pgmap v144: 97 pgs: 97 active+clean; 20 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 342 MiB/s wr, 2.71k op/s 2024-05-20T20:10:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:20 smithi154 ceph-mon[36314]: pgmap v144: 97 pgs: 97 active+clean; 20 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 342 MiB/s wr, 2.71k op/s 2024-05-20T20:10:21.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:20 smithi033 ceph-mon[36179]: pgmap v144: 97 pgs: 97 active+clean; 20 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 342 MiB/s wr, 2.71k op/s 2024-05-20T20:10:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.613+0000 7f6397200700 1 -- 172.21.15.28:0/301889152 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f639010aa80 msgr2=0x7f639010af00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:21.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.613+0000 7f6397200700 1 --2- 172.21.15.28:0/301889152 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f639010aa80 0x7f639010af00 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f637c009990 tx=0x7f637c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:10:21.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.616+0000 7f6397200700 1 -- 172.21.15.28:0/301889152 shutdown_connections 2024-05-20T20:10:21.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.616+0000 7f6397200700 1 --2- 172.21.15.28:0/301889152 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f639011cd20 0x7f639011f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:21.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.616+0000 7f6397200700 1 --2- 172.21.15.28:0/301889152 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6390111d20 0x7f639010a540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:21.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.616+0000 7f6397200700 1 --2- 172.21.15.28:0/301889152 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f639010aa80 0x7f639010af00 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:21.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.616+0000 7f6397200700 1 -- 172.21.15.28:0/301889152 >> 172.21.15.28:0/301889152 conn(0x7f639006e6b0 msgr2=0x7f639006eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:21.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.624+0000 7f6397200700 1 -- 172.21.15.28:0/301889152 shutdown_connections 2024-05-20T20:10:21.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.625+0000 7f6397200700 1 -- 172.21.15.28:0/301889152 wait complete. 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.626+0000 7f6397200700 1 Processor -- start 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.627+0000 7f6397200700 1 -- start start 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.627+0000 7f6397200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f639010aa80 0x7f639011b220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.627+0000 7f6397200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6390111d20 0x7f639011b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.627+0000 7f6397200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f639011cd20 0x7f6390114500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f6395200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f639011cd20 0x7f6390114500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f6395200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f639011cd20 0x7f6390114500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43942/0 (socket says 172.21.15.28:43942) 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f638f400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6390111d20 0x7f639011b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f638f400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6390111d20 0x7f639011b760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53810/0 (socket says 172.21.15.28:53810) 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f6395200700 1 -- 172.21.15.28:0/972669895 learned_addr learned my addr 172.21.15.28:0/972669895 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6390122630 con 0x7f639010aa80 2024-05-20T20:10:21.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f63901224b0 con 0x7f639011cd20 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.630+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f63901227b0 con 0x7f6390111d20 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.631+0000 7f6395200700 1 -- 172.21.15.28:0/972669895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6390111d20 msgr2=0x7f639011b760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.631+0000 7f6395200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6390111d20 0x7f639011b760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.631+0000 7f6395200700 1 -- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f639010aa80 msgr2=0x7f639011b220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.631+0000 7f6395200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f639010aa80 0x7f639011b220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.631+0000 7f6395200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6390114e00 con 0x7f639011cd20 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.632+0000 7f638f400700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6390111d20 0x7f639011b760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.632+0000 7f6395200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f639011cd20 0x7f6390114500 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f6388007d40 tx=0x7f638800a340 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.633+0000 7f638cc00700 1 -- 172.21.15.28:0/972669895 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6388016020 con 0x7f639011cd20 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.633+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f63901150f0 con 0x7f639011cd20 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.633+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6390119060 con 0x7f639011cd20 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.634+0000 7f638cc00700 1 -- 172.21.15.28:0/972669895 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f638800ad40 con 0x7f639011cd20 2024-05-20T20:10:21.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.636+0000 7f638cc00700 1 -- 172.21.15.28:0/972669895 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6388016020 con 0x7f639011cd20 2024-05-20T20:10:21.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.642+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6370005330 con 0x7f639011cd20 2024-05-20T20:10:21.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.643+0000 7f638cc00700 1 -- 172.21.15.28:0/972669895 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f6388019020 con 0x7f639011cd20 2024-05-20T20:10:21.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.645+0000 7f638cc00700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63740b2ca0 0x7f63740b5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:21.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.646+0000 7f638cc00700 1 -- 172.21.15.28:0/972669895 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f63880e1e20 con 0x7f639011cd20 2024-05-20T20:10:21.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.646+0000 7f638fe00700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63740b2ca0 0x7f63740b5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:21.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.648+0000 7f638cc00700 1 -- 172.21.15.28:0/972669895 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f63880a98c0 con 0x7f639011cd20 2024-05-20T20:10:21.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:21.648+0000 7f638fe00700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63740b2ca0 0x7f63740b5160 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f6380005fd0 tx=0x7f6380005d00 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:21.786 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:22.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.390+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f63700058f0 con 0x7f639011cd20 2024-05-20T20:10:22.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.393+0000 7f638cc00700 1 -- 172.21.15.28:0/972669895 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f63880a91c0 con 0x7f639011cd20 2024-05-20T20:10:22.398 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:10:22.400 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:10:22.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.399+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63740b2ca0 msgr2=0x7f63740b5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:22.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.399+0000 7f6397200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63740b2ca0 0x7f63740b5160 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f6380005fd0 tx=0x7f6380005d00 comp rx=0 tx=0).stop 2024-05-20T20:10:22.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.399+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f639011cd20 msgr2=0x7f6390114500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:22.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.399+0000 7f6397200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f639011cd20 0x7f6390114500 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f6388007d40 tx=0x7f638800a340 comp rx=0 tx=0).stop 2024-05-20T20:10:22.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.400+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 shutdown_connections 2024-05-20T20:10:22.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.400+0000 7f6397200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f63740b2ca0 0x7f63740b5160 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:22.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.400+0000 7f6397200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f639011cd20 0x7f6390114500 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:22.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.400+0000 7f6397200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f639010aa80 0x7f639011b220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:22.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.400+0000 7f6397200700 1 --2- 172.21.15.28:0/972669895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6390111d20 0x7f639011b760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:22.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.400+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 >> 172.21.15.28:0/972669895 conn(0x7f639006e6b0 msgr2=0x7f63900720e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:22.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.402+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 shutdown_connections 2024-05-20T20:10:22.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:22.402+0000 7f6397200700 1 -- 172.21.15.28:0/972669895 wait complete. 2024-05-20T20:10:22.411 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:10:23.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:22 smithi028 ceph-mon[31178]: pgmap v145: 97 pgs: 97 active+clean; 20 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 223 MiB/s wr, 1.81k op/s 2024-05-20T20:10:23.307 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:22 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/972669895' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:23.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:22 smithi154 ceph-mon[36314]: pgmap v145: 97 pgs: 97 active+clean; 20 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 223 MiB/s wr, 1.81k op/s 2024-05-20T20:10:23.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:22 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/972669895' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:23.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:22 smithi033 ceph-mon[36179]: pgmap v145: 97 pgs: 97 active+clean; 20 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 223 MiB/s wr, 1.81k op/s 2024-05-20T20:10:23.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:22 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/972669895' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:24.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.959+0000 7f02ca800640 1 -- 172.21.15.28:0/4128801282 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a56f0 msgr2=0x7f02bc0b7520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:24.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.959+0000 7f02ca800640 1 --2- 172.21.15.28:0/4128801282 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a56f0 0x7f02bc0b7520 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f02b0009900 tx=0x7f02b002a4a0 comp rx=0 tx=0).stop 2024-05-20T20:10:24.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.974+0000 7f02ca800640 1 -- 172.21.15.28:0/4128801282 shutdown_connections 2024-05-20T20:10:24.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.974+0000 7f02ca800640 1 --2- 172.21.15.28:0/4128801282 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02bc0b7a60 0x7f02bc0b9e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:24.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.974+0000 7f02ca800640 1 --2- 172.21.15.28:0/4128801282 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a56f0 0x7f02bc0b7520 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:24.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.974+0000 7f02ca800640 1 --2- 172.21.15.28:0/4128801282 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02bc0a4dd0 0x7f02bc0a51b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:24.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.974+0000 7f02ca800640 1 -- 172.21.15.28:0/4128801282 >> 172.21.15.28:0/4128801282 conn(0x7f02bc01a630 msgr2=0x7f02bc01aa40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:24.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.974+0000 7f02ca800640 1 -- 172.21.15.28:0/4128801282 shutdown_connections 2024-05-20T20:10:24.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.974+0000 7f02ca800640 1 -- 172.21.15.28:0/4128801282 wait complete. 2024-05-20T20:10:24.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.976+0000 7f02ca800640 1 Processor -- start 2024-05-20T20:10:24.982 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.979+0000 7f02ca800640 1 -- start start 2024-05-20T20:10:24.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.984+0000 7f02ca800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a4dd0 0x7f02bc0b5a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:24.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.984+0000 7f02ca800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02bc0a56f0 0x7f02bc0b5fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:24.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.984+0000 7f02ca800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02bc0b7a60 0x7f02bc0aecf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:24.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.984+0000 7f02ca800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc0bd610 con 0x7f02bc0b7a60 2024-05-20T20:10:24.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.984+0000 7f02ca800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc0bd490 con 0x7f02bc0a56f0 2024-05-20T20:10:24.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.984+0000 7f02ca800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc0bd790 con 0x7f02bc0a4dd0 2024-05-20T20:10:24.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.987+0000 7f02c3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a4dd0 0x7f02bc0b5a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:24.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.987+0000 7f02c3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a4dd0 0x7f02bc0b5a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53828/0 (socket says 172.21.15.28:53828) 2024-05-20T20:10:24.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.987+0000 7f02c3400640 1 -- 172.21.15.28:0/3598407334 learned_addr learned my addr 172.21.15.28:0/3598407334 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:24.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.988+0000 7f02c2a00640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02bc0a56f0 0x7f02bc0b5fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:24.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.988+0000 7f02c3e00640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02bc0b7a60 0x7f02bc0aecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:24.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.989+0000 7f02c3400640 1 -- 172.21.15.28:0/3598407334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02bc0a56f0 msgr2=0x7f02bc0b5fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:24.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.989+0000 7f02c3400640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02bc0a56f0 0x7f02bc0b5fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:24.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.989+0000 7f02c3400640 1 -- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02bc0b7a60 msgr2=0x7f02bc0aecf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:24.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.989+0000 7f02c3400640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02bc0b7a60 0x7f02bc0aecf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:24.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.989+0000 7f02c3400640 1 -- 172.21.15.28:0/3598407334 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f02bc0af5b0 con 0x7f02bc0a4dd0 2024-05-20T20:10:24.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:24.991+0000 7f02c2a00640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02bc0a56f0 0x7f02bc0b5fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:10:25.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.000+0000 7f02c3400640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a4dd0 0x7f02bc0b5a90 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f02b800d950 tx=0x7f02b800de10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:25.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.003+0000 7f02b7e00640 1 -- 172.21.15.28:0/3598407334 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b8017020 con 0x7f02bc0a4dd0 2024-05-20T20:10:25.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.004+0000 7f02b7e00640 1 -- 172.21.15.28:0/3598407334 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f02b8009c30 con 0x7f02bc0a4dd0 2024-05-20T20:10:25.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.004+0000 7f02b7e00640 1 -- 172.21.15.28:0/3598407334 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b800f690 con 0x7f02bc0a4dd0 2024-05-20T20:10:25.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.008+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f02bc0af8a0 con 0x7f02bc0a4dd0 2024-05-20T20:10:25.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.008+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f02bc0b3850 con 0x7f02bc0a4dd0 2024-05-20T20:10:25.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.020+0000 7f02b7e00640 1 -- 172.21.15.28:0/3598407334 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f02b801a020 con 0x7f02bc0a4dd0 2024-05-20T20:10:25.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.026+0000 7f02b7e00640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02ac0b29e0 0x7f02ac0b4ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.028+0000 7f02b7e00640 1 -- 172.21.15.28:0/3598407334 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f02b80a5440 con 0x7f02bc0a4dd0 2024-05-20T20:10:25.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.028+0000 7f02cd5d6740 1 Processor -- start 2024-05-20T20:10:25.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.029+0000 7f02c2a00640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02ac0b29e0 0x7f02ac0b4ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.032+0000 7f02c2a00640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02ac0b29e0 0x7f02ac0b4ea0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f02b0000c00 tx=0x7f02b0005df0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:25.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.033+0000 7f02cd5d6740 1 -- start start 2024-05-20T20:10:25.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197698250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a1976a2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a5440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a197608b90 con 0x55a197697e70 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a1975f8730 con 0x55a197698790 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a1975fc770 con 0x55a1976a3050 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02c2a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a1976a2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02c2a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a1976a2b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43980/0 (socket says 172.21.15.28:43980) 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.034+0000 7f02c2a00640 1 -- 172.21.15.28:0/2409335648 learned_addr learned my addr 172.21.15.28:0/2409335648 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:25.037 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.036+0000 7f02c2a00640 1 -- 172.21.15.28:0/2409335648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 msgr2=0x55a1976a5440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.038 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.036+0000 7f02c3e00640 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a5440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.038+0000 7f02c2a00640 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a5440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.038+0000 7f02c2a00640 1 -- 172.21.15.28:0/2409335648 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 msgr2=0x55a197698250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.038+0000 7f02c2a00640 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197698250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.038+0000 7f02c2a00640 1 -- 172.21.15.28:0/2409335648 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a1975918b0 con 0x55a197698790 2024-05-20T20:10:25.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.038+0000 7f02c3e00640 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a5440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:25.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.041+0000 7f02c2a00640 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a1976a2b10 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f02b0009900 tx=0x7f02b003c000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3d50c4170db1777d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:25.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.041+0000 7f02b4c00640 1 -- 172.21.15.28:0/2409335648 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b004c070 con 0x55a197698790 2024-05-20T20:10:25.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.042+0000 7f02b4c00640 1 -- 172.21.15.28:0/2409335648 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f02b00035f0 con 0x55a197698790 2024-05-20T20:10:25.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.043+0000 7f02b4c00640 1 -- 172.21.15.28:0/2409335648 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b004c070 con 0x55a197698790 2024-05-20T20:10:25.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.045+0000 7f02cd5d6740 1 -- 172.21.15.28:0/2409335648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 msgr2=0x55a1976a2b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:25.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.045+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a1976a2b10 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f02b0009900 tx=0x7f02b003c000 comp rx=0 tx=0).stop 2024-05-20T20:10:25.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.053+0000 7f02cd5d6740 1 -- 172.21.15.28:0/2409335648 shutdown_connections 2024-05-20T20:10:25.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.053+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a1976a2b10 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.053+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a5440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.054 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.053+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/2409335648 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197698250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.054 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.053+0000 7f02cd5d6740 1 -- 172.21.15.28:0/2409335648 >> 172.21.15.28:0/2409335648 conn(0x55a197694380 msgr2=0x55a197694760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:25.054 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.053+0000 7f02cd5d6740 1 -- 172.21.15.28:0/2409335648 shutdown_connections 2024-05-20T20:10:25.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.053+0000 7f02cd5d6740 1 -- 172.21.15.28:0/2409335648 wait complete. 2024-05-20T20:10:25.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.054+0000 7f02cd5d6740 1 -- start start 2024-05-20T20:10:25.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.054+0000 7f02cd5d6740 1 Processor -- start 2024-05-20T20:10:25.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.060+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197697e70 0x55a1976a2990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.060+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197698790 0x55a19769fc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.060+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976a01c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.061+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a1972de790 con 0x55a197698790 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.061+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a1975a0bc0 con 0x55a1976a3050 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.061+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a1976a5c20 con 0x55a197697e70 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.061+0000 7f02c3e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976a01c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.061+0000 7f02c3e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976a01c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43992/0 (socket says 172.21.15.28:43992) 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.061+0000 7f02c3e00640 1 -- 172.21.15.28:0/630891844 learned_addr learned my addr 172.21.15.28:0/630891844 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.061+0000 7f02cd5d6740 10 client.0 _ll_register_callbacks cb 0x55a19757e400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.062+0000 7f02c2a00640 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197698790 0x55a19769fc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.062+0000 7f02c3400640 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197697e70 0x55a1976a2990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.062+0000 7f02c3400640 1 -- 172.21.15.28:0/630891844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 msgr2=0x55a1976a01c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.062+0000 7f02c3400640 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976a01c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.062+0000 7f02c3400640 1 -- 172.21.15.28:0/630891844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197698790 msgr2=0x55a19769fc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.062+0000 7f02c3400640 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197698790 0x55a19769fc80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.062+0000 7f02c3400640 1 -- 172.21.15.28:0/630891844 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a19730d210 con 0x55a197697e70 2024-05-20T20:10:25.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.066+0000 7f02a9600640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:10:25.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.066+0000 7f02c3400640 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197697e70 0x55a1976a2990 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f02b80df230 tx=0x7f02b80a3500 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bb82871efee2b97d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:25.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.067+0000 7f02a9600640 1 -- 172.21.15.28:0/630891844 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b80a5640 con 0x55a197697e70 2024-05-20T20:10:25.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.067+0000 7f02a9600640 1 -- 172.21.15.28:0/630891844 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f02b80a50e0 con 0x55a197697e70 2024-05-20T20:10:25.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.074+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f02b00035f0 con 0x55a197697e70 2024-05-20T20:10:25.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.074+0000 7f02a9600640 1 -- 172.21.15.28:0/630891844 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f02b80a5640 con 0x55a197697e70 2024-05-20T20:10:25.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.074+0000 7f02cd5d6740 10 client.26365 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:25.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.074+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a1973d9620 con 0x55a197697e70 2024-05-20T20:10:25.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02a9600640 1 -- 172.21.15.28:0/630891844 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f02b00035f0 con 0x55a197697e70 2024-05-20T20:10:25.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02a9600640 1 -- 172.21.15.28:0/630891844 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f02b80a87a0 con 0x55a197697e70 2024-05-20T20:10:25.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02cd5d6740 10 client.26365 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:25.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02cd5d6740 10 client.26365 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:25.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02cd5d6740 20 client.26365 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:25.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a1976b29a0 0x55a1976b4d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02cd5d6740 4 client.26365 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a19760aae0 con 0x55a1976b29a0 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.075+0000 7f02c2a00640 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a1976b29a0 0x55a1976b4d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.077+0000 7f02c2a00640 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a1976b29a0 0x55a1976b4d90 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=90ebe1ba98b5164 server_cookie=5e30becc2bbb33d4 in_seq=0 out_seq=0 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.077+0000 7f02a9600640 10 client.26365 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.078+0000 7f02a9600640 1 -- 172.21.15.28:0/630891844 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a19760aae0 con 0x55a1976b29a0 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.078+0000 7f02a9600640 10 client.26365 handle_command_reply: tid=0 2024-05-20T20:10:25.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.078+0000 7f02cd5d6740 1 client.26365 shutdown 2024-05-20T20:10:25.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197697e70 msgr2=0x55a1976a2990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:25.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197697e70 0x55a1976a2990 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f02b80df230 tx=0x7f02b80a3500 comp rx=0 tx=0).stop 2024-05-20T20:10:25.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 shutdown_connections 2024-05-20T20:10:25.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a1976b29a0 0x55a1976b4d90 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976a01c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197698790 0x55a19769fc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/630891844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197697e70 0x55a1976a2990 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 >> 172.21.15.28:0/630891844 conn(0x55a197694380 msgr2=0x55a19769a100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:25.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02a9600640 0 client.26365 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:25.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 shutdown_connections 2024-05-20T20:10:25.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.081+0000 7f02cd5d6740 1 -- 172.21.15.28:0/630891844 wait complete. 2024-05-20T20:10:25.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.086+0000 7f02cd5d6740 20 client.26365 trim_cache size 0 max 16384 2024-05-20T20:10:25.136 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.134+0000 7f02cd5d6740 1 Processor -- start 2024-05-20T20:10:25.136 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.134+0000 7f02cd5d6740 1 -- start start 2024-05-20T20:10:25.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.135+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a1976aec80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.136+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197698790 0x55a1976ba610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.137+0000 7f02c2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197698790 0x55a1976ba610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.137+0000 7f02c2a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197698790 0x55a1976ba610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53862/0 (socket says 172.21.15.28:53862) 2024-05-20T20:10:25.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.137+0000 7f02c2a00640 1 -- 172.21.15.28:0/1169579718 learned_addr learned my addr 172.21.15.28:0/1169579718 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:25.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.137+0000 7f02c3400640 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a1976aec80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.137+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976bab50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.138+0000 7f02c3e00640 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976bab50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.138+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc1617e0 con 0x55a197697e70 2024-05-20T20:10:25.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.138+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc163320 con 0x55a1976a3050 2024-05-20T20:10:25.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.138+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc161eb0 con 0x55a197698790 2024-05-20T20:10:25.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.139+0000 7f02c2a00640 1 -- 172.21.15.28:0/1169579718 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 msgr2=0x55a1976bab50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.139+0000 7f02c2a00640 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976bab50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.139+0000 7f02c2a00640 1 -- 172.21.15.28:0/1169579718 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 msgr2=0x55a1976aec80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.139+0000 7f02c2a00640 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a1976aec80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.139+0000 7f02c2a00640 1 -- 172.21.15.28:0/1169579718 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a1976985c0 con 0x55a197698790 2024-05-20T20:10:25.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.139+0000 7f02c3400640 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a1976aec80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:25.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.139+0000 7f02c2a00640 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197698790 0x55a1976ba610 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f02b0003f60 tx=0x7f02b003f000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fe81b89072f52277 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:25.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.140+0000 7f02abe00640 1 -- 172.21.15.28:0/1169579718 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b004c030 con 0x55a197698790 2024-05-20T20:10:25.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.140+0000 7f02abe00640 1 -- 172.21.15.28:0/1169579718 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55a19760aae0 con 0x55a197698790 2024-05-20T20:10:25.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.140+0000 7f02abe00640 1 -- 172.21.15.28:0/1169579718 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b0035050 con 0x55a197698790 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197698790 msgr2=0x55a1976ba610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197698790 0x55a1976ba610 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f02b0003f60 tx=0x7f02b003f000 comp rx=0 tx=0).stop 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 shutdown_connections 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a1976a3050 0x55a1976bab50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a197698790 0x55a1976ba610 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/1169579718 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a1976aec80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 >> 172.21.15.28:0/1169579718 conn(0x55a197694380 msgr2=0x55a1976933b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:25.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 shutdown_connections 2024-05-20T20:10:25.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.141+0000 7f02cd5d6740 1 -- 172.21.15.28:0/1169579718 wait complete. 2024-05-20T20:10:25.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.142+0000 7f02cd5d6740 1 -- start start 2024-05-20T20:10:25.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.142+0000 7f02cd5d6740 1 Processor -- start 2024-05-20T20:10:25.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197715860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a197715da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02cd5d6740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a8400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc162c50 con 0x55a197697e70 2024-05-20T20:10:25.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc162580 con 0x55a197698790 2024-05-20T20:10:25.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02cd5d6740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f02bc15f7c0 con 0x55a1976a3050 2024-05-20T20:10:25.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02cd5d6740 10 client.0 _ll_register_callbacks cb 0x7f02b0009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:25.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02c3400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197715860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.143+0000 7f02c2a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a197715da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197715860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48826/0 (socket says 172.21.15.28:48826) 2024-05-20T20:10:25.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c2a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a197715da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44016/0 (socket says 172.21.15.28:44016) 2024-05-20T20:10:25.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3400640 1 -- 172.21.15.28:0/827533119 learned_addr learned my addr 172.21.15.28:0/827533119 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:25.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3e00640 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a8400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3400640 1 -- 172.21.15.28:0/827533119 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 msgr2=0x55a1976a8400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3400640 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a8400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3400640 1 -- 172.21.15.28:0/827533119 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 msgr2=0x55a197715da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3400640 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a197715da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3400640 1 -- 172.21.15.28:0/827533119 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a1976a8af0 con 0x55a197697e70 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.144+0000 7f02c3e00640 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a8400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.145+0000 7f02c3400640 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197715860 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f02b80a5cf0 tx=0x7f02b80a8480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1191bcfda7d6ad73 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.145+0000 7f02a9600640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.145+0000 7f02a9600640 1 -- 172.21.15.28:0/827533119 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f02b80b7060 con 0x55a197697e70 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.145+0000 7f02a9600640 1 -- 172.21.15.28:0/827533119 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f02b80a4b40 con 0x55a197697e70 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.145+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55a19760aae0 con 0x55a197697e70 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.146+0000 7f02a9600640 1 -- 172.21.15.28:0/827533119 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f02b80a57e0 con 0x55a197697e70 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.146+0000 7f02cd5d6740 10 client.16230 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:25.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.146+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a19771d010 con 0x55a197697e70 2024-05-20T20:10:25.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.146+0000 7f02a9600640 1 -- 172.21.15.28:0/827533119 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55a19760aae0 con 0x55a197697e70 2024-05-20T20:10:25.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.147+0000 7f02a9600640 1 -- 172.21.15.28:0/827533119 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f02b80a4f50 con 0x55a197697e70 2024-05-20T20:10:25.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.147+0000 7f02cd5d6740 10 client.16230 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:25.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.147+0000 7f02cd5d6740 10 client.16230 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:25.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.148+0000 7f02cd5d6740 20 client.16230 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:25.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.148+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a19771d6e0 0x55a19771fb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:25.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.148+0000 7f02cd5d6740 4 client.16230 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:10:25.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.148+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a1976042c0 con 0x55a19771d6e0 2024-05-20T20:10:25.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.148+0000 7f02c2a00640 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a19771d6e0 0x55a19771fb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:25.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.149+0000 7f02c2a00640 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a19771d6e0 0x55a19771fb00 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1782bebab3652cd3 server_cookie=f44db3006112cc60 in_seq=0 out_seq=0 2024-05-20T20:10:25.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.149+0000 7f02a9600640 10 client.16230 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.188+0000 7f02a9600640 1 -- 172.21.15.28:0/827533119 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55a1976042c0 con 0x55a19771d6e0 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.188+0000 7f02a9600640 10 client.16230 handle_command_reply: tid=0 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.188+0000 7f02cd5d6740 1 client.16230 shutdown 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 msgr2=0x55a197715860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197715860 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f02b80a5cf0 tx=0x7f02b80a8480 comp rx=0 tx=0).stop 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 shutdown_connections 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a19771d6e0 0x55a19771fb00 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a197698790 0x55a197715da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a1976a3050 0x55a1976a8400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 --2- 172.21.15.28:0/827533119 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a197697e70 0x55a197715860 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02a9600640 0 client.16230 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:25.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 >> 172.21.15.28:0/827533119 conn(0x55a197694380 msgr2=0x55a1976a14b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:25.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.189+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 shutdown_connections 2024-05-20T20:10:25.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.190+0000 7f02cd5d6740 1 -- 172.21.15.28:0/827533119 wait complete. 2024-05-20T20:10:25.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.190+0000 7f02cd5d6740 20 client.16230 trim_cache size 0 max 16384 2024-05-20T20:10:25.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:10:25.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:10:25.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:10:25.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02ac0b29e0 msgr2=0x7f02ac0b4ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:25.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02ac0b29e0 0x7f02ac0b4ea0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f02b0000c00 tx=0x7f02b0005df0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a4dd0 msgr2=0x7f02bc0b5a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a4dd0 0x7f02bc0b5a90 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f02b800d950 tx=0x7f02b800de10 comp rx=0 tx=0).stop 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 shutdown_connections 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f02bc0a56f0 0x7f02bc0b5fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02ac0b29e0 0x7f02ac0b4ea0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f02bc0b7a60 0x7f02bc0aecf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 --2- 172.21.15.28:0/3598407334 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f02bc0a4dd0 0x7f02bc0b5a90 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.194+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 >> 172.21.15.28:0/3598407334 conn(0x7f02bc01a630 msgr2=0x7f02bc0a1d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.195+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 shutdown_connections 2024-05-20T20:10:25.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:25.195+0000 7f02ca800640 1 -- 172.21.15.28:0/3598407334 wait complete. 2024-05-20T20:10:25.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:24 smithi028 ceph-mon[31178]: pgmap v146: 97 pgs: 97 active+clean; 20 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 292 MiB/s wr, 2.33k op/s 2024-05-20T20:10:25.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:24 smithi154 ceph-mon[36314]: pgmap v146: 97 pgs: 97 active+clean; 20 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 292 MiB/s wr, 2.33k op/s 2024-05-20T20:10:25.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:24 smithi033 ceph-mon[36179]: pgmap v146: 97 pgs: 97 active+clean; 20 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 292 MiB/s wr, 2.33k op/s 2024-05-20T20:10:26.780 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:26 smithi028 ceph-mon[31178]: pgmap v147: 97 pgs: 97 active+clean; 21 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 234 MiB/s wr, 1.86k op/s 2024-05-20T20:10:26.784 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 144.7 --await-for 45.0' 2024-05-20T20:10:26.785 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 144.7 --await-for 45.0 2024-05-20T20:10:26.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:26 smithi154 ceph-mon[36314]: pgmap v147: 97 pgs: 97 active+clean; 21 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 234 MiB/s wr, 1.86k op/s 2024-05-20T20:10:26.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:26 smithi033 ceph-mon[36179]: pgmap v147: 97 pgs: 97 active+clean; 21 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 234 MiB/s wr, 1.86k op/s 2024-05-20T20:10:28.375 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:29.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:28 smithi028 ceph-mon[31178]: pgmap v148: 97 pgs: 97 active+clean; 21 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 212 MiB/s wr, 1.67k op/s 2024-05-20T20:10:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:28 smithi154 ceph-mon[36314]: pgmap v148: 97 pgs: 97 active+clean; 21 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 212 MiB/s wr, 1.67k op/s 2024-05-20T20:10:29.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:28 smithi033 ceph-mon[36179]: pgmap v148: 97 pgs: 97 active+clean; 21 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 212 MiB/s wr, 1.67k op/s 2024-05-20T20:10:30.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 -- 172.21.15.28:0/1706087231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 msgr2=0x7f86841103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 --2- 172.21.15.28:0/1706087231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 0x7f86841103d0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f867c00b0a0 tx=0x7f867c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 -- 172.21.15.28:0/1706087231 shutdown_connections 2024-05-20T20:10:30.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 --2- 172.21.15.28:0/1706087231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 0x7f86841103d0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 --2- 172.21.15.28:0/1706087231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f86841095b0 0x7f8684109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 --2- 172.21.15.28:0/1706087231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f868406b220 0x7f868406b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 -- 172.21.15.28:0/1706087231 >> 172.21.15.28:0/1706087231 conn(0x7f86840793f0 msgr2=0x7f8684079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 -- 172.21.15.28:0/1706087231 shutdown_connections 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.728+0000 7f868b800700 1 -- 172.21.15.28:0/1706087231 wait complete. 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.730+0000 7f868b800700 1 Processor -- start 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.730+0000 7f868b800700 1 -- start start 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f868b800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f868406b220 0x7f8684102510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f868b800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 0x7f8684102a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f868b800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f86841095b0 0x7f8684102f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f868b800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8684113730 con 0x7f86841095b0 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f868b800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f86841135b0 con 0x7f868406bbe0 2024-05-20T20:10:30.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f868b800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f86841138b0 con 0x7f868406b220 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f868406b220 0x7f8684102510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f868406b220 0x7f8684102510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43868/0 (socket says 172.21.15.28:43868) 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 -- 172.21.15.28:0/776049994 learned_addr learned my addr 172.21.15.28:0/776049994 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8689800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f86841095b0 0x7f8684102f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8683e00700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 0x7f8684102a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 -- 172.21.15.28:0/776049994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 msgr2=0x7f8684102a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 0x7f8684102a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 -- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f86841095b0 msgr2=0x7f8684102f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f86841095b0 0x7f8684102f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8688e00700 1 -- 172.21.15.28:0/776049994 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f86841a81b0 con 0x7f868406b220 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.731+0000 7f8689800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f86841095b0 0x7f8684102f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.732+0000 7f8688e00700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f868406b220 0x7f8684102510 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f867800e9c0 tx=0x7f867800ed80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:30.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.732+0000 7f8681600700 1 -- 172.21.15.28:0/776049994 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f867800cbd0 con 0x7f868406b220 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.732+0000 7f8681600700 1 -- 172.21.15.28:0/776049994 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f867800cd70 con 0x7f868406b220 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.732+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f86841a84a0 con 0x7f868406b220 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.732+0000 7f8681600700 1 -- 172.21.15.28:0/776049994 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8678010430 con 0x7f868406b220 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.732+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f86841a89e0 con 0x7f868406b220 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 Processor -- start 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 -- start start 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d8f310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164d892e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d89d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558164ce7ec0 con 0x558164d8f850 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558164cd7a60 con 0x558164d89900 2024-05-20T20:10:30.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.737+0000 7f86960beb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558164cdbaa0 con 0x558164d8ef30 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.738+0000 7f8689800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d8f310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.738+0000 7f8689800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d8f310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43872/0 (socket says 172.21.15.28:43872) 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.738+0000 7f8689800700 1 -- 172.21.15.28:0/3447694921 learned_addr learned my addr 172.21.15.28:0/3447694921 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.738+0000 7f8681600700 1 -- 172.21.15.28:0/776049994 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f8678004750 con 0x7f868406b220 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.738+0000 7f8689800700 1 -- 172.21.15.28:0/3447694921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 msgr2=0x558164d89d60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.738+0000 7f8689800700 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d89d60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.738+0000 7f8689800700 1 -- 172.21.15.28:0/3447694921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 msgr2=0x558164d892e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.739+0000 7f8688e00700 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164d892e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.739+0000 7f8689800700 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164d892e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.739+0000 7f8689800700 1 -- 172.21.15.28:0/3447694921 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558164ca04a0 con 0x558164d8ef30 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.739+0000 7f8688e00700 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164d892e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.739+0000 7f8689800700 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d8f310 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f867c00b0a0 tx=0x7f867c009360 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c9a9829e08d9fb04 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:30.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.740+0000 7f8676000700 1 -- 172.21.15.28:0/3447694921 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f867c00bcb0 con 0x558164d8ef30 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.740+0000 7f8676000700 1 -- 172.21.15.28:0/3447694921 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f867c0074e0 con 0x558164d8ef30 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.740+0000 7f8676000700 1 -- 172.21.15.28:0/3447694921 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f867c0052a0 con 0x558164d8ef30 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.740+0000 7f8681600700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f866c0b2af0 0x7f866c0b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.740+0000 7f8681600700 1 -- 172.21.15.28:0/776049994 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f86780e2960 con 0x7f868406b220 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f86960beb80 1 -- 172.21.15.28:0/3447694921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 msgr2=0x558164d8f310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f86960beb80 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d8f310 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f867c00b0a0 tx=0x7f867c009360 comp rx=0 tx=0).stop 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f8688e00700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f866c0b2af0 0x7f866c0b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f86960beb80 1 -- 172.21.15.28:0/3447694921 shutdown_connections 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f86960beb80 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d89d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f86960beb80 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d8f310 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f86960beb80 1 --2- 172.21.15.28:0/3447694921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164d892e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.741+0000 7f86960beb80 1 -- 172.21.15.28:0/3447694921 >> 172.21.15.28:0/3447694921 conn(0x558164cb7c60 msgr2=0x558164d84090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.742+0000 7f86960beb80 1 -- 172.21.15.28:0/3447694921 shutdown_connections 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.743+0000 7f86960beb80 1 -- 172.21.15.28:0/3447694921 wait complete. 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.743+0000 7f86960beb80 1 -- start start 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.744+0000 7f8688e00700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f866c0b2af0 0x7f866c0b4fb0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f86780a6e60 tx=0x7f8678005020 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.744+0000 7f86960beb80 1 Processor -- start 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f86960beb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d955d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f86960beb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8ef30 0x558164d92970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f86960beb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8f850 0x558164d92eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f86960beb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558164bde020 con 0x558164d8ef30 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f86960beb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558164c787e0 con 0x558164d89900 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f86960beb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558164d9ea70 con 0x558164d8f850 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f86960beb80 10 client.0 _ll_register_callbacks cb 0x558164c81cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:30.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d955d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d955d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42342/0 (socket says 172.21.15.28:42342) 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 -- 172.21.15.28:0/2793288010 learned_addr learned my addr 172.21.15.28:0/2793288010 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8f850 0x558164d92eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 -- 172.21.15.28:0/2793288010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8f850 msgr2=0x558164d92eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8f850 0x558164d92eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 -- 172.21.15.28:0/2793288010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8ef30 msgr2=0x558164d92970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8ef30 0x558164d92970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.745+0000 7f8689800700 1 -- 172.21.15.28:0/2793288010 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558164c27950 con 0x558164d89900 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.746+0000 7f8689800700 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d955d0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f867c00b070 tx=0x7f867c007db0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8eace9b139937e0e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.746+0000 7f866aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.746+0000 7f866aa00700 1 -- 172.21.15.28:0/2793288010 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f867c007690 con 0x558164d89900 2024-05-20T20:10:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.746+0000 7f866aa00700 1 -- 172.21.15.28:0/2793288010 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f867c02d040 con 0x558164d89900 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.747+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f867c0074e0 con 0x558164d89900 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.747+0000 7f866aa00700 1 -- 172.21.15.28:0/2793288010 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f867c007830 con 0x558164d89900 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 10 client.26465 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558164ca80e0 con 0x558164d89900 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f866aa00700 1 -- 172.21.15.28:0/2793288010 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f867c0074e0 con 0x558164d89900 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f866aa00700 1 -- 172.21.15.28:0/2793288010 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f867c011070 con 0x558164d89900 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 10 client.26465 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 10 client.26465 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 20 client.26465 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164dab410 0x558164dad800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 4 client.26465 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.748+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558164ce9e10 con 0x558164dab410 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.749+0000 7f8683e00700 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164dab410 0x558164dad800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.749+0000 7f866aa00700 10 client.26465 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.751+0000 7f8683e00700 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164dab410 0x558164dad800 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d568033a69bb49cd server_cookie=78bc5e224aa0a190 in_seq=0 out_seq=0 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.751+0000 7f866aa00700 1 -- 172.21.15.28:0/2793288010 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558164ce9e10 con 0x558164dab410 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.751+0000 7f866aa00700 10 client.26465 handle_command_reply: tid=0 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 client.26465 shutdown 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 msgr2=0x558164d955d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d955d0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f867c00b070 tx=0x7f867c007db0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 shutdown_connections 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164dab410 0x558164dad800 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8f850 0x558164d92eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d89900 0x558164d955d0 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 --2- 172.21.15.28:0/2793288010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8ef30 0x558164d92970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 >> 172.21.15.28:0/2793288010 conn(0x558164cb7c60 msgr2=0x558164d882d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:30.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.752+0000 7f866aa00700 0 client.26465 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:30.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.753+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 shutdown_connections 2024-05-20T20:10:30.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.753+0000 7f86960beb80 1 -- 172.21.15.28:0/2793288010 wait complete. 2024-05-20T20:10:30.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.753+0000 7f86960beb80 20 client.26465 trim_cache size 0 max 16384 2024-05-20T20:10:30.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.781+0000 7f86960beb80 1 Processor -- start 2024-05-20T20:10:30.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f86960beb80 1 -- start start 2024-05-20T20:10:30.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f86960beb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d89900 0x558164d892e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f86960beb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d90340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f86960beb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8f850 0x558164d91260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f86960beb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f86841abbb0 con 0x558164d89900 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f86960beb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f86841ad6f0 con 0x558164d8f850 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f86960beb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f86841ac280 con 0x558164d8ef30 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.782+0000 7f8683e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d90340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.783+0000 7f8683e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d90340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43882/0 (socket says 172.21.15.28:43882) 2024-05-20T20:10:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.783+0000 7f8683e00700 1 -- 172.21.15.28:0/2579339787 learned_addr learned my addr 172.21.15.28:0/2579339787 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:30.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.783+0000 7f8689800700 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d89900 0x558164d892e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.783+0000 7f8689800700 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8f850 0x558164d91260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.783+0000 7f8683e00700 1 -- 172.21.15.28:0/2579339787 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8f850 msgr2=0x558164d91260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:30.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.784+0000 7f8683e00700 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8f850 0x558164d91260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.784+0000 7f8683e00700 1 -- 172.21.15.28:0/2579339787 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d89900 msgr2=0x558164d892e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:30.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.784+0000 7f8683e00700 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d89900 0x558164d892e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.784+0000 7f8683e00700 1 -- 172.21.15.28:0/2579339787 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558164d7c210 con 0x558164d8ef30 2024-05-20T20:10:30.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.784+0000 7f8689800700 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8f850 0x558164d91260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:30.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.785+0000 7f8683e00700 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d90340 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x558164d93f40 tx=0x7f8670009840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c03dcaeeb0b80a1f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:30.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.785+0000 7f8675600700 1 -- 172.21.15.28:0/2579339787 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8670012dc0 con 0x558164d8ef30 2024-05-20T20:10:30.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.785+0000 7f8675600700 1 -- 172.21.15.28:0/2579339787 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f867001a860 con 0x558164d8ef30 2024-05-20T20:10:30.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.785+0000 7f8675600700 1 -- 172.21.15.28:0/2579339787 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8670010970 con 0x558164d8ef30 2024-05-20T20:10:30.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.786+0000 7f86960beb80 1 -- 172.21.15.28:0/2579339787 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 msgr2=0x558164d90340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.786+0000 7f86960beb80 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d90340 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x558164d93f40 tx=0x7f8670009840 comp rx=0 tx=0).stop 2024-05-20T20:10:30.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.787+0000 7f86960beb80 1 -- 172.21.15.28:0/2579339787 shutdown_connections 2024-05-20T20:10:30.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.787+0000 7f86960beb80 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8f850 0x558164d91260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.787+0000 7f86960beb80 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d89900 0x558164d892e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.787+0000 7f86960beb80 1 --2- 172.21.15.28:0/2579339787 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d8ef30 0x558164d90340 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.787+0000 7f86960beb80 1 -- 172.21.15.28:0/2579339787 >> 172.21.15.28:0/2579339787 conn(0x558164cb7c60 msgr2=0x558164d8fc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:30.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.787+0000 7f86960beb80 1 -- 172.21.15.28:0/2579339787 shutdown_connections 2024-05-20T20:10:30.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.787+0000 7f86960beb80 1 -- 172.21.15.28:0/2579339787 wait complete. 2024-05-20T20:10:30.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.788+0000 7f86960beb80 1 -- start start 2024-05-20T20:10:30.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.788+0000 7f86960beb80 1 Processor -- start 2024-05-20T20:10:30.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.789+0000 7f86960beb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d89900 0x558164db0ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.789+0000 7f86960beb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8ef30 0x558164daf290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.789+0000 7f86960beb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164daf7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.789+0000 7f86960beb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f86841ad020 con 0x558164d8f850 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.789+0000 7f86960beb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f86841ac950 con 0x558164d8ef30 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.789+0000 7f86960beb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f868419da90 con 0x558164d89900 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.789+0000 7f86960beb80 10 client.0 _ll_register_callbacks cb 0x558164c81cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.790+0000 7f8683e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8ef30 0x558164daf290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.790+0000 7f8683e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8ef30 0x558164daf290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42362/0 (socket says 172.21.15.28:42362) 2024-05-20T20:10:30.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.790+0000 7f8683e00700 1 -- 172.21.15.28:0/3831501860 learned_addr learned my addr 172.21.15.28:0/3831501860 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:30.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.790+0000 7f8689800700 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d89900 0x558164db0ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.791+0000 7f8683e00700 1 -- 172.21.15.28:0/3831501860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d89900 msgr2=0x558164db0ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:30.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.791+0000 7f8683e00700 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d89900 0x558164db0ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.791+0000 7f8683e00700 1 -- 172.21.15.28:0/3831501860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 msgr2=0x558164daf7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:30.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.791+0000 7f8683e00700 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164daf7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.791+0000 7f8683e00700 1 -- 172.21.15.28:0/3831501860 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558164db0010 con 0x558164d8ef30 2024-05-20T20:10:30.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.792+0000 7f8683e00700 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8ef30 0x558164daf290 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f8670004790 tx=0x7f8670009a60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c4c4eefe273c69bf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:30.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.792+0000 7f866aa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:30.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.793+0000 7f866aa00700 1 -- 172.21.15.28:0/3831501860 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8670012dc0 con 0x558164d8ef30 2024-05-20T20:10:30.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.793+0000 7f866aa00700 1 -- 172.21.15.28:0/3831501860 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f867001a3b0 con 0x558164d8ef30 2024-05-20T20:10:30.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.793+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f867001a860 con 0x558164d8ef30 2024-05-20T20:10:30.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.793+0000 7f866aa00700 1 -- 172.21.15.28:0/3831501860 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f867001a5f0 con 0x558164d8ef30 2024-05-20T20:10:30.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.793+0000 7f86960beb80 10 client.26468 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:30.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.793+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558164e10c50 con 0x558164d8ef30 2024-05-20T20:10:30.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f866aa00700 1 -- 172.21.15.28:0/3831501860 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f867001a860 con 0x558164d8ef30 2024-05-20T20:10:30.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f866aa00700 1 -- 172.21.15.28:0/3831501860 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8670003b40 con 0x558164d8ef30 2024-05-20T20:10:30.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f86960beb80 10 client.26468 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:30.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f86960beb80 10 client.26468 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:30.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f86960beb80 20 client.26468 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:30.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f86960beb80 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164e112e0 0x558164d9df30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:30.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f86960beb80 4 client.26468 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:10:30.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.794+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x558164ce35f0 con 0x558164e112e0 2024-05-20T20:10:30.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.795+0000 7f8689800700 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164e112e0 0x558164d9df30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:30.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.796+0000 7f8689800700 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164e112e0 0x558164d9df30 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b6193315086f4b0c server_cookie=ee9a304a5f4bdbeb in_seq=0 out_seq=0 2024-05-20T20:10:30.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.797+0000 7f866aa00700 10 client.26468 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:30.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.797+0000 7f866aa00700 1 -- 172.21.15.28:0/3831501860 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x558164ce35f0 con 0x558164e112e0 2024-05-20T20:10:30.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.797+0000 7f866aa00700 10 client.26468 handle_command_reply: tid=0 2024-05-20T20:10:30.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.797+0000 7f86960beb80 1 client.26468 shutdown 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8ef30 msgr2=0x558164daf290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8ef30 0x558164daf290 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f8670004790 tx=0x7f8670009a60 comp rx=0 tx=0).stop 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 shutdown_connections 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558164e112e0 0x558164d9df30 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558164d8f850 0x558164daf7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558164d89900 0x558164db0ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 --2- 172.21.15.28:0/3831501860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558164d8ef30 0x558164daf290 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 >> 172.21.15.28:0/3831501860 conn(0x558164cb7c60 msgr2=0x558164d91af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:30.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.798+0000 7f866aa00700 0 client.26468 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:30.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.799+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 shutdown_connections 2024-05-20T20:10:30.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.800+0000 7f86960beb80 1 -- 172.21.15.28:0/3831501860 wait complete. 2024-05-20T20:10:30.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.801+0000 7f86960beb80 20 client.26468 trim_cache size 0 max 16384 2024-05-20T20:10:30.803 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:10:30.803 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:10:30.803 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "6cb86fcf-a8bc-4fc9-9c0a-21f5cb4e5b08", 2024-05-20T20:10:30.803 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:10:30.803 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:10:30.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.803+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f866c0b2af0 msgr2=0x7f866c0b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.803+0000 7f868b800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f866c0b2af0 0x7f866c0b4fb0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f86780a6e60 tx=0x7f8678005020 comp rx=0 tx=0).stop 2024-05-20T20:10:30.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.803+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f868406b220 msgr2=0x7f8684102510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:30.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.803+0000 7f868b800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f868406b220 0x7f8684102510 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f867800e9c0 tx=0x7f867800ed80 comp rx=0 tx=0).stop 2024-05-20T20:10:30.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 shutdown_connections 2024-05-20T20:10:30.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f86841095b0 0x7f8684102f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f868406b220 0x7f8684102510 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f866c0b2af0 0x7f866c0b4fb0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 --2- 172.21.15.28:0/776049994 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f868406bbe0 0x7f8684102a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:30.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 >> 172.21.15.28:0/776049994 conn(0x7f86840793f0 msgr2=0x7f86841011a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:30.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 shutdown_connections 2024-05-20T20:10:30.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:30.804+0000 7f868b800700 1 -- 172.21.15.28:0/776049994 wait complete. 2024-05-20T20:10:31.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:30 smithi028 ceph-mon[31178]: pgmap v149: 97 pgs: 97 active+clean; 22 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 320 MiB/s wr, 2.49k op/s 2024-05-20T20:10:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:30 smithi154 ceph-mon[36314]: pgmap v149: 97 pgs: 97 active+clean; 22 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 320 MiB/s wr, 2.49k op/s 2024-05-20T20:10:31.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:30 smithi033 ceph-mon[36179]: pgmap v149: 97 pgs: 97 active+clean; 22 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 320 MiB/s wr, 2.49k op/s 2024-05-20T20:10:32.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:31 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:10:32.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:31 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:10:32.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:31 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:10:32.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:31 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:10:32.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:31 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:10:32.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:31 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:10:32.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:31 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:10:32.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:31 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:10:32.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:31 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:10:32.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:31 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:10:32.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:31 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:10:32.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:31 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:10:33.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:32 smithi154 ceph-mon[36314]: pgmap v150: 97 pgs: 97 active+clean; 22 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 213 MiB/s wr, 1.61k op/s 2024-05-20T20:10:33.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:32 smithi033 ceph-mon[36179]: pgmap v150: 97 pgs: 97 active+clean; 22 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 213 MiB/s wr, 1.61k op/s 2024-05-20T20:10:33.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:32 smithi028 ceph-mon[31178]: pgmap v150: 97 pgs: 97 active+clean; 22 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 213 MiB/s wr, 1.61k op/s 2024-05-20T20:10:34.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:33 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:10:34.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:33 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:10:34.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:33 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:10:34.730 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:10:35.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:34 smithi028 ceph-mon[31178]: pgmap v151: 97 pgs: 97 active+clean; 23 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 292 MiB/s wr, 2.06k op/s 2024-05-20T20:10:35.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:34 smithi154 ceph-mon[36314]: pgmap v151: 97 pgs: 97 active+clean; 23 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 292 MiB/s wr, 2.06k op/s 2024-05-20T20:10:35.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:34 smithi033 ceph-mon[36179]: pgmap v151: 97 pgs: 97 active+clean; 23 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 292 MiB/s wr, 2.06k op/s 2024-05-20T20:10:36.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.279+0000 7f27e7e00700 1 -- 172.21.15.28:0/3056675034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e006b240 msgr2=0x7f27e006b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.279+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3056675034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e006b240 0x7f27e006b6c0 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f27d800b0a0 tx=0x7f27d800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.279+0000 7f27e7e00700 1 -- 172.21.15.28:0/3056675034 shutdown_connections 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.279+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3056675034 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e006b240 0x7f27e006b6c0 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.279+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3056675034 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f27e01084f0 0x7f27e0105fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.279+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3056675034 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e01064e0 0x7f27e0106960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.279+0000 7f27e7e00700 1 -- 172.21.15.28:0/3056675034 >> 172.21.15.28:0/3056675034 conn(0x7f27e00793e0 msgr2=0x7f27e00797f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.280+0000 7f27e7e00700 1 -- 172.21.15.28:0/3056675034 shutdown_connections 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.280+0000 7f27e7e00700 1 -- 172.21.15.28:0/3056675034 wait complete. 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.281+0000 7f27e7e00700 1 Processor -- start 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e7e00700 1 -- start start 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e006b240 0x7f27e01a0170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f27e01064e0 0x7f27e01a06b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e01084f0 0x7f27e01a49f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e7e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f27e0119d60 con 0x7f27e01064e0 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e7e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f27e0119be0 con 0x7f27e006b240 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e7e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f27e0119ee0 con 0x7f27e01084f0 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.282+0000 7f27e5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e006b240 0x7f27e01a0170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.283+0000 7f27e5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e006b240 0x7f27e01a0170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42382/0 (socket says 172.21.15.28:42382) 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.283+0000 7f27e5400700 1 -- 172.21.15.28:0/3386590978 learned_addr learned my addr 172.21.15.28:0/3386590978 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.283+0000 7f27e4a00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f27e01064e0 0x7f27e01a06b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.283+0000 7f27e5e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e01084f0 0x7f27e01a49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.284+0000 7f27e5e00700 1 -- 172.21.15.28:0/3386590978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e006b240 msgr2=0x7f27e01a0170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.284+0000 7f27e5e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e006b240 0x7f27e01a0170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.284+0000 7f27e5e00700 1 -- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f27e01064e0 msgr2=0x7f27e01a06b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.284+0000 7f27e5e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f27e01064e0 0x7f27e01a06b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.284+0000 7f27e5e00700 1 -- 172.21.15.28:0/3386590978 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f27e01a5130 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.285+0000 7f27e5400700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e006b240 0x7f27e01a0170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.285+0000 7f27e5e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e01084f0 0x7f27e01a49f0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f27d8009fd0 tx=0x7f27d8004dc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.285+0000 7f27de000700 1 -- 172.21.15.28:0/3386590978 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d8021070 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.285+0000 7f27de000700 1 -- 172.21.15.28:0/3386590978 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f27d8003c00 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.285+0000 7f27de000700 1 -- 172.21.15.28:0/3386590978 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d80046d0 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.285+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f27e01a53c0 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.286+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f27e006cf60 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.292+0000 7f27de000700 1 -- 172.21.15.28:0/3386590978 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f27d8007980 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.295+0000 7f27de000700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f27c80b2ba0 0x7f27c80b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.295+0000 7f27de000700 1 -- 172.21.15.28:0/3386590978 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f27d800f020 con 0x7f27e01084f0 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.295+0000 7f27f27e9b80 1 Processor -- start 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.295+0000 7f27f27e9b80 1 -- start start 2024-05-20T20:10:36.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.296+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa44080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.296+0000 7f27e5400700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f27c80b2ba0 0x7f27c80b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.296+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa445c0 0x5575efa3e060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.296+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa3eae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.296+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5575ef99bfd0 con 0x5575efa3e680 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.296+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5575ef98bb70 con 0x5575efa43ca0 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.296+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5575ef98fbb0 con 0x5575efa445c0 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.297+0000 7f27e5e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa3eae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.297+0000 7f27e5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa44080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.297+0000 7f27e5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa44080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42386/0 (socket says 172.21.15.28:42386) 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.297+0000 7f27e5400700 1 -- 172.21.15.28:0/2123631953 learned_addr learned my addr 172.21.15.28:0/2123631953 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.297+0000 7f27e4a00700 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa445c0 0x5575efa3e060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.298+0000 7f27e5400700 1 -- 172.21.15.28:0/2123631953 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa445c0 msgr2=0x5575efa3e060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:36.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.298+0000 7f27e5400700 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa445c0 0x5575efa3e060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.298+0000 7f27e5400700 1 -- 172.21.15.28:0/2123631953 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 msgr2=0x5575efa3eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.298+0000 7f27e5400700 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa3eae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.298+0000 7f27e5400700 1 -- 172.21.15.28:0/2123631953 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575ef9545b0 con 0x5575efa43ca0 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.298+0000 7f27e5400700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f27c80b2ba0 0x7f27c80b5060 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f27d400afd0 tx=0x7f27d4013de0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.299+0000 7f27e5400700 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa44080 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f27d400faf0 tx=0x7f27d4019000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b919f689caacacf3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.300+0000 7f27d2a00700 1 -- 172.21.15.28:0/2123631953 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d400d650 con 0x5575efa43ca0 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.300+0000 7f27d2a00700 1 -- 172.21.15.28:0/2123631953 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f27d4024070 con 0x5575efa43ca0 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.300+0000 7f27d2a00700 1 -- 172.21.15.28:0/2123631953 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d401f910 con 0x5575efa43ca0 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.301+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2123631953 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 msgr2=0x5575efa44080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.301+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa44080 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f27d400faf0 tx=0x7f27d4019000 comp rx=0 tx=0).stop 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.302+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2123631953 shutdown_connections 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.302+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa3eae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.302+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa44080 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.302+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2123631953 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa445c0 0x5575efa3e060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.302+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2123631953 >> 172.21.15.28:0/2123631953 conn(0x5575ef96bd70 msgr2=0x5575efa38f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:36.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.303+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2123631953 shutdown_connections 2024-05-20T20:10:36.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.303+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2123631953 wait complete. 2024-05-20T20:10:36.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.304+0000 7f27f27e9b80 1 -- start start 2024-05-20T20:10:36.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.304+0000 7f27f27e9b80 1 Processor -- start 2024-05-20T20:10:36.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.312+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa4a290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.312+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa43ca0 0x5575efa47630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.312+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa47ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.314+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5575ef892130 con 0x5575efa43ca0 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.314+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5575ef92c8f0 con 0x5575efa47b70 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.314+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5575efa4afd0 con 0x5575efa3e680 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.315+0000 7f27f27e9b80 10 client.0 _ll_register_callbacks cb 0x5575ef935e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.316+0000 7f27e5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa4a290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.316+0000 7f27e5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa4a290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43930/0 (socket says 172.21.15.28:43930) 2024-05-20T20:10:36.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.317+0000 7f27e5400700 1 -- 172.21.15.28:0/3728359805 learned_addr learned my addr 172.21.15.28:0/3728359805 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:36.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.318+0000 7f27e5e00700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa47ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.318+0000 7f27e4a00700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa43ca0 0x5575efa47630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.319+0000 7f27e5400700 1 -- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 msgr2=0x5575efa47ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:36.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.319+0000 7f27e5400700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa47ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.319+0000 7f27e5400700 1 -- 172.21.15.28:0/3728359805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa43ca0 msgr2=0x5575efa47630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:36.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.319+0000 7f27e5400700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa43ca0 0x5575efa47630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.320+0000 7f27e5400700 1 -- 172.21.15.28:0/3728359805 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575ef8dba60 con 0x5575efa3e680 2024-05-20T20:10:36.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.321+0000 7f27e5e00700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa47ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:36.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.322+0000 7f27e5400700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa4a290 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f27d400a700 tx=0x7f27d400db50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=101c477220bc4f21 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:36.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.322+0000 7f27c7400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:10:36.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.323+0000 7f27c7400700 1 -- 172.21.15.28:0/3728359805 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d402f070 con 0x5575efa3e680 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.324+0000 7f27c7400700 1 -- 172.21.15.28:0/3728359805 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f27d401f790 con 0x5575efa3e680 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.324+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f27d4024070 con 0x5575efa3e680 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.325+0000 7f27c7400700 1 -- 172.21.15.28:0/3728359805 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f27d4039050 con 0x5575efa3e680 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.325+0000 7f27f27e9b80 10 client.26401 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.325+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5575ef95c1f0 con 0x5575efa3e680 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.326+0000 7f27c7400700 1 -- 172.21.15.28:0/3728359805 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f27d4024070 con 0x5575efa3e680 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.326+0000 7f27c7400700 1 -- 172.21.15.28:0/3728359805 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f27d40347b0 con 0x5575efa3e680 2024-05-20T20:10:36.327 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.326+0000 7f27f27e9b80 10 client.26401 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:36.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.327+0000 7f27f27e9b80 10 client.26401 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:10:36.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.327+0000 7f27f27e9b80 20 client.26401 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:36.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.327+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efa57ec0 0x5575efa5a2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.327+0000 7f27f27e9b80 4 client.26401 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:36.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.327+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5575ef99df20 con 0x5575efa57ec0 2024-05-20T20:10:36.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.328+0000 7f27e4a00700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efa57ec0 0x5575efa5a2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.330+0000 7f27c7400700 10 client.26401 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:36.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.332+0000 7f27e4a00700 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efa57ec0 0x5575efa5a2b0 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7cb83a39d30e4af1 server_cookie=56a2baaf509324b2 in_seq=0 out_seq=0 2024-05-20T20:10:36.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.333+0000 7f27c7400700 1 -- 172.21.15.28:0/3728359805 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5575ef99df20 con 0x5575efa57ec0 2024-05-20T20:10:36.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.333+0000 7f27c7400700 10 client.26401 handle_command_reply: tid=0 2024-05-20T20:10:36.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.334+0000 7f27f27e9b80 1 client.26401 shutdown 2024-05-20T20:10:36.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.336+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 msgr2=0x5575efa4a290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.336+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa4a290 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f27d400a700 tx=0x7f27d400db50 comp rx=0 tx=0).stop 2024-05-20T20:10:36.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 shutdown_connections 2024-05-20T20:10:36.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efa57ec0 0x5575efa5a2b0 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa47ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa4a290 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/3728359805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa43ca0 0x5575efa47630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 >> 172.21.15.28:0/3728359805 conn(0x5575ef96bd70 msgr2=0x5575efa384c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:36.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27c7400700 0 client.26401 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:36.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.343+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 shutdown_connections 2024-05-20T20:10:36.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.344+0000 7f27f27e9b80 1 -- 172.21.15.28:0/3728359805 wait complete. 2024-05-20T20:10:36.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.344+0000 7f27f27e9b80 20 client.26401 trim_cache size 0 max 16384 2024-05-20T20:10:36.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.397+0000 7f27f27e9b80 1 Processor -- start 2024-05-20T20:10:36.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.397+0000 7f27f27e9b80 1 -- start start 2024-05-20T20:10:36.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.398+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa3e060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.398+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa45240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.398+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa47b70 0x5575efa46630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.398+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f27e00744d0 con 0x5575efa47b70 2024-05-20T20:10:36.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.398+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f27e01be560 con 0x5575efa43ca0 2024-05-20T20:10:36.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.398+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f27e0074ba0 con 0x5575efa3e680 2024-05-20T20:10:36.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.398+0000 7f27e4a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa45240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e4a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa45240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42416/0 (socket says 172.21.15.28:42416) 2024-05-20T20:10:36.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e4a00700 1 -- 172.21.15.28:0/2587343303 learned_addr learned my addr 172.21.15.28:0/2587343303 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:36.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5e00700 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa47b70 0x5575efa46630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5400700 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa3e060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5400700 1 -- 172.21.15.28:0/2587343303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 msgr2=0x5575efa45240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5400700 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa45240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5400700 1 -- 172.21.15.28:0/2587343303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa47b70 msgr2=0x5575efa46630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5400700 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa47b70 0x5575efa46630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5400700 1 -- 172.21.15.28:0/2587343303 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575efa45ba0 con 0x5575efa3e680 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.399+0000 7f27e5e00700 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa47b70 0x5575efa46630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.400+0000 7f27e5400700 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa3e060 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f27d4034d90 tx=0x7f27d400ad40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e2f6c6d1e3166800 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:36.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.400+0000 7f27d2000700 1 -- 172.21.15.28:0/2587343303 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d402c070 con 0x5575efa3e680 2024-05-20T20:10:36.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.400+0000 7f27d2000700 1 -- 172.21.15.28:0/2587343303 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f27d4024040 con 0x5575efa3e680 2024-05-20T20:10:36.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.400+0000 7f27d2000700 1 -- 172.21.15.28:0/2587343303 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d402f040 con 0x5575efa3e680 2024-05-20T20:10:36.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.402+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2587343303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 msgr2=0x5575efa3e060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.402+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa3e060 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f27d4034d90 tx=0x7f27d400ad40 comp rx=0 tx=0).stop 2024-05-20T20:10:36.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.402+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2587343303 shutdown_connections 2024-05-20T20:10:36.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.402+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa47b70 0x5575efa46630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.402+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa3e680 0x5575efa3e060 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.402+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/2587343303 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa43ca0 0x5575efa45240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.402+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2587343303 >> 172.21.15.28:0/2587343303 conn(0x5575ef96bd70 msgr2=0x5575efa45f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:36.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.404+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2587343303 shutdown_connections 2024-05-20T20:10:36.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.404+0000 7f27f27e9b80 1 -- 172.21.15.28:0/2587343303 wait complete. 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.405+0000 7f27f27e9b80 1 -- start start 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.405+0000 7f27f27e9b80 1 Processor -- start 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.406+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa50550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.406+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa43ca0 0x5575efa50a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.406+0000 7f27f27e9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa4d3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.406+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f27e01bde90 con 0x5575efa3e680 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.406+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f27e0075270 con 0x5575efa47b70 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.406+0000 7f27f27e9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f27e0072470 con 0x5575efa43ca0 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.406+0000 7f27f27e9b80 10 client.0 _ll_register_callbacks cb 0x5575ef935e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:36.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa50550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa50550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57772/0 (socket says 172.21.15.28:57772) 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 -- 172.21.15.28:0/1724943105 learned_addr learned my addr 172.21.15.28:0/1724943105 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 -- 172.21.15.28:0/1724943105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa43ca0 msgr2=0x5575efa50a90 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa43ca0 0x5575efa50a90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 -- 172.21.15.28:0/1724943105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 msgr2=0x5575efa4d3a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa4d3a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.407+0000 7f27e5400700 1 -- 172.21.15.28:0/1724943105 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575efa4dbe0 con 0x5575efa3e680 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.408+0000 7f27e5400700 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa50550 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f27d400af80 tx=0x7f27d400f6c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=54da90f4df5ca375 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.408+0000 7f27c7400700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.408+0000 7f27c7400700 1 -- 172.21.15.28:0/1724943105 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27d402c040 con 0x5575efa3e680 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.408+0000 7f27c7400700 1 -- 172.21.15.28:0/1724943105 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f27d401f6c0 con 0x5575efa3e680 2024-05-20T20:10:36.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.408+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f27d4024040 con 0x5575efa3e680 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.409+0000 7f27c7400700 1 -- 172.21.15.28:0/1724943105 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f27d402f040 con 0x5575efa3e680 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.409+0000 7f27f27e9b80 10 client.16254 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.409+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5575efac3420 con 0x5575efa3e680 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.409+0000 7f27c7400700 1 -- 172.21.15.28:0/1724943105 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f27d4024040 con 0x5575efa3e680 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.410+0000 7f27c7400700 1 -- 172.21.15.28:0/1724943105 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f27d40109b0 con 0x5575efa3e680 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.410+0000 7f27f27e9b80 10 client.16254 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.410+0000 7f27f27e9b80 10 client.16254 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.410+0000 7f27f27e9b80 20 client.16254 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.410+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efac3a80 0x5575efac5e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.410+0000 7f27f27e9b80 4 client.16254 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 144.7, "await_for": 45.0}] 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.410+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 144.7, "await_for": 45.0}) -- 0x5575ef997700 con 0x5575efac3a80 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.411+0000 7f27e4a00700 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efac3a80 0x5575efac5e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.411+0000 7f27e4a00700 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efac3a80 0x5575efac5e70 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=283efecfde9a830f server_cookie=8bb8601b3c7bf3aa in_seq=0 out_seq=0 2024-05-20T20:10:36.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.411+0000 7f27c7400700 10 client.16254 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.472+0000 7f27c7400700 1 -- 172.21.15.28:0/1724943105 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x5575ef997700 con 0x5575efac3a80 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.472+0000 7f27c7400700 10 client.16254 handle_command_reply: tid=0 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.472+0000 7f27f27e9b80 1 client.16254 shutdown 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.473+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 msgr2=0x5575efa50550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.474+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa50550 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f27d400af80 tx=0x7f27d400f6c0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 shutdown_connections 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5575efac3a80 0x5575efac5e70 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575efa47b70 0x5575efa4d3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27c7400700 0 client.16254 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:36.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575efa3e680 0x5575efa50550 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27f27e9b80 1 --2- 172.21.15.28:0/1724943105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575efa43ca0 0x5575efa50a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 >> 172.21.15.28:0/1724943105 conn(0x5575ef96bd70 msgr2=0x5575efa53090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:36.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.475+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 shutdown_connections 2024-05-20T20:10:36.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.478+0000 7f27f27e9b80 1 -- 172.21.15.28:0/1724943105 wait complete. 2024-05-20T20:10:36.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.478+0000 7f27f27e9b80 20 client.16254 trim_cache size 0 max 16384 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 10, 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "65623019": { 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 10, 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:10:36.484 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 144.7, 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:36.485 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:36.486 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:36.486 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:36.486 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:10:36.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.481+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f27c80b2ba0 msgr2=0x7f27c80b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f27c80b2ba0 0x7f27c80b5060 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f27d400afd0 tx=0x7f27d4013de0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e01084f0 msgr2=0x7f27e01a49f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:36.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e01084f0 0x7f27e01a49f0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f27d8009fd0 tx=0x7f27d8004dc0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 shutdown_connections 2024-05-20T20:10:36.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f27e01084f0 0x7f27e01a49f0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f27c80b2ba0 0x7f27c80b5060 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f27e006b240 0x7f27e01a0170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 --2- 172.21.15.28:0/3386590978 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f27e01064e0 0x7f27e01a06b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:36.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 >> 172.21.15.28:0/3386590978 conn(0x7f27e00793e0 msgr2=0x7f27e01002b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:36.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 shutdown_connections 2024-05-20T20:10:36.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:36.484+0000 7f27e7e00700 1 -- 172.21.15.28:0/3386590978 wait complete. 2024-05-20T20:10:36.556 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:37.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:36 smithi154 ceph-mon[36314]: pgmap v152: 97 pgs: 97 active+clean; 23 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 262 MiB/s wr, 1.78k op/s 2024-05-20T20:10:37.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:36 smithi033 ceph-mon[36179]: pgmap v152: 97 pgs: 97 active+clean; 23 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 262 MiB/s wr, 1.78k op/s 2024-05-20T20:10:37.292 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:36 smithi028 ceph-mon[31178]: pgmap v152: 97 pgs: 97 active+clean; 23 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 262 MiB/s wr, 1.78k op/s 2024-05-20T20:10:37.929 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set '65623019', quiesce took 23.4 seconds. Will release after: 1.3000000000000007 2024-05-20T20:10:38.497 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 -- 172.21.15.28:0/1890300269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198103430 msgr2=0x7fe19810f940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.497 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 --2- 172.21.15.28:0/1890300269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198103430 0x7fe19810f940 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fe190009c10 tx=0x7fe19002a6f0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.498 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 -- 172.21.15.28:0/1890300269 shutdown_connections 2024-05-20T20:10:38.498 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 --2- 172.21.15.28:0/1890300269 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198103430 0x7fe19810f940 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.498 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 --2- 172.21.15.28:0/1890300269 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe198102a70 0x7fe198102ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.498 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 --2- 172.21.15.28:0/1890300269 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe198108a80 0x7fe198108e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.498 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 -- 172.21.15.28:0/1890300269 >> 172.21.15.28:0/1890300269 conn(0x7fe1980fe500 msgr2=0x7fe198100920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:38.498 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.493+0000 7fe19e800640 1 -- 172.21.15.28:0/1890300269 shutdown_connections 2024-05-20T20:10:38.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.494+0000 7fe19e800640 1 -- 172.21.15.28:0/1890300269 wait complete. 2024-05-20T20:10:38.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.496+0000 7fe19e800640 1 Processor -- start 2024-05-20T20:10:38.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.497+0000 7fe19e800640 1 -- start start 2024-05-20T20:10:38.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.500+0000 7fe19e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe198102a70 0x7fe198074240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.500+0000 7fe19e800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe198103430 0x7fe19806d2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.500+0000 7fe19e800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198108a80 0x7fe19806d800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.500+0000 7fe19e800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe198112d40 con 0x7fe198102a70 2024-05-20T20:10:38.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.500+0000 7fe19e800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe198112bc0 con 0x7fe198103430 2024-05-20T20:10:38.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.500+0000 7fe19e800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe198112ec0 con 0x7fe198108a80 2024-05-20T20:10:38.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.501+0000 7fe197400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe198102a70 0x7fe198074240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.501+0000 7fe197400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe198102a70 0x7fe198074240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51610/0 (socket says 172.21.15.28:51610) 2024-05-20T20:10:38.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.501+0000 7fe197400640 1 -- 172.21.15.28:0/1103518254 learned_addr learned my addr 172.21.15.28:0/1103518254 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:38.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.501+0000 7fe197e00640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198108a80 0x7fe19806d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.502+0000 7fe197e00640 1 -- 172.21.15.28:0/1103518254 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe198103430 msgr2=0x7fe19806d2c0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:10:38.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.502+0000 7fe197e00640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe198103430 0x7fe19806d2c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.502+0000 7fe197e00640 1 -- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe198102a70 msgr2=0x7fe198074240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.502+0000 7fe197e00640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe198102a70 0x7fe198074240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.502+0000 7fe197e00640 1 -- 172.21.15.28:0/1103518254 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe19806ddc0 con 0x7fe198108a80 2024-05-20T20:10:38.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.505+0000 7fe197e00640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198108a80 0x7fe19806d800 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fe190034040 tx=0x7fe190003c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:38.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.505+0000 7fe18fe00640 1 -- 172.21.15.28:0/1103518254 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe190040020 con 0x7fe198108a80 2024-05-20T20:10:38.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.506+0000 7fe18fe00640 1 -- 172.21.15.28:0/1103518254 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe1900318f0 con 0x7fe198108a80 2024-05-20T20:10:38.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.506+0000 7fe18fe00640 1 -- 172.21.15.28:0/1103518254 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe190036040 con 0x7fe198108a80 2024-05-20T20:10:38.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.506+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe19806e050 con 0x7fe198108a80 2024-05-20T20:10:38.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.506+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe198071fa0 con 0x7fe198108a80 2024-05-20T20:10:38.512 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.512+0000 7fe1a1626740 1 Processor -- start 2024-05-20T20:10:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.519+0000 7fe18fe00640 1 -- 172.21.15.28:0/1103518254 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fe190043020 con 0x7fe198108a80 2024-05-20T20:10:38.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.520+0000 7fe1a1626740 1 -- start start 2024-05-20T20:10:38.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.523+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6ccb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.523+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6cd090 0x56049a6cd510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.523+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6e0510 0x56049a6e2900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.523+0000 7fe1a1626740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56049a63db90 con 0x56049a6ce1d0 2024-05-20T20:10:38.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.523+0000 7fe1a1626740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56049a62d730 con 0x56049a6e0510 2024-05-20T20:10:38.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.523+0000 7fe1a1626740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56049a631770 con 0x56049a6cd090 2024-05-20T20:10:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.524+0000 7fe18fe00640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe17c0b2ae0 0x7fe17c0b4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.524+0000 7fe18fe00640 1 -- 172.21.15.28:0/1103518254 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fe190102e10 con 0x7fe198108a80 2024-05-20T20:10:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.524+0000 7fe197400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6cd090 0x56049a6cd510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.524+0000 7fe197400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6cd090 0x56049a6cd510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50356/0 (socket says 172.21.15.28:50356) 2024-05-20T20:10:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.524+0000 7fe197400640 1 -- 172.21.15.28:0/3535450990 learned_addr learned my addr 172.21.15.28:0/3535450990 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.525+0000 7fe197e00640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe17c0b2ae0 0x7fe17c0b4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.525+0000 7fe197400640 1 -- 172.21.15.28:0/3535450990 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6e0510 msgr2=0x56049a6e2900 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:38.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.525+0000 7fe197400640 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6e0510 0x56049a6e2900 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.525+0000 7fe197400640 1 -- 172.21.15.28:0/3535450990 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 msgr2=0x56049a6ccb50 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:38.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.526+0000 7fe196a00640 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6ccb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.526+0000 7fe197400640 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6ccb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.526+0000 7fe197400640 1 -- 172.21.15.28:0/3535450990 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56049a5c68b0 con 0x56049a6cd090 2024-05-20T20:10:38.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.526+0000 7fe196a00640 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6ccb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:38.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.527+0000 7fe197e00640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe17c0b2ae0 0x7fe17c0b4fa0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7fe1900c6180 tx=0x7fe190030fc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:38.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.527+0000 7fe197400640 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6cd090 0x56049a6cd510 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fe184009d60 tx=0x7fe184011810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e4984d9e83c247ef server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:38.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.528+0000 7fe18cc00640 1 -- 172.21.15.28:0/3535450990 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe18401e070 con 0x56049a6cd090 2024-05-20T20:10:38.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.528+0000 7fe18cc00640 1 -- 172.21.15.28:0/3535450990 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe184019ae0 con 0x56049a6cd090 2024-05-20T20:10:38.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.528+0000 7fe18cc00640 1 -- 172.21.15.28:0/3535450990 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe18400ea00 con 0x56049a6cd090 2024-05-20T20:10:38.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.529+0000 7fe1a1626740 1 -- 172.21.15.28:0/3535450990 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6cd090 msgr2=0x56049a6cd510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.529+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6cd090 0x56049a6cd510 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fe184009d60 tx=0x7fe184011810 comp rx=0 tx=0).stop 2024-05-20T20:10:38.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.529+0000 7fe1a1626740 1 -- 172.21.15.28:0/3535450990 shutdown_connections 2024-05-20T20:10:38.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.529+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6e0510 0x56049a6e2900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.529+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6cd090 0x56049a6cd510 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.529+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3535450990 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6ccb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.529+0000 7fe1a1626740 1 -- 172.21.15.28:0/3535450990 >> 172.21.15.28:0/3535450990 conn(0x56049a6c91a0 msgr2=0x56049a6c9580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:38.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.530+0000 7fe1a1626740 1 -- 172.21.15.28:0/3535450990 shutdown_connections 2024-05-20T20:10:38.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.530+0000 7fe1a1626740 1 -- 172.21.15.28:0/3535450990 wait complete. 2024-05-20T20:10:38.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.531+0000 7fe1a1626740 1 -- start start 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.531+0000 7fe1a1626740 1 Processor -- start 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.532+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6d9d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.532+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6da260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.532+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a6dfa40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.532+0000 7fe1a1626740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56049a313790 con 0x56049a6ce1d0 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.532+0000 7fe1a1626740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56049a5d5bc0 con 0x56049a6cd090 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.532+0000 7fe1a1626740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56049a6d4060 con 0x56049a6e0510 2024-05-20T20:10:38.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.532+0000 7fe1a1626740 10 client.0 _ll_register_callbacks cb 0x56049a5b3400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:38.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.533+0000 7fe197400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6da260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.533+0000 7fe196a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6d9d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.533+0000 7fe196a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6d9d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37242/0 (socket says 172.21.15.28:37242) 2024-05-20T20:10:38.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.533+0000 7fe196a00640 1 -- 172.21.15.28:0/2441438146 learned_addr learned my addr 172.21.15.28:0/2441438146 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:38.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.533+0000 7fe196a00640 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a6dfa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.534+0000 7fe196a00640 1 -- 172.21.15.28:0/2441438146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 msgr2=0x56049a6dfa40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:38.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.534+0000 7fe196a00640 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a6dfa40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.534+0000 7fe196a00640 1 -- 172.21.15.28:0/2441438146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 msgr2=0x56049a6da260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:38.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.534+0000 7fe196a00640 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6da260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.534+0000 7fe196a00640 1 -- 172.21.15.28:0/2441438146 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56049a342210 con 0x56049a6cd090 2024-05-20T20:10:38.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.535+0000 7fe197400640 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6da260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:38.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.535+0000 7fe196a00640 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6d9d20 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7fe188015c40 tx=0x7fe188005f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=314c470bc8686356 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:38.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.535+0000 7fe181600640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:38.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.536+0000 7fe181600640 1 -- 172.21.15.28:0/2441438146 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe18800eb80 con 0x56049a6cd090 2024-05-20T20:10:38.535 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.536+0000 7fe181600640 1 -- 172.21.15.28:0/2441438146 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fe18800ed20 con 0x56049a6cd090 2024-05-20T20:10:38.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.536+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe184019ae0 con 0x56049a6cd090 2024-05-20T20:10:38.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.537+0000 7fe181600640 1 -- 172.21.15.28:0/2441438146 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe188004030 con 0x56049a6cd090 2024-05-20T20:10:38.536 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.537+0000 7fe1a1626740 10 client.26489 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:38.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.537+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56049a40e620 con 0x56049a6cd090 2024-05-20T20:10:38.537 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.537+0000 7fe181600640 1 -- 172.21.15.28:0/2441438146 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe184019ae0 con 0x56049a6cd090 2024-05-20T20:10:38.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.538+0000 7fe181600640 1 -- 172.21.15.28:0/2441438146 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe18800f040 con 0x56049a6cd090 2024-05-20T20:10:38.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.539+0000 7fe1a1626740 10 client.26489 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:38.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.539+0000 7fe1a1626740 10 client.26489 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:38.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.539+0000 7fe1a1626740 20 client.26489 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:38.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.539+0000 7fe1a1626740 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a6f86e0 0x56049a6faad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.539+0000 7fe1a1626740 4 client.26489 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:38.539 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.539+0000 7fe197400640 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a6f86e0 0x56049a6faad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.540+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56049a63fae0 con 0x56049a6f86e0 2024-05-20T20:10:38.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.540+0000 7fe197400640 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a6f86e0 0x56049a6faad0 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9b055c8b0dea4bbc server_cookie=9f3e3e3361d4c446 in_seq=0 out_seq=0 2024-05-20T20:10:38.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.540+0000 7fe181600640 10 client.26489 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:38.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.541+0000 7fe181600640 1 -- 172.21.15.28:0/2441438146 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56049a63fae0 con 0x56049a6f86e0 2024-05-20T20:10:38.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.541+0000 7fe181600640 10 client.26489 handle_command_reply: tid=0 2024-05-20T20:10:38.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.541+0000 7fe1a1626740 1 client.26489 shutdown 2024-05-20T20:10:38.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.543+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 msgr2=0x56049a6d9d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.543 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.543+0000 7fe1a1626740 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6d9d20 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7fe188015c40 tx=0x7fe188005f90 comp rx=0 tx=0).stop 2024-05-20T20:10:38.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.545+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 shutdown_connections 2024-05-20T20:10:38.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.545+0000 7fe1a1626740 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a6f86e0 0x56049a6faad0 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.545+0000 7fe1a1626740 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a6dfa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.545+0000 7fe1a1626740 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a6da260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.545+0000 7fe1a1626740 1 --2- 172.21.15.28:0/2441438146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6d9d20 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.545+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 >> 172.21.15.28:0/2441438146 conn(0x56049a6c91a0 msgr2=0x56049a6ceff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:38.546 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.545+0000 7fe181600640 0 client.26489 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.547+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 shutdown_connections 2024-05-20T20:10:38.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.547+0000 7fe1a1626740 1 -- 172.21.15.28:0/2441438146 wait complete. 2024-05-20T20:10:38.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.548+0000 7fe1a1626740 20 client.26489 trim_cache size 0 max 16384 2024-05-20T20:10:38.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.571+0000 7fe1a1626740 1 Processor -- start 2024-05-20T20:10:38.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.571+0000 7fe1a1626740 1 -- start start 2024-05-20T20:10:38.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.571+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6c8190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6ce1d0 0x56049a700350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6e0510 0x56049a700890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe1a1626740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe1981b4d20 con 0x56049a6e0510 2024-05-20T20:10:38.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe1a1626740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe1981b6860 con 0x56049a6cd090 2024-05-20T20:10:38.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe1a1626740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe1981b53f0 con 0x56049a6ce1d0 2024-05-20T20:10:38.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe196a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6c8190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe196a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6c8190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37256/0 (socket says 172.21.15.28:37256) 2024-05-20T20:10:38.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe196a00640 1 -- 172.21.15.28:0/894134734 learned_addr learned my addr 172.21.15.28:0/894134734 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:38.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe197400640 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6ce1d0 0x56049a700350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.572+0000 7fe196a00640 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6e0510 0x56049a700890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.573+0000 7fe196a00640 1 -- 172.21.15.28:0/894134734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6ce1d0 msgr2=0x56049a700350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.573+0000 7fe196a00640 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6ce1d0 0x56049a700350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.573+0000 7fe196a00640 1 -- 172.21.15.28:0/894134734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6e0510 msgr2=0x56049a700890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.573+0000 7fe196a00640 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6e0510 0x56049a700890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.573+0000 7fe196a00640 1 -- 172.21.15.28:0/894134734 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56049a6d2c40 con 0x56049a6cd090 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.573+0000 7fe196a00640 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6c8190 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7fe188004000 tx=0x7fe188015ba0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3a7fcecacfc466fc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.573+0000 7fe183e00640 1 -- 172.21.15.28:0/894134734 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe188004030 con 0x56049a6cd090 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.574+0000 7fe183e00640 1 -- 172.21.15.28:0/894134734 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe188005850 con 0x56049a6cd090 2024-05-20T20:10:38.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.574+0000 7fe183e00640 1 -- 172.21.15.28:0/894134734 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe188013820 con 0x56049a6cd090 2024-05-20T20:10:38.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.574+0000 7fe1a1626740 1 -- 172.21.15.28:0/894134734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 msgr2=0x56049a6c8190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.574+0000 7fe1a1626740 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6c8190 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7fe188004000 tx=0x7fe188015ba0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 -- 172.21.15.28:0/894134734 shutdown_connections 2024-05-20T20:10:38.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6e0510 0x56049a700890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6ce1d0 0x56049a700350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 --2- 172.21.15.28:0/894134734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a6c8190 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 -- 172.21.15.28:0/894134734 >> 172.21.15.28:0/894134734 conn(0x56049a6c91a0 msgr2=0x56049a6c7a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:38.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 -- 172.21.15.28:0/894134734 shutdown_connections 2024-05-20T20:10:38.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 -- 172.21.15.28:0/894134734 wait complete. 2024-05-20T20:10:38.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.575+0000 7fe1a1626740 1 -- start start 2024-05-20T20:10:38.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.576+0000 7fe1a1626740 1 Processor -- start 2024-05-20T20:10:38.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.578+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a70c280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.578+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a70c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.578+0000 7fe1a1626740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a707820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.578+0000 7fe1a1626740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe1981b6190 con 0x56049a6ce1d0 2024-05-20T20:10:38.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.578+0000 7fe1a1626740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe1981b5ac0 con 0x56049a6cd090 2024-05-20T20:10:38.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.578+0000 7fe1a1626740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe1981b2cc0 con 0x56049a6e0510 2024-05-20T20:10:38.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.579+0000 7fe1a1626740 10 client.0 _ll_register_callbacks cb 0x7fe188009900 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:38.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.579+0000 7fe196a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a70c280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.579+0000 7fe196a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a70c280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37260/0 (socket says 172.21.15.28:37260) 2024-05-20T20:10:38.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.579+0000 7fe196a00640 1 -- 172.21.15.28:0/3179570898 learned_addr learned my addr 172.21.15.28:0/3179570898 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:38.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.580+0000 7fe196a00640 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a707820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.581+0000 7fe196a00640 1 -- 172.21.15.28:0/3179570898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 msgr2=0x56049a707820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:38.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.581+0000 7fe196a00640 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a707820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.581+0000 7fe196a00640 1 -- 172.21.15.28:0/3179570898 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 msgr2=0x56049a70c7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:38.581 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.581+0000 7fe196a00640 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a70c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.583 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.581+0000 7fe196a00640 1 -- 172.21.15.28:0/3179570898 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56049a708020 con 0x56049a6cd090 2024-05-20T20:10:38.583 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.583+0000 7fe196a00640 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a70c280 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7fe188009c20 tx=0x7fe1880093f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7546e68eae7b92e1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:38.583 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.583+0000 7fe181600640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:38.588 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.588+0000 7fe181600640 1 -- 172.21.15.28:0/3179570898 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe188032040 con 0x56049a6cd090 2024-05-20T20:10:38.589 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.589+0000 7fe181600640 1 -- 172.21.15.28:0/3179570898 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fe1880150f0 con 0x56049a6cd090 2024-05-20T20:10:38.589 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.589+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe188005850 con 0x56049a6cd090 2024-05-20T20:10:38.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.590+0000 7fe181600640 1 -- 172.21.15.28:0/3179570898 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe188013510 con 0x56049a6cd090 2024-05-20T20:10:38.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.591+0000 7fe1a1626740 10 client.26495 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:38.590 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.591+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56049a762c60 con 0x56049a6cd090 2024-05-20T20:10:38.591 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.591+0000 7fe181600640 1 -- 172.21.15.28:0/3179570898 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe188005850 con 0x56049a6cd090 2024-05-20T20:10:38.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.592+0000 7fe181600640 1 -- 172.21.15.28:0/3179570898 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe188021470 con 0x56049a6cd090 2024-05-20T20:10:38.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.593+0000 7fe1a1626740 10 client.26495 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:38.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.593+0000 7fe1a1626740 10 client.26495 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:38.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.593+0000 7fe1a1626740 20 client.26495 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:38.592 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.593+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a763340 0x56049a765730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:38.593 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.593+0000 7fe1a1626740 4 client.26495 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:10:38.593 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.593+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56049a6392c0 con 0x56049a763340 2024-05-20T20:10:38.593 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.593+0000 7fe197400640 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a763340 0x56049a765730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:38.593 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.594+0000 7fe197400640 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a763340 0x56049a765730 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d04a21c4a4b8b671 server_cookie=1f33e14e914aef7f in_seq=0 out_seq=0 2024-05-20T20:10:38.593 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.594+0000 7fe181600640 10 client.26495 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:38.608 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.608+0000 7fe181600640 1 -- 172.21.15.28:0/3179570898 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x56049a6392c0 con 0x56049a763340 2024-05-20T20:10:38.608 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.608+0000 7fe181600640 10 client.26495 handle_command_reply: tid=0 2024-05-20T20:10:38.608 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.608+0000 7fe1a1626740 1 client.26495 shutdown 2024-05-20T20:10:38.608 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.608+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 msgr2=0x56049a70c280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.608 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.608+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a70c280 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7fe188009c20 tx=0x7fe1880093f0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.609 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.609+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 shutdown_connections 2024-05-20T20:10:38.609 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.609+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56049a763340 0x56049a765730 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.609 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.609+0000 7fe181600640 0 client.26495 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:38.609 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.609+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56049a6e0510 0x56049a707820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.609 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.609+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56049a6ce1d0 0x56049a70c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.609 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.609+0000 7fe1a1626740 1 --2- 172.21.15.28:0/3179570898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56049a6cd090 0x56049a70c280 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.609 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.609+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 >> 172.21.15.28:0/3179570898 conn(0x56049a6c91a0 msgr2=0x56049a6c8310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:38.611 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.612+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 shutdown_connections 2024-05-20T20:10:38.612 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.612+0000 7fe1a1626740 1 -- 172.21.15.28:0/3179570898 wait complete. 2024-05-20T20:10:38.612 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.613+0000 7fe1a1626740 20 client.26495 trim_cache size 0 max 16384 2024-05-20T20:10:38.612 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:10:38.613 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:10:38.613 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:10:38.614 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe17c0b2ae0 msgr2=0x7fe17c0b4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.614 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe17c0b2ae0 0x7fe17c0b4fa0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7fe1900c6180 tx=0x7fe190030fc0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.614 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198108a80 msgr2=0x7fe19806d800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:38.614 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198108a80 0x7fe19806d800 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fe190034040 tx=0x7fe190003c10 comp rx=0 tx=0).stop 2024-05-20T20:10:38.614 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 shutdown_connections 2024-05-20T20:10:38.615 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe198103430 0x7fe19806d2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.615 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe17c0b2ae0 0x7fe17c0b4fa0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.615 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe198108a80 0x7fe19806d800 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.615 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 --2- 172.21.15.28:0/1103518254 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe198102a70 0x7fe198074240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:38.615 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 >> 172.21.15.28:0/1103518254 conn(0x7fe1980fe500 msgr2=0x7fe1980fec50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:38.615 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 shutdown_connections 2024-05-20T20:10:38.615 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:38.614+0000 7fe19e800640 1 -- 172.21.15.28:0/1103518254 wait complete. 2024-05-20T20:10:39.230 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:10:39.301 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:38 smithi028 ceph-mon[31178]: pgmap v153: 97 pgs: 97 active+clean; 23 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 226 MiB/s wr, 1.50k op/s 2024-05-20T20:10:39.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:38 smithi154 ceph-mon[36314]: pgmap v153: 97 pgs: 97 active+clean; 23 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 226 MiB/s wr, 1.50k op/s 2024-05-20T20:10:39.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:38 smithi033 ceph-mon[36179]: pgmap v153: 97 pgs: 97 active+clean; 23 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 226 MiB/s wr, 1.50k op/s 2024-05-20T20:10:40.565 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:41.183 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:40 smithi028 ceph-mon[31178]: pgmap v154: 97 pgs: 97 active+clean; 25 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 348 MiB/s wr, 2.03k op/s 2024-05-20T20:10:41.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:40 smithi154 ceph-mon[36314]: pgmap v154: 97 pgs: 97 active+clean; 25 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 348 MiB/s wr, 2.03k op/s 2024-05-20T20:10:41.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:40 smithi033 ceph-mon[36179]: pgmap v154: 97 pgs: 97 active+clean; 25 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 348 MiB/s wr, 2.03k op/s 2024-05-20T20:10:41.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.938+0000 7fd2d0800700 1 -- 172.21.15.28:0/3184647091 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c8111d20 msgr2=0x7fd2c810a540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:41.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.938+0000 7fd2d0800700 1 --2- 172.21.15.28:0/3184647091 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c8111d20 0x7fd2c810a540 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fd2bc009990 tx=0x7fd2bc009ca0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.938+0000 7fd2d0800700 1 -- 172.21.15.28:0/3184647091 shutdown_connections 2024-05-20T20:10:41.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.938+0000 7fd2d0800700 1 --2- 172.21.15.28:0/3184647091 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2c811cd20 0x7fd2c811f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.938+0000 7fd2d0800700 1 --2- 172.21.15.28:0/3184647091 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c8111d20 0x7fd2c810a540 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.938+0000 7fd2d0800700 1 --2- 172.21.15.28:0/3184647091 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd2c810aa80 0x7fd2c810af00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.938+0000 7fd2d0800700 1 -- 172.21.15.28:0/3184647091 >> 172.21.15.28:0/3184647091 conn(0x7fd2c806e6b0 msgr2=0x7fd2c806eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:41.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.942+0000 7fd2d0800700 1 -- 172.21.15.28:0/3184647091 shutdown_connections 2024-05-20T20:10:41.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.942+0000 7fd2d0800700 1 -- 172.21.15.28:0/3184647091 wait complete. 2024-05-20T20:10:41.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.943+0000 7fd2d0800700 1 Processor -- start 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.943+0000 7fd2d0800700 1 -- start start 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2d0800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2c810aa80 0x7fd2c81ae790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2d0800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd2c8111d20 0x7fd2c81aecd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2d0800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c811cd20 0x7fd2c81a8950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2d0800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c8122630 con 0x7fd2c811cd20 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2d0800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c81224b0 con 0x7fd2c8111d20 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2d0800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c81227b0 con 0x7fd2c810aa80 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2cde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2c810aa80 0x7fd2c81ae790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2cde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2c810aa80 0x7fd2c81ae790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50410/0 (socket says 172.21.15.28:50410) 2024-05-20T20:10:41.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2cde00700 1 -- 172.21.15.28:0/12271447 learned_addr learned my addr 172.21.15.28:0/12271447 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.944+0000 7fd2ce800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c811cd20 0x7fd2c81a8950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.945+0000 7fd2ce800700 1 -- 172.21.15.28:0/12271447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2c810aa80 msgr2=0x7fd2c81ae790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.945+0000 7fd2ce800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2c810aa80 0x7fd2c81ae790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.945+0000 7fd2ce800700 1 -- 172.21.15.28:0/12271447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd2c8111d20 msgr2=0x7fd2c81aecd0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.945+0000 7fd2ce800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd2c8111d20 0x7fd2c81aecd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.945+0000 7fd2ce800700 1 -- 172.21.15.28:0/12271447 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd2c81a90b0 con 0x7fd2c811cd20 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.945+0000 7fd2ce800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c811cd20 0x7fd2c81a8950 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fd2c400cc20 tx=0x7fd2c400cf30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:41.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.946+0000 7fd2c2a00700 1 -- 172.21.15.28:0/12271447 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2c4016020 con 0x7fd2c811cd20 2024-05-20T20:10:41.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.946+0000 7fd2c2a00700 1 -- 172.21.15.28:0/12271447 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd2c4012420 con 0x7fd2c811cd20 2024-05-20T20:10:41.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.946+0000 7fd2c2a00700 1 -- 172.21.15.28:0/12271447 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2c40115a0 con 0x7fd2c811cd20 2024-05-20T20:10:41.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.946+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd2c81a93a0 con 0x7fd2c811cd20 2024-05-20T20:10:41.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.946+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd2c8112500 con 0x7fd2c811cd20 2024-05-20T20:10:41.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.954+0000 7fd2db12db80 1 Processor -- start 2024-05-20T20:10:41.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.961+0000 7fd2c2a00700 1 -- 172.21.15.28:0/12271447 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd2c4019020 con 0x7fd2c811cd20 2024-05-20T20:10:41.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.964+0000 7fd2c2a00700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd2b00b2af0 0x7fd2b00b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.965 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.964+0000 7fd2c2a00700 1 -- 172.21.15.28:0/12271447 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd2c40e2400 con 0x7fd2c811cd20 2024-05-20T20:10:41.965 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.965+0000 7fd2db12db80 1 -- start start 2024-05-20T20:10:41.966 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.966+0000 7fd2cd400700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd2b00b2af0 0x7fd2b00b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.966+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a5290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.966+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 0x558f0c39f260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.966+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c39fce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.966+0000 7fd2db12db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558f0c2fde40 con 0x558f0c3a4eb0 2024-05-20T20:10:41.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.966+0000 7fd2db12db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558f0c2ed9e0 con 0x558f0c39f880 2024-05-20T20:10:41.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.966+0000 7fd2db12db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558f0c2f1a20 con 0x558f0c3a57d0 2024-05-20T20:10:41.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.967+0000 7fd2cde00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a5290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.967+0000 7fd2cde00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a5290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51678/0 (socket says 172.21.15.28:51678) 2024-05-20T20:10:41.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.967+0000 7fd2cde00700 1 -- 172.21.15.28:0/3927318313 learned_addr learned my addr 172.21.15.28:0/3927318313 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:41.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.968+0000 7fd2ce800700 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c39fce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.968+0000 7fd2ce800700 1 -- 172.21.15.28:0/3927318313 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 msgr2=0x558f0c39f260 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:10:41.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.968+0000 7fd2ce800700 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 0x558f0c39f260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.968+0000 7fd2ce800700 1 -- 172.21.15.28:0/3927318313 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a4eb0 msgr2=0x558f0c3a5290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:41.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.968+0000 7fd2ce800700 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a5290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.969+0000 7fd2ce800700 1 -- 172.21.15.28:0/3927318313 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558f0c2b6420 con 0x558f0c39f880 2024-05-20T20:10:41.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.969+0000 7fd2cd400700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd2b00b2af0 0x7fd2b00b4fb0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fd2c811f440 tx=0x7fd2b8006c60 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:41.969 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.970+0000 7fd2ce800700 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c39fce0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fd2c40dfc10 tx=0x7fd2c40a9380 comp rx=0 tx=0).ready entity=mon.2 client_cookie=898922076e10054a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:41.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.970+0000 7fd2b7400700 1 -- 172.21.15.28:0/3927318313 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2c40adb50 con 0x558f0c39f880 2024-05-20T20:10:41.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.970+0000 7fd2b7400700 1 -- 172.21.15.28:0/3927318313 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd2c40b2600 con 0x558f0c39f880 2024-05-20T20:10:41.970 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.970+0000 7fd2b7400700 1 -- 172.21.15.28:0/3927318313 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2c40b23f0 con 0x558f0c39f880 2024-05-20T20:10:41.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.971+0000 7fd2db12db80 1 -- 172.21.15.28:0/3927318313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 msgr2=0x558f0c39fce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:41.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.971+0000 7fd2db12db80 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c39fce0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fd2c40dfc10 tx=0x7fd2c40a9380 comp rx=0 tx=0).stop 2024-05-20T20:10:41.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.972+0000 7fd2db12db80 1 -- 172.21.15.28:0/3927318313 shutdown_connections 2024-05-20T20:10:41.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.972+0000 7fd2db12db80 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c39fce0 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.972+0000 7fd2db12db80 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a5290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.972+0000 7fd2db12db80 1 --2- 172.21.15.28:0/3927318313 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 0x558f0c39f260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.972+0000 7fd2db12db80 1 -- 172.21.15.28:0/3927318313 >> 172.21.15.28:0/3927318313 conn(0x558f0c2cdbe0 msgr2=0x558f0c39a010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:41.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.972+0000 7fd2db12db80 1 -- 172.21.15.28:0/3927318313 shutdown_connections 2024-05-20T20:10:41.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.972+0000 7fd2db12db80 1 -- 172.21.15.28:0/3927318313 wait complete. 2024-05-20T20:10:41.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.973+0000 7fd2db12db80 1 -- start start 2024-05-20T20:10:41.973 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.973+0000 7fd2db12db80 1 Processor -- start 2024-05-20T20:10:41.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.974+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c3ab540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.974 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.975 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a57d0 0x558f0c3a8e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.975 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2ce800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a57d0 0x558f0c3a8e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.975 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2ce800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a57d0 0x558f0c3a8e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51694/0 (socket says 172.21.15.28:51694) 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2cd400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2ce800700 1 -- 172.21.15.28:0/2029717387 learned_addr learned my addr 172.21.15.28:0/2029717387 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558f0c1f3fa0 con 0x558f0c3a57d0 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558f0c28e760 con 0x558f0c39f880 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.975+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558f0c3ac240 con 0x558f0c3a4eb0 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2db12db80 10 client.0 _ll_register_callbacks cb 0x558f0c297c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2cd400700 1 -- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 msgr2=0x558f0c3ab540 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2cde00700 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c3ab540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2cd400700 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c3ab540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2cd400700 1 -- 172.21.15.28:0/2029717387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a57d0 msgr2=0x558f0c3a8e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:41.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2cd400700 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a57d0 0x558f0c3a8e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2cd400700 1 -- 172.21.15.28:0/2029717387 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558f0c23d8d0 con 0x558f0c3a4eb0 2024-05-20T20:10:41.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.976+0000 7fd2ce800700 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a57d0 0x558f0c3a8e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:41.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.977+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd2c40b2600 con 0x558f0c3a4eb0 2024-05-20T20:10:41.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.977+0000 7fd2cd400700 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8950 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fd2b8009e30 tx=0x7fd2b8004770 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5becc9753d8ac15b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:41.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.977+0000 7fd2abe00700 10 client.26428 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:10:41.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.978+0000 7fd2abe00700 1 -- 172.21.15.28:0/2029717387 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2b8023060 con 0x558f0c3a4eb0 2024-05-20T20:10:41.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.978+0000 7fd2abe00700 1 -- 172.21.15.28:0/2029717387 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd2b8012ad0 con 0x558f0c3a4eb0 2024-05-20T20:10:41.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.978+0000 7fd2abe00700 1 -- 172.21.15.28:0/2029717387 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd2b8012d10 con 0x558f0c3a4eb0 2024-05-20T20:10:41.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.978+0000 7fd2db12db80 10 client.26428 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:41.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.978+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x558f0c2be060 con 0x558f0c3a4eb0 2024-05-20T20:10:41.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2abe00700 1 -- 172.21.15.28:0/2029717387 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd2b8013420 con 0x558f0c3a4eb0 2024-05-20T20:10:41.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2db12db80 10 client.26428 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:41.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2db12db80 10 client.26428 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:41.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2db12db80 20 client.26428 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:41.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2db12db80 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c3b90b0 0x558f0c3bb4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:41.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2db12db80 4 client.26428 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:41.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558f0c2ffd90 con 0x558f0c3b90b0 2024-05-20T20:10:41.980 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.979+0000 7fd2cde00700 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c3b90b0 0x558f0c3bb4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:41.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.980+0000 7fd2cde00700 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c3b90b0 0x558f0c3bb4f0 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3d136c4818072868 server_cookie=fd3f6d21a931c7bc in_seq=0 out_seq=0 2024-05-20T20:10:41.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.980+0000 7fd2abe00700 10 client.26428 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:41.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.981+0000 7fd2abe00700 1 -- 172.21.15.28:0/2029717387 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558f0c2ffd90 con 0x558f0c3b90b0 2024-05-20T20:10:41.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.981+0000 7fd2abe00700 10 client.26428 handle_command_reply: tid=0 2024-05-20T20:10:41.981 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.981+0000 7fd2db12db80 1 client.26428 shutdown 2024-05-20T20:10:41.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.981+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 msgr2=0x558f0c3a8950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:41.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.981+0000 7fd2db12db80 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8950 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fd2b8009e30 tx=0x7fd2b8004770 comp rx=0 tx=0).stop 2024-05-20T20:10:41.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 shutdown_connections 2024-05-20T20:10:41.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c3b90b0 0x558f0c3bb4f0 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c3a57d0 0x558f0c3a8e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c39f880 0x558f0c3ab540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 --2- 172.21.15.28:0/2029717387 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8950 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:41.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 >> 172.21.15.28:0/2029717387 conn(0x558f0c2cdbe0 msgr2=0x558f0c39e250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:41.983 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2abe00700 0 client.26428 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:41.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 shutdown_connections 2024-05-20T20:10:41.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 1 -- 172.21.15.28:0/2029717387 wait complete. 2024-05-20T20:10:41.984 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:41.982+0000 7fd2db12db80 20 client.26428 trim_cache size 0 max 16384 2024-05-20T20:10:42.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.024+0000 7fd2db12db80 1 Processor -- start 2024-05-20T20:10:42.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.025+0000 7fd2db12db80 1 -- start start 2024-05-20T20:10:42.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.025+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c39f260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.025+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.025+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 0x558f0c39a0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.025+0000 7fd2db12db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c8119ac0 con 0x558f0c39f880 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.025+0000 7fd2db12db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c81c6f70 con 0x558f0c3a4eb0 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.025+0000 7fd2db12db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c811a190 con 0x558f0c3a57d0 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.026+0000 7fd2cd400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.026+0000 7fd2cd400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37272/0 (socket says 172.21.15.28:37272) 2024-05-20T20:10:42.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.026+0000 7fd2cd400700 1 -- 172.21.15.28:0/298494089 learned_addr learned my addr 172.21.15.28:0/298494089 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:42.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.026+0000 7fd2cde00700 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c39f260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.027+0000 7fd2cd400700 1 -- 172.21.15.28:0/298494089 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 msgr2=0x558f0c39a0f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:10:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.027+0000 7fd2cd400700 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 0x558f0c39a0f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.027+0000 7fd2cd400700 1 -- 172.21.15.28:0/298494089 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 msgr2=0x558f0c39f260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.027+0000 7fd2cd400700 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c39f260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.027+0000 7fd2cd400700 1 -- 172.21.15.28:0/298494089 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558f0c3a55d0 con 0x558f0c3a4eb0 2024-05-20T20:10:42.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.030+0000 7fd2cd400700 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8c80 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7fd2b8009ee0 tx=0x7fd2b801a000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d37ae0a0027a37aa server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:42.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.031+0000 7fd2b6a00700 1 -- 172.21.15.28:0/298494089 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2b8013420 con 0x558f0c3a4eb0 2024-05-20T20:10:42.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.032+0000 7fd2b6a00700 1 -- 172.21.15.28:0/298494089 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd2c40b2600 con 0x558f0c3a4eb0 2024-05-20T20:10:42.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.032+0000 7fd2b6a00700 1 -- 172.21.15.28:0/298494089 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2b8013420 con 0x558f0c3a4eb0 2024-05-20T20:10:42.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.033+0000 7fd2db12db80 1 -- 172.21.15.28:0/298494089 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a4eb0 msgr2=0x558f0c3a8c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:42.033 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.033+0000 7fd2db12db80 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8c80 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7fd2b8009ee0 tx=0x7fd2b801a000 comp rx=0 tx=0).stop 2024-05-20T20:10:42.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.034+0000 7fd2db12db80 1 -- 172.21.15.28:0/298494089 shutdown_connections 2024-05-20T20:10:42.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.034+0000 7fd2db12db80 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a57d0 0x558f0c39a0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.034+0000 7fd2db12db80 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c39f260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.034+0000 7fd2db12db80 1 --2- 172.21.15.28:0/298494089 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a4eb0 0x558f0c3a8c80 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.035+0000 7fd2db12db80 1 -- 172.21.15.28:0/298494089 >> 172.21.15.28:0/298494089 conn(0x558f0c2cdbe0 msgr2=0x558f0c3999e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:42.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.035+0000 7fd2db12db80 1 -- 172.21.15.28:0/298494089 shutdown_connections 2024-05-20T20:10:42.035 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.035+0000 7fd2db12db80 1 -- 172.21.15.28:0/298494089 wait complete. 2024-05-20T20:10:42.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.036+0000 7fd2db12db80 1 -- start start 2024-05-20T20:10:42.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.037+0000 7fd2db12db80 1 Processor -- start 2024-05-20T20:10:42.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.038+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c3c09c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:42.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.039+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3c0f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:42.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.039+0000 7fd2db12db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a57d0 0x558f0c3c1440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.039+0000 7fd2db12db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c81c68a0 con 0x558f0c39f880 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2db12db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c811a860 con 0x558f0c3a57d0 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2db12db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd2c8117a60 con 0x558f0c3a4eb0 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2db12db80 10 client.0 _ll_register_callbacks cb 0x558f0c297c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2cd400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3c0f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2cde00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c3c09c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2cde00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c3c09c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51704/0 (socket says 172.21.15.28:51704) 2024-05-20T20:10:42.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2cde00700 1 -- 172.21.15.28:0/1677071512 learned_addr learned my addr 172.21.15.28:0/1677071512 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.040+0000 7fd2ce800700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a57d0 0x558f0c3c1440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.041+0000 7fd2cde00700 1 -- 172.21.15.28:0/1677071512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 msgr2=0x558f0c3c0f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.041+0000 7fd2cde00700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3c0f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.041+0000 7fd2cde00700 1 -- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a57d0 msgr2=0x558f0c3c1440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.041+0000 7fd2cde00700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a57d0 0x558f0c3c1440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.041+0000 7fd2cde00700 1 -- 172.21.15.28:0/1677071512 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558f0c422dc0 con 0x558f0c39f880 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.041+0000 7fd2cd400700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3c0f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:42.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.042+0000 7fd2ce800700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a57d0 0x558f0c3c1440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:10:42.045 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.044+0000 7fd2abe00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:10:42.045 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.044+0000 7fd2cde00700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c3c09c0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x558f0c3bbd30 tx=0x7fd2bc003860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=31386be9998863c8 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:42.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.047+0000 7fd2abe00700 1 -- 172.21.15.28:0/1677071512 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd2bc0052c0 con 0x558f0c39f880 2024-05-20T20:10:42.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.047+0000 7fd2abe00700 1 -- 172.21.15.28:0/1677071512 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd2bc005460 con 0x558f0c39f880 2024-05-20T20:10:42.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.048+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd2c40b2600 con 0x558f0c39f880 2024-05-20T20:10:42.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.048+0000 7fd2abe00700 1 -- 172.21.15.28:0/1677071512 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd2bc035050 con 0x558f0c39f880 2024-05-20T20:10:42.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.049+0000 7fd2db12db80 10 client.16281 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:42.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.049+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558f0c4240f0 con 0x558f0c39f880 2024-05-20T20:10:42.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2abe00700 1 -- 172.21.15.28:0/1677071512 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd2c40b2600 con 0x558f0c39f880 2024-05-20T20:10:42.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2abe00700 1 -- 172.21.15.28:0/1677071512 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd2bc00f8b0 con 0x558f0c39f880 2024-05-20T20:10:42.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2db12db80 10 client.16281 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:42.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2db12db80 10 client.16281 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:42.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2db12db80 20 client.16281 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:42.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2db12db80 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c424800 0x558f0c426c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:42.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2db12db80 4 client.16281 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:10:42.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.050+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x558f0c2f9570 con 0x558f0c424800 2024-05-20T20:10:42.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.053+0000 7fd2cd400700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c424800 0x558f0c426c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:42.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.055+0000 7fd2cd400700 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c424800 0x558f0c426c20 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=96faa9f969723508 server_cookie=9eb188fe386e4364 in_seq=0 out_seq=0 2024-05-20T20:10:42.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.055+0000 7fd2abe00700 10 client.16281 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:42.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.056+0000 7fd2abe00700 1 -- 172.21.15.28:0/1677071512 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x558f0c2f9570 con 0x558f0c424800 2024-05-20T20:10:42.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.056+0000 7fd2abe00700 10 client.16281 handle_command_reply: tid=0 2024-05-20T20:10:42.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.056+0000 7fd2db12db80 1 client.16281 shutdown 2024-05-20T20:10:42.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 msgr2=0x558f0c3c09c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:42.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c3c09c0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x558f0c3bbd30 tx=0x7fd2bc003860 comp rx=0 tx=0).stop 2024-05-20T20:10:42.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 shutdown_connections 2024-05-20T20:10:42.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558f0c424800 0x558f0c426c20 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558f0c3a57d0 0x558f0c3c1440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558f0c39f880 0x558f0c3c09c0 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 --2- 172.21.15.28:0/1677071512 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558f0c3a4eb0 0x558f0c3c0f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.057+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 >> 172.21.15.28:0/1677071512 conn(0x558f0c2cdbe0 msgr2=0x558f0c3a3b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:42.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.058+0000 7fd2abe00700 0 client.16281 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:42.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.058+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 shutdown_connections 2024-05-20T20:10:42.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.058+0000 7fd2db12db80 1 -- 172.21.15.28:0/1677071512 wait complete. 2024-05-20T20:10:42.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.059+0000 7fd2db12db80 20 client.16281 trim_cache size 0 max 16384 2024-05-20T20:10:42.059 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:10:42.060 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:10:42.060 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:10:42.060 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:10:42.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.060+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd2b00b2af0 msgr2=0x7fd2b00b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:42.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.060+0000 7fd2d0800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd2b00b2af0 0x7fd2b00b4fb0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7fd2c811f440 tx=0x7fd2b8006c60 comp rx=0 tx=0).stop 2024-05-20T20:10:42.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.060+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c811cd20 msgr2=0x7fd2c81a8950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:42.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.060+0000 7fd2d0800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c811cd20 0x7fd2c81a8950 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fd2c400cc20 tx=0x7fd2c400cf30 comp rx=0 tx=0).stop 2024-05-20T20:10:42.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.060+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 shutdown_connections 2024-05-20T20:10:42.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.061+0000 7fd2d0800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd2c811cd20 0x7fd2c81a8950 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.061+0000 7fd2d0800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2c810aa80 0x7fd2c81ae790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.061+0000 7fd2d0800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd2b00b2af0 0x7fd2b00b4fb0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.061+0000 7fd2d0800700 1 --2- 172.21.15.28:0/12271447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd2c8111d20 0x7fd2c81aecd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:42.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.061+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 >> 172.21.15.28:0/12271447 conn(0x7fd2c806e6b0 msgr2=0x7fd2c811ad10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:42.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.062+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 shutdown_connections 2024-05-20T20:10:42.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:42.062+0000 7fd2d0800700 1 -- 172.21.15.28:0/12271447 wait complete. 2024-05-20T20:10:43.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:42 smithi028 ceph-mon[31178]: pgmap v155: 97 pgs: 97 active+clean; 25 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 240 MiB/s wr, 1.22k op/s 2024-05-20T20:10:43.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:42 smithi154 ceph-mon[36314]: pgmap v155: 97 pgs: 97 active+clean; 25 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 240 MiB/s wr, 1.22k op/s 2024-05-20T20:10:43.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:42 smithi033 ceph-mon[36179]: pgmap v155: 97 pgs: 97 active+clean; 25 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 240 MiB/s wr, 1.22k op/s 2024-05-20T20:10:44.137 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:10:44.138 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:10:44.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.836+0000 7f907ba00700 1 -- 172.21.15.28:0/1743502802 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741037f0 msgr2=0x7f9074103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.836+0000 7f907ba00700 1 --2- 172.21.15.28:0/1743502802 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741037f0 0x7f9074103bd0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f90680098f0 tx=0x7f9068009c00 comp rx=0 tx=0).stop 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.836+0000 7f907ba00700 1 -- 172.21.15.28:0/1743502802 shutdown_connections 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.836+0000 7f907ba00700 1 --2- 172.21.15.28:0/1743502802 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90741088f0 0x7f9074110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.836+0000 7f907ba00700 1 --2- 172.21.15.28:0/1743502802 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741037f0 0x7f9074103bd0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.836+0000 7f907ba00700 1 --2- 172.21.15.28:0/1743502802 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741041a0 0x7f90741081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.836+0000 7f907ba00700 1 -- 172.21.15.28:0/1743502802 >> 172.21.15.28:0/1743502802 conn(0x7f907407a6a0 msgr2=0x7f907407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.837+0000 7f907ba00700 1 -- 172.21.15.28:0/1743502802 shutdown_connections 2024-05-20T20:10:44.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.837+0000 7f907ba00700 1 -- 172.21.15.28:0/1743502802 wait complete. 2024-05-20T20:10:44.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 Processor -- start 2024-05-20T20:10:44.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 -- start start 2024-05-20T20:10:44.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741037f0 0x7f907419bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:44.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741041a0 0x7f907419c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:44.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90741088f0 0x7f90741a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:44.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f90741136e0 con 0x7f90741037f0 2024-05-20T20:10:44.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9074113560 con 0x7f90741088f0 2024-05-20T20:10:44.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.838+0000 7f907ba00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9074113860 con 0x7f90741041a0 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9079000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741037f0 0x7f907419bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9079000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741037f0 0x7f907419bdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:51716/0 (socket says 172.21.15.28:51716) 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9079000700 1 -- 172.21.15.28:0/1926507591 learned_addr learned my addr 172.21.15.28:0/1926507591 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9073e00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741041a0 0x7f907419c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9079a00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90741088f0 0x7f90741a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9073e00700 1 -- 172.21.15.28:0/1926507591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90741088f0 msgr2=0x7f90741a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9073e00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90741088f0 0x7f90741a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9073e00700 1 -- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741037f0 msgr2=0x7f907419bdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9073e00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741037f0 0x7f907419bdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:44.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9073e00700 1 -- 172.21.15.28:0/1926507591 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f90741a0de0 con 0x7f90741041a0 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9079000700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741037f0 0x7f907419bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.839+0000 7f9073e00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741041a0 0x7f907419c310 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f906000e9c0 tx=0x7f906000ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.840+0000 7f9071600700 1 -- 172.21.15.28:0/1926507591 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f906000c9c0 con 0x7f90741041a0 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.840+0000 7f9071600700 1 -- 172.21.15.28:0/1926507591 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f906000cb60 con 0x7f90741041a0 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.840+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f90741a10d0 con 0x7f90741041a0 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.840+0000 7f9071600700 1 -- 172.21.15.28:0/1926507591 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9060018420 con 0x7f90741041a0 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.840+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f90741a8950 con 0x7f90741041a0 2024-05-20T20:10:44.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.841+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f907406a230 con 0x7f90741041a0 2024-05-20T20:10:44.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.843+0000 7f9071600700 1 -- 172.21.15.28:0/1926507591 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f906004ab30 con 0x7f90741041a0 2024-05-20T20:10:44.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.845+0000 7f9071600700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f905c0b2a70 0x7f905c0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:44.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.845+0000 7f9071600700 1 -- 172.21.15.28:0/1926507591 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f90600e26e0 con 0x7f90741041a0 2024-05-20T20:10:44.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.845+0000 7f9079000700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f905c0b2a70 0x7f905c0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:44.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.846+0000 7f9079000700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f905c0b2a70 0x7f905c0b4f30 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f9068009680 tx=0x7f9068019040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:44.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:44.847+0000 7f9071600700 1 -- 172.21.15.28:0/1926507591 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f90600aa070 con 0x7f90741041a0 2024-05-20T20:10:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:45.025 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:44 smithi028 ceph-mon[31178]: pgmap v156: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 332 MiB/s wr, 1.40k op/s 2024-05-20T20:10:45.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.133+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f90741a8bb0 con 0x7f90741041a0 2024-05-20T20:10:45.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.134+0000 7f9071600700 1 -- 172.21.15.28:0/1926507591 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f906004ae20 con 0x7f90741041a0 2024-05-20T20:10:45.134 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:10:45.134 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:10:45.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.136+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f905c0b2a70 msgr2=0x7f905c0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:45.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.136+0000 7f907ba00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f905c0b2a70 0x7f905c0b4f30 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f9068009680 tx=0x7f9068019040 comp rx=0 tx=0).stop 2024-05-20T20:10:45.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.136+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741041a0 msgr2=0x7f907419c310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:45.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.136+0000 7f907ba00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741041a0 0x7f907419c310 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f906000e9c0 tx=0x7f906000ecd0 comp rx=0 tx=0).stop 2024-05-20T20:10:45.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 shutdown_connections 2024-05-20T20:10:45.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f905c0b2a70 0x7f905c0b4f30 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:45.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f90741088f0 0x7f90741a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:45.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f90741037f0 0x7f907419bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:45.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 --2- 172.21.15.28:0/1926507591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f90741041a0 0x7f907419c310 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:45.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 >> 172.21.15.28:0/1926507591 conn(0x7f907407a6a0 msgr2=0x7f9074101600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:45.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 shutdown_connections 2024-05-20T20:10:45.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:45.137+0000 7f907ba00700 1 -- 172.21.15.28:0/1926507591 wait complete. 2024-05-20T20:10:45.139 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:10:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:44 smithi154 ceph-mon[36314]: pgmap v156: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 332 MiB/s wr, 1.40k op/s 2024-05-20T20:10:45.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:44 smithi033 ceph-mon[36179]: pgmap v156: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 332 MiB/s wr, 1.40k op/s 2024-05-20T20:10:46.276 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:45 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1926507591' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:46.284 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:45 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1926507591' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:46.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:45 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1926507591' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:46.808 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id 65623019 --release --await' 2024-05-20T20:10:46.809 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id 65623019 --release --await 2024-05-20T20:10:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:46 smithi028 ceph-mon[31178]: pgmap v157: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 299 MiB/s wr, 1.06k op/s 2024-05-20T20:10:47.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.367+0000 7fc8eaa00700 1 -- 172.21.15.28:0/1069963789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e41096a0 msgr2=0x7fc8e4109a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:47.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.367+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/1069963789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e41096a0 0x7fc8e4109a80 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fc8d4009990 tx=0x7fc8d4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.367+0000 7fc8eaa00700 1 -- 172.21.15.28:0/1069963789 shutdown_connections 2024-05-20T20:10:47.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.367+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/1069963789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc8e4103f50 0x7fc8e4110430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.367+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/1069963789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e41096a0 0x7fc8e4109a80 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.367+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/1069963789 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc8e4103590 0x7fc8e4103a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.367+0000 7fc8eaa00700 1 -- 172.21.15.28:0/1069963789 >> 172.21.15.28:0/1069963789 conn(0x7fc8e407a6f0 msgr2=0x7fc8e407ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:47.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.368+0000 7fc8eaa00700 1 -- 172.21.15.28:0/1069963789 shutdown_connections 2024-05-20T20:10:47.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.368+0000 7fc8eaa00700 1 -- 172.21.15.28:0/1069963789 wait complete. 2024-05-20T20:10:47.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.369+0000 7fc8eaa00700 1 Processor -- start 2024-05-20T20:10:47.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.369+0000 7fc8eaa00700 1 -- start start 2024-05-20T20:10:47.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8eaa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e4103590 0x7fc8e419bd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:47.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8eaa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc8e4103f50 0x7fc8e419c2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:47.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8eaa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc8e41096a0 0x7fc8e41a0650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:47.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8eaa00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc8e4113710 con 0x7fc8e4103f50 2024-05-20T20:10:47.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8eaa00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc8e4113590 con 0x7fc8e41096a0 2024-05-20T20:10:47.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8eaa00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc8e4113890 con 0x7fc8e4103590 2024-05-20T20:10:47.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8e3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e4103590 0x7fc8e419bd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:47.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8e8a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc8e41096a0 0x7fc8e41a0650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:47.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8e3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc8e4103f50 0x7fc8e419c2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:47.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8e3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e4103590 0x7fc8e419bd80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:48948/0 (socket says 172.21.15.28:48948) 2024-05-20T20:10:47.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.370+0000 7fc8e3e00700 1 -- 172.21.15.28:0/3002795079 learned_addr learned my addr 172.21.15.28:0/3002795079 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:47.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.371+0000 7fc8e3e00700 1 -- 172.21.15.28:0/3002795079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc8e41096a0 msgr2=0x7fc8e41a0650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.371+0000 7fc8e3e00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc8e41096a0 0x7fc8e41a0650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.371+0000 7fc8e3e00700 1 -- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc8e4103f50 msgr2=0x7fc8e419c2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.371+0000 7fc8e3e00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc8e4103f50 0x7fc8e419c2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.371+0000 7fc8e3e00700 1 -- 172.21.15.28:0/3002795079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc8e41a0d90 con 0x7fc8e4103590 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.371+0000 7fc8e3e00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e4103590 0x7fc8e419bd80 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fc8d40057f0 tx=0x7fc8d400bfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.372+0000 7fc8e0c00700 1 -- 172.21.15.28:0/3002795079 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc8d4026070 con 0x7fc8e4103590 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.372+0000 7fc8e0c00700 1 -- 172.21.15.28:0/3002795079 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc8d4021410 con 0x7fc8e4103590 2024-05-20T20:10:47.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.372+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc8e41a1020 con 0x7fc8e4103590 2024-05-20T20:10:47.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.372+0000 7fc8e0c00700 1 -- 172.21.15.28:0/3002795079 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc8d4017650 con 0x7fc8e4103590 2024-05-20T20:10:47.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.372+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc8e41a8900 con 0x7fc8e4103590 2024-05-20T20:10:47.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.373+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc8e406a230 con 0x7fc8e4103590 2024-05-20T20:10:47.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:46 smithi154 ceph-mon[36314]: pgmap v157: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 299 MiB/s wr, 1.06k op/s 2024-05-20T20:10:47.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.375+0000 7fc8e0c00700 1 -- 172.21.15.28:0/3002795079 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc8d4029070 con 0x7fc8e4103590 2024-05-20T20:10:47.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:46 smithi033 ceph-mon[36179]: pgmap v157: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 299 MiB/s wr, 1.06k op/s 2024-05-20T20:10:47.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.378+0000 7fc8e0c00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8c80b2a70 0x7fc8c80b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:47.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.378+0000 7fc8e0c00700 1 -- 172.21.15.28:0/3002795079 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fc8d40e3230 con 0x7fc8e4103590 2024-05-20T20:10:47.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.378+0000 7fc8e3400700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8c80b2a70 0x7fc8c80b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:47.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.379+0000 7fc8e3400700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8c80b2a70 0x7fc8c80b4f30 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fc8d0009680 tx=0x7fc8d0009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:47.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.381+0000 7fc8e0c00700 1 -- 172.21.15.28:0/3002795079 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fc8d4013070 con 0x7fc8e4103590 2024-05-20T20:10:47.426 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:47.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.713+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fc8e41a8ad0 con 0x7fc8e4103590 2024-05-20T20:10:47.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.714+0000 7fc8e0c00700 1 -- 172.21.15.28:0/3002795079 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fc8d40aace0 con 0x7fc8e4103590 2024-05-20T20:10:47.714 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:10:47.715 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:10:47.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.717+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8c80b2a70 msgr2=0x7fc8c80b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:47.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.717+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8c80b2a70 0x7fc8c80b4f30 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fc8d0009680 tx=0x7fc8d0009260 comp rx=0 tx=0).stop 2024-05-20T20:10:47.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.717+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e4103590 msgr2=0x7fc8e419bd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:47.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.717+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e4103590 0x7fc8e419bd80 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fc8d40057f0 tx=0x7fc8d400bfd0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 shutdown_connections 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8c80b2a70 0x7fc8c80b4f30 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc8e41096a0 0x7fc8e41a0650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc8e4103590 0x7fc8e419bd80 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 --2- 172.21.15.28:0/3002795079 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc8e4103f50 0x7fc8e419c2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 >> 172.21.15.28:0/3002795079 conn(0x7fc8e407a6f0 msgr2=0x7fc8e410ea10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 shutdown_connections 2024-05-20T20:10:47.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:47.718+0000 7fc8eaa00700 1 -- 172.21.15.28:0/3002795079 wait complete. 2024-05-20T20:10:47.720 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:10:48.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:47 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3002795079' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:48.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:47 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3002795079' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:48.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:47 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3002795079' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:10:49.062 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:10:49.218 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:48 smithi028 ceph-mon[31178]: pgmap v158: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 260 MiB/s wr, 821 op/s 2024-05-20T20:10:49.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:48 smithi154 ceph-mon[36314]: pgmap v158: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 260 MiB/s wr, 821 op/s 2024-05-20T20:10:49.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:48 smithi033 ceph-mon[36179]: pgmap v158: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 260 MiB/s wr, 821 op/s 2024-05-20T20:10:49.875 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:50.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.690+0000 7fb414000700 1 -- 172.21.15.28:0/1803331429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c10fff0 msgr2=0x7fb40c1123e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.690+0000 7fb414000700 1 --2- 172.21.15.28:0/1803331429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c10fff0 0x7fb40c1123e0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7fb404009990 tx=0x7fb404009ca0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.691+0000 7fb414000700 1 -- 172.21.15.28:0/1803331429 shutdown_connections 2024-05-20T20:10:50.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.691+0000 7fb414000700 1 --2- 172.21.15.28:0/1803331429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c10fff0 0x7fb40c1123e0 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.691+0000 7fb414000700 1 --2- 172.21.15.28:0/1803331429 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb40c06b250 0x7fb40c06b630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.691+0000 7fb414000700 1 --2- 172.21.15.28:0/1803331429 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06bb70 0x7fb40c10fab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.691+0000 7fb414000700 1 -- 172.21.15.28:0/1803331429 >> 172.21.15.28:0/1803331429 conn(0x7fb40c0793f0 msgr2=0x7fb40c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:50.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.691+0000 7fb414000700 1 -- 172.21.15.28:0/1803331429 shutdown_connections 2024-05-20T20:10:50.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.691+0000 7fb414000700 1 -- 172.21.15.28:0/1803331429 wait complete. 2024-05-20T20:10:50.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.692+0000 7fb414000700 1 Processor -- start 2024-05-20T20:10:50.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.692+0000 7fb414000700 1 -- start start 2024-05-20T20:10:50.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.692+0000 7fb414000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06b250 0x7fb40c19bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.692+0000 7fb414000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c06bb70 0x7fb40c19c410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.692+0000 7fb414000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb40c10fff0 0x7fb40c1a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb414000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c115960 con 0x7fb40c06b250 2024-05-20T20:10:50.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb414000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c1157e0 con 0x7fb40c06bb70 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb414000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c115ae0 con 0x7fb40c10fff0 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb411600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06b250 0x7fb40c19bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb411600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06b250 0x7fb40c19bed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35170/0 (socket says 172.21.15.28:35170) 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb411600700 1 -- 172.21.15.28:0/3945998468 learned_addr learned my addr 172.21.15.28:0/3945998468 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb410c00700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c06bb70 0x7fb40c19c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb412000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb40c10fff0 0x7fb40c1a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb410c00700 1 -- 172.21.15.28:0/3945998468 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb40c10fff0 msgr2=0x7fb40c1a0750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb410c00700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb40c10fff0 0x7fb40c1a0750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb410c00700 1 -- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06b250 msgr2=0x7fb40c19bed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb410c00700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06b250 0x7fb40c19bed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb410c00700 1 -- 172.21.15.28:0/3945998468 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb40c1a0e90 con 0x7fb40c06bb70 2024-05-20T20:10:50.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.693+0000 7fb411600700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06b250 0x7fb40c19bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:50.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.694+0000 7fb410c00700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c06bb70 0x7fb40c19c410 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7fb3f800e9f0 tx=0x7fb3f800ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:50.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.694+0000 7fb40a000700 1 -- 172.21.15.28:0/3945998468 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3f800cb70 con 0x7fb40c06bb70 2024-05-20T20:10:50.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.694+0000 7fb40a000700 1 -- 172.21.15.28:0/3945998468 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3f800ee90 con 0x7fb40c06bb70 2024-05-20T20:10:50.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.694+0000 7fb40a000700 1 -- 172.21.15.28:0/3945998468 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3f8003d40 con 0x7fb40c06bb70 2024-05-20T20:10:50.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.694+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb40c1a1180 con 0x7fb40c06bb70 2024-05-20T20:10:50.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.694+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb40c1a8a00 con 0x7fb40c06bb70 2024-05-20T20:10:50.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.698+0000 7fb40a000700 1 -- 172.21.15.28:0/3945998468 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb3f8003ee0 con 0x7fb40c06bb70 2024-05-20T20:10:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.699+0000 7fb41e9e4b80 1 Processor -- start 2024-05-20T20:10:50.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.699+0000 7fb41e9e4b80 1 -- start start 2024-05-20T20:10:50.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d7ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb40a000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3f40b29f0 0x7fb3f40b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d8530 0x55a7a32d1fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32d2a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a7a322ff40 con 0x55a7a32d7c10 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a7a321fae0 con 0x55a7a32d8530 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a7a3223b20 con 0x55a7a32d25f0 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb40a000700 1 -- 172.21.15.28:0/3945998468 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb3f80e2700 con 0x7fb40c06bb70 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb412000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d7ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb412000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d7ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35172/0 (socket says 172.21.15.28:35172) 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb412000700 1 -- 172.21.15.28:0/658841164 learned_addr learned my addr 172.21.15.28:0/658841164 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb410c00700 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d8530 0x55a7a32d1fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb412000700 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32d2a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.700+0000 7fb411600700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3f40b29f0 0x7fb3f40b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb410c00700 1 -- 172.21.15.28:0/658841164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 msgr2=0x55a7a32d2a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb410c00700 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32d2a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb410c00700 1 -- 172.21.15.28:0/658841164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 msgr2=0x55a7a32d7ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb410c00700 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d7ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb410c00700 1 -- 172.21.15.28:0/658841164 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a7a31e8520 con 0x55a7a32d8530 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb410c00700 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d8530 0x55a7a32d1fd0 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7fb3f80dfe00 tx=0x7fb3f80a9230 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b1cdd6b33a60adf6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb3fea00700 1 -- 172.21.15.28:0/658841164 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3f80a9990 con 0x55a7a32d8530 2024-05-20T20:10:50.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb3fea00700 1 -- 172.21.15.28:0/658841164 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3f80add40 con 0x55a7a32d8530 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.701+0000 7fb3fea00700 1 -- 172.21.15.28:0/658841164 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3f80ce540 con 0x55a7a32d8530 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/658841164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d8530 msgr2=0x55a7a32d1fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d8530 0x55a7a32d1fd0 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7fb3f80dfe00 tx=0x7fb3f80a9230 comp rx=0 tx=0).stop 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb411600700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3f40b29f0 0x7fb3f40b4eb0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fb400009c10 tx=0x7fb40000b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/658841164 shutdown_connections 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32d2a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d7ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/658841164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d8530 0x55a7a32d1fd0 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/658841164 >> 172.21.15.28:0/658841164 conn(0x55a7a31ffce0 msgr2=0x55a7a32ccf00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:50.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/658841164 shutdown_connections 2024-05-20T20:10:50.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.702+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/658841164 wait complete. 2024-05-20T20:10:50.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.703+0000 7fb41e9e4b80 1 -- start start 2024-05-20T20:10:50.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.703+0000 7fb41e9e4b80 1 Processor -- start 2024-05-20T20:10:50.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32de250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32db5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32dbb30 0x55a7a32dbfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a7a31260a0 con 0x55a7a32d7c10 2024-05-20T20:10:50.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a7a31c0860 con 0x55a7a32dbb30 2024-05-20T20:10:50.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a7a32defa0 con 0x55a7a32d25f0 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb41e9e4b80 10 client.0 _ll_register_callbacks cb 0x55a7a31c9d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb412000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32dbb30 0x55a7a32dbfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb411600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32de250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb412000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32dbb30 0x55a7a32dbfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42128/0 (socket says 172.21.15.28:42128) 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb412000700 1 -- 172.21.15.28:0/475762375 learned_addr learned my addr 172.21.15.28:0/475762375 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb410c00700 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32db5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb412000700 1 -- 172.21.15.28:0/475762375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 msgr2=0x55a7a32de250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.704+0000 7fb412000700 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32de250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb412000700 1 -- 172.21.15.28:0/475762375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 msgr2=0x55a7a32db5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb412000700 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32db5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb412000700 1 -- 172.21.15.28:0/475762375 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a7a316f9d0 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb412000700 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32dbb30 0x55a7a32dbfb0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7fb404005800 tx=0x7fb404005fb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ffab509691c834a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb3f3400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb3f3400700 1 -- 172.21.15.28:0/475762375 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb40402c060 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb3f3400700 1 -- 172.21.15.28:0/475762375 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb404005270 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3f80add40 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.705+0000 7fb3f3400700 1 -- 172.21.15.28:0/475762375 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb404005470 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 10 client.26525 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a7a31f0160 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb3f3400700 1 -- 172.21.15.28:0/475762375 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3f80add40 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb3f3400700 1 -- 172.21.15.28:0/475762375 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb40400f620 con 0x55a7a32dbb30 2024-05-20T20:10:50.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 10 client.26525 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 10 client.26525 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 20 client.26525 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a32ebf10 0x55a7a32ee300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 4 client.26525 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.706+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a7a3231e90 con 0x55a7a32ebf10 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.707+0000 7fb411600700 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a32ebf10 0x55a7a32ee300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.707+0000 7fb411600700 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a32ebf10 0x55a7a32ee300 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=698612ac1c511e34 server_cookie=b4e1598c99f98ed7 in_seq=0 out_seq=0 2024-05-20T20:10:50.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.707+0000 7fb3f3400700 10 client.26525 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:50.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.708+0000 7fb3f3400700 1 -- 172.21.15.28:0/475762375 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a7a3231e90 con 0x55a7a32ebf10 2024-05-20T20:10:50.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.708+0000 7fb3f3400700 10 client.26525 handle_command_reply: tid=0 2024-05-20T20:10:50.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.708+0000 7fb41e9e4b80 1 client.26525 shutdown 2024-05-20T20:10:50.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32dbb30 msgr2=0x55a7a32dbfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32dbb30 0x55a7a32dbfb0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7fb404005800 tx=0x7fb404005fb0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 shutdown_connections 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a32ebf10 0x55a7a32ee300 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32dbb30 0x55a7a32dbfb0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32d25f0 0x55a7a32de250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/475762375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32db5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb3f3400700 0 client.26525 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 >> 172.21.15.28:0/475762375 conn(0x55a7a31ffce0 msgr2=0x55a7a32cc430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 shutdown_connections 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/475762375 wait complete. 2024-05-20T20:10:50.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.709+0000 7fb41e9e4b80 20 client.26525 trim_cache size 0 max 16384 2024-05-20T20:10:50.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 Processor -- start 2024-05-20T20:10:50.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 -- start start 2024-05-20T20:10:50.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32d1fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d91b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32da610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c1aff40 con 0x55a7a32d7c10 2024-05-20T20:10:50.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c1b1a80 con 0x55a7a32d25f0 2024-05-20T20:10:50.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c1b0610 con 0x55a7a32dbb30 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb410c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d91b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb410c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d91b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35192/0 (socket says 172.21.15.28:35192) 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb410c00700 1 -- 172.21.15.28:0/1859141257 learned_addr learned my addr 172.21.15.28:0/1859141257 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.732+0000 7fb411600700 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32d1fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb412000700 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32da610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb410c00700 1 -- 172.21.15.28:0/1859141257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 msgr2=0x55a7a32da610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb410c00700 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32da610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb410c00700 1 -- 172.21.15.28:0/1859141257 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 msgr2=0x55a7a32d1fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb410c00700 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32d1fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb410c00700 1 -- 172.21.15.28:0/1859141257 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a7a32ce680 con 0x55a7a32d7c10 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb411600700 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32d1fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.733+0000 7fb410c00700 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d91b0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fb3f8000c00 tx=0x7fb3f80a9820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b2e38edda7e56936 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb3fe000700 1 -- 172.21.15.28:0/1859141257 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3f80cae60 con 0x55a7a32d7c10 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb3fe000700 1 -- 172.21.15.28:0/1859141257 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3f80add40 con 0x55a7a32d7c10 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb3fe000700 1 -- 172.21.15.28:0/1859141257 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3f80e0080 con 0x55a7a32d7c10 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1859141257 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 msgr2=0x55a7a32d91b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d91b0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fb3f8000c00 tx=0x7fb3f80a9820 comp rx=0 tx=0).stop 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1859141257 shutdown_connections 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32da610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32d1fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1859141257 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32d91b0 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.734+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1859141257 >> 172.21.15.28:0/1859141257 conn(0x55a7a31ffce0 msgr2=0x55a7a32d9f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.735+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1859141257 shutdown_connections 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.735+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1859141257 wait complete. 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.735+0000 7fb41e9e4b80 1 -- start start 2024-05-20T20:10:50.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.735+0000 7fb41e9e4b80 1 Processor -- start 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32e1200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32e1740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb41e9e4b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32e1c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c1b13b0 con 0x55a7a32d7c10 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c1b0ce0 con 0x55a7a32d25f0 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb41e9e4b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb40c1adee0 con 0x55a7a32dbb30 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb41e9e4b80 10 client.0 _ll_register_callbacks cb 0x55a7a31c9d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb411600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32e1200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb410c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32e1740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb411600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32e1200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42150/0 (socket says 172.21.15.28:42150) 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb411600700 1 -- 172.21.15.28:0/1549424601 learned_addr learned my addr 172.21.15.28:0/1549424601 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:50.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.736+0000 7fb412000700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32e1c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb412000700 1 -- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 msgr2=0x55a7a32e1200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb412000700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32e1200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb412000700 1 -- 172.21.15.28:0/1549424601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 msgr2=0x55a7a32e1740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb412000700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32e1740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb412000700 1 -- 172.21.15.28:0/1549424601 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a7a32d2c20 con 0x55a7a32dbb30 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb411600700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32e1200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb410c00700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32e1740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb412000700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32e1c80 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fb4040098e0 tx=0x7fb4040038b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=353305197c5c3fbd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:50.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.737+0000 7fb3f3400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:10:50.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.738+0000 7fb3f3400700 1 -- 172.21.15.28:0/1549424601 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb40403d060 con 0x55a7a32dbb30 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.738+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3f80add40 con 0x55a7a32dbb30 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.738+0000 7fb3f3400700 1 -- 172.21.15.28:0/1549424601 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fb404003ca0 con 0x55a7a32dbb30 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.738+0000 7fb3f3400700 1 -- 172.21.15.28:0/1549424601 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb40402c040 con 0x55a7a32dbb30 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.738+0000 7fb41e9e4b80 10 client.26461 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.738+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a7a3357d00 con 0x55a7a32dbb30 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.738+0000 7fb3f3400700 1 -- 172.21.15.28:0/1549424601 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3f80add40 con 0x55a7a32dbb30 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb3f3400700 1 -- 172.21.15.28:0/1549424601 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb4040388d0 con 0x55a7a32dbb30 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb41e9e4b80 10 client.26461 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb41e9e4b80 10 client.26461 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb41e9e4b80 20 client.26461 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a33582c0 0x55a7a335a6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb41e9e4b80 4 client.26461 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "65623019", "release": true, "await": true}] 2024-05-20T20:10:50.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "65623019", "release": true, "await": true}) -- 0x55a7a322b670 con 0x55a7a33582c0 2024-05-20T20:10:50.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.739+0000 7fb411600700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a33582c0 0x55a7a335a6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:50.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.740+0000 7fb411600700 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a33582c0 0x55a7a335a6e0 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1dd6fd3623b87584 server_cookie=7a2a1315d7ef8bea in_seq=0 out_seq=0 2024-05-20T20:10:50.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.740+0000 7fb3f3400700 10 client.26461 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:50.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.748+0000 7fb3f3400700 1 -- 172.21.15.28:0/1549424601 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x55a7a322b670 con 0x55a7a33582c0 2024-05-20T20:10:50.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.748+0000 7fb3f3400700 10 client.26461 handle_command_reply: tid=0 2024-05-20T20:10:50.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.748+0000 7fb41e9e4b80 1 client.26461 shutdown 2024-05-20T20:10:50.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 msgr2=0x55a7a32e1c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32e1c80 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fb4040098e0 tx=0x7fb4040038b0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 shutdown_connections 2024-05-20T20:10:50.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a7a33582c0 0x55a7a335a6e0 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a7a32dbb30 0x55a7a32e1c80 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a7a32d25f0 0x55a7a32e1200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 --2- 172.21.15.28:0/1549424601 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a7a32d7c10 0x55a7a32e1740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 >> 172.21.15.28:0/1549424601 conn(0x55a7a31ffce0 msgr2=0x55a7a32da790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:50.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.749+0000 7fb3f3400700 0 client.26461 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:50.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.751+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 shutdown_connections 2024-05-20T20:10:50.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.751+0000 7fb41e9e4b80 1 -- 172.21.15.28:0/1549424601 wait complete. 2024-05-20T20:10:50.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.752+0000 7fb41e9e4b80 20 client.26461 trim_cache size 0 max 16384 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 12, 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout: "65623019": { 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 12, 2024-05-20T20:10:50.756 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 144.7, 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:10:50.757 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:10:50.758 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:50.758 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:50.758 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:50.758 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:50.758 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:10:50.758 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.753+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3f40b29f0 msgr2=0x7fb3f40b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.753+0000 7fb414000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3f40b29f0 0x7fb3f40b4eb0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fb400009c10 tx=0x7fb40000b040 comp rx=0 tx=0).stop 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.753+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c06bb70 msgr2=0x7fb40c19c410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.753+0000 7fb414000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c06bb70 0x7fb40c19c410 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7fb3f800e9f0 tx=0x7fb3f800ed00 comp rx=0 tx=0).stop 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.754+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 shutdown_connections 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.754+0000 7fb414000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb40c10fff0 0x7fb40c1a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.754+0000 7fb414000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb40c06b250 0x7fb40c19bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.754+0000 7fb414000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3f40b29f0 0x7fb3f40b4eb0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.754+0000 7fb414000700 1 --2- 172.21.15.28:0/3945998468 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb40c06bb70 0x7fb40c19c410 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.754+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 >> 172.21.15.28:0/3945998468 conn(0x7fb40c0793f0 msgr2=0x7fb40c1114e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.756+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 shutdown_connections 2024-05-20T20:10:50.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:50.757+0000 7fb414000700 1 -- 172.21.15.28:0/3945998468 wait complete. 2024-05-20T20:10:51.058 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:50 smithi028 ceph-mon[31178]: pgmap v159: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 272 MiB/s wr, 889 op/s 2024-05-20T20:10:51.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:50 smithi154 ceph-mon[36314]: pgmap v159: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 272 MiB/s wr, 889 op/s 2024-05-20T20:10:51.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:50 smithi033 ceph-mon[36179]: pgmap v159: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 272 MiB/s wr, 889 op/s 2024-05-20T20:10:51.932 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.932+0000 7f51d7000640 1 -- 172.21.15.28:0/2912028072 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 msgr2=0x7f51d0117f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:51.932 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.932+0000 7f51d7000640 1 --2- 172.21.15.28:0/2912028072 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 0x7f51d0117f90 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f51c8009cd0 tx=0x7f51c802a980 comp rx=0 tx=0).stop 2024-05-20T20:10:51.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.933+0000 7f51d7000640 1 -- 172.21.15.28:0/2912028072 shutdown_connections 2024-05-20T20:10:51.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.933+0000 7f51d7000640 1 --2- 172.21.15.28:0/2912028072 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 0x7f51d0117f90 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.933+0000 7f51d7000640 1 --2- 172.21.15.28:0/2912028072 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51d010a370 0x7f51d0115660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.933+0000 7f51d7000640 1 --2- 172.21.15.28:0/2912028072 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51d0109a50 0x7f51d0109e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.933+0000 7f51d7000640 1 -- 172.21.15.28:0/2912028072 >> 172.21.15.28:0/2912028072 conn(0x7f51d006db20 msgr2=0x7f51d006df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:51.936 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.936+0000 7f51d7000640 1 -- 172.21.15.28:0/2912028072 shutdown_connections 2024-05-20T20:10:51.936 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.936+0000 7f51d7000640 1 -- 172.21.15.28:0/2912028072 wait complete. 2024-05-20T20:10:51.938 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.939+0000 7f51d7000640 1 Processor -- start 2024-05-20T20:10:51.939 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.939+0000 7f51d7000640 1 -- start start 2024-05-20T20:10:51.940 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.940+0000 7f51d7000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51d0109a50 0x7f51d01a97b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.941+0000 7f51d7000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51d010a370 0x7f51d01a9cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.941+0000 7f51d7000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 0x7f51d01a3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.942+0000 7f51cf400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51d010a370 0x7f51d01a9cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.942+0000 7f51cf400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51d010a370 0x7f51d01a9cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49018/0 (socket says 172.21.15.28:49018) 2024-05-20T20:10:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.942+0000 7f51cf400640 1 -- 172.21.15.28:0/13028591 learned_addr learned my addr 172.21.15.28:0/13028591 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.942+0000 7f51d5000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 0x7f51d01a3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.942+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f51d011b4e0 con 0x7f51d0109a50 2024-05-20T20:10:51.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.942+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f51d011b360 con 0x7f51d0115ba0 2024-05-20T20:10:51.943 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.943+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f51d011b660 con 0x7f51d010a370 2024-05-20T20:10:51.943 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.943+0000 7f51cfe00640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51d0109a50 0x7f51d01a97b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.944+0000 7f51cfe00640 1 -- 172.21.15.28:0/13028591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51d010a370 msgr2=0x7f51d01a9cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.944+0000 7f51cfe00640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51d010a370 0x7f51d01a9cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.944 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.944+0000 7f51cfe00640 1 -- 172.21.15.28:0/13028591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 msgr2=0x7f51d01a3960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:51.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.944+0000 7f51cfe00640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 0x7f51d01a3960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.945+0000 7f51cfe00640 1 -- 172.21.15.28:0/13028591 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51d01a40e0 con 0x7f51d0109a50 2024-05-20T20:10:51.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.945+0000 7f51cfe00640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51d0109a50 0x7f51d01a97b0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f51bc00ca30 tx=0x7f51bc00cef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:51.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.946+0000 7f51ccc00640 1 -- 172.21.15.28:0/13028591 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51bc007b20 con 0x7f51d0109a50 2024-05-20T20:10:51.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.947+0000 7f51ccc00640 1 -- 172.21.15.28:0/13028591 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51bc0054d0 con 0x7f51d0109a50 2024-05-20T20:10:51.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.947+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f51d01a43d0 con 0x7f51d0109a50 2024-05-20T20:10:51.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.947+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f51d01b0670 con 0x7f51d0109a50 2024-05-20T20:10:51.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.947+0000 7f51ccc00640 1 -- 172.21.15.28:0/13028591 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51bc00f450 con 0x7f51d0109a50 2024-05-20T20:10:51.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.957+0000 7f51d9e11740 1 Processor -- start 2024-05-20T20:10:51.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.957+0000 7f51d9e11740 1 -- start start 2024-05-20T20:10:51.959 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.958+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b165880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.959 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.959+0000 7f51d5000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b165880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.959 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.959+0000 7f51d5000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b165880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35234/0 (socket says 172.21.15.28:35234) 2024-05-20T20:10:51.959 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.959+0000 7f51ccc00640 1 -- 172.21.15.28:0/13028591 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f51bc004ee0 con 0x7f51d0109a50 2024-05-20T20:10:51.959 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.959+0000 7f51d5000640 1 -- 172.21.15.28:0/2373714025 learned_addr learned my addr 172.21.15.28:0/2373714025 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:51.959 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.959+0000 7f51d9e11740 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b170140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.959+0000 7f51d9e11740 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b172a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.960+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56447b0cdb90 con 0x56447b1654a0 2024-05-20T20:10:51.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.960+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56447b0bd730 con 0x56447b170680 2024-05-20T20:10:51.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.960+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56447b0c1770 con 0x56447b165dc0 2024-05-20T20:10:51.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.960+0000 7f51cf400640 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b172a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.960+0000 7f51cfe00640 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b170140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.961+0000 7f51cf400640 1 -- 172.21.15.28:0/2373714025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 msgr2=0x56447b170140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:51.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.962+0000 7f51cf400640 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b170140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.962+0000 7f51cf400640 1 -- 172.21.15.28:0/2373714025 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 msgr2=0x56447b165880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:51.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.962+0000 7f51cf400640 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b165880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.963+0000 7f51cf400640 1 -- 172.21.15.28:0/2373714025 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56447b0568b0 con 0x56447b170680 2024-05-20T20:10:51.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.963+0000 7f51cfe00640 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b170140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:51.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.963+0000 7f51d5000640 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b165880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:51.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.963+0000 7f51ccc00640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f51b40b2b60 0x7f51b40b5020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.965 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.964+0000 7f51cf400640 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b172a70 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f51d01a5150 tx=0x7f51c000f730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=91c381b62de601ab server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:51.966 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.964+0000 7f51ccc00640 1 -- 172.21.15.28:0/13028591 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f51bc0e2340 con 0x7f51d0109a50 2024-05-20T20:10:51.966 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.965+0000 7f51d5000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f51b40b2b60 0x7f51b40b5020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.966 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.966+0000 7f51c5600640 1 -- 172.21.15.28:0/2373714025 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51c0015070 con 0x56447b170680 2024-05-20T20:10:51.968 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.967+0000 7f51c5600640 1 -- 172.21.15.28:0/2373714025 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51c00118f0 con 0x56447b170680 2024-05-20T20:10:51.968 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.968+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 msgr2=0x56447b172a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:51.968 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.968+0000 7f51d9e11740 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b172a70 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f51d01a5150 tx=0x7f51c000f730 comp rx=0 tx=0).stop 2024-05-20T20:10:51.969 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.969+0000 7f51d5000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f51b40b2b60 0x7f51b40b5020 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f51c8005fe0 tx=0x7f51c802ae80 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.969+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 shutdown_connections 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.969+0000 7f51d9e11740 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b172a70 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.969+0000 7f51d9e11740 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b170140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.969+0000 7f51d9e11740 1 --2- 172.21.15.28:0/2373714025 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b165880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.969+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 >> 172.21.15.28:0/2373714025 conn(0x56447b1619b0 msgr2=0x56447b161d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.970+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 shutdown_connections 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.970+0000 7f51d9e11740 1 -- 172.21.15.28:0/2373714025 wait complete. 2024-05-20T20:10:51.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.971+0000 7f51d9e11740 1 -- start start 2024-05-20T20:10:51.971 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.971+0000 7f51d9e11740 1 Processor -- start 2024-05-20T20:10:51.973 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.972+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b16ff80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.973 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.972+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b16d270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.973 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.973+0000 7f51cfe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b16ff80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.973 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b16d7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.975 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51d9e11740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56447ada3790 con 0x56447b1654a0 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51d9e11740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56447b065bc0 con 0x56447b170680 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51d9e11740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56447b173300 con 0x56447b165dc0 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51cf400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b16d270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51cf400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b16d270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49044/0 (socket says 172.21.15.28:49044) 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51cf400640 1 -- 172.21.15.28:0/269213330 learned_addr learned my addr 172.21.15.28:0/269213330 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.974+0000 7f51cfe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b16ff80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35242/0 (socket says 172.21.15.28:35242) 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.975+0000 7f51d5000640 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b16d7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.976 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.975+0000 7f51d9e11740 10 client.0 _ll_register_callbacks cb 0x56447b043400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:51.977 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.977+0000 7f51cf400640 1 -- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 msgr2=0x56447b16d7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:51.977 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.977+0000 7f51cf400640 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b16d7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.977 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.977+0000 7f51cf400640 1 -- 172.21.15.28:0/269213330 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 msgr2=0x56447b16ff80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:51.977 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.977+0000 7f51cf400640 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b16ff80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.977 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.977+0000 7f51cf400640 1 -- 172.21.15.28:0/269213330 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56447add2210 con 0x56447b165dc0 2024-05-20T20:10:51.978 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.978+0000 7f51cf400640 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b16d270 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f51c0000c00 tx=0x7f51c000b9d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d178e294e8e7256d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:51.978 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.978+0000 7f51ba000640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:10:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.979+0000 7f51ba000640 1 -- 172.21.15.28:0/269213330 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51c0015070 con 0x56447b165dc0 2024-05-20T20:10:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.979+0000 7f51ba000640 1 -- 172.21.15.28:0/269213330 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f51c0011aa0 con 0x56447b165dc0 2024-05-20T20:10:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.979+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f51c00118f0 con 0x56447b165dc0 2024-05-20T20:10:51.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.980+0000 7f51ba000640 1 -- 172.21.15.28:0/269213330 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f51c0011ca0 con 0x56447b165dc0 2024-05-20T20:10:51.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.980+0000 7f51d9e11740 10 client.26473 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:51.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.980+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56447ae9e620 con 0x56447b165dc0 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.980+0000 7f51ba000640 1 -- 172.21.15.28:0/269213330 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51c00118f0 con 0x56447b165dc0 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.980+0000 7f51ba000640 1 -- 172.21.15.28:0/269213330 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f51c0005440 con 0x56447b165dc0 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.981+0000 7f51d9e11740 10 client.26473 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.981+0000 7f51d9e11740 10 client.26473 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.981+0000 7f51d9e11740 20 client.26473 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.981+0000 7f51d9e11740 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1803c0 0x56447b182820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.981+0000 7f51d9e11740 4 client.26473 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:51.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.981+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56447b0cfae0 con 0x56447b1803c0 2024-05-20T20:10:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.981+0000 7f51cfe00640 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1803c0 0x56447b182820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.982+0000 7f51cfe00640 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1803c0 0x56447b182820 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=de51cdac1fe82de6 server_cookie=45510133fd0942bb in_seq=0 out_seq=0 2024-05-20T20:10:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.982+0000 7f51ba000640 10 client.26473 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:51.982 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.983+0000 7f51ba000640 1 -- 172.21.15.28:0/269213330 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56447b0cfae0 con 0x56447b1803c0 2024-05-20T20:10:51.983 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.983+0000 7f51ba000640 10 client.26473 handle_command_reply: tid=0 2024-05-20T20:10:51.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.983+0000 7f51d9e11740 1 client.26473 shutdown 2024-05-20T20:10:51.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.983+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 msgr2=0x56447b16d270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.983+0000 7f51d9e11740 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b16d270 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f51c0000c00 tx=0x7f51c000b9d0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 shutdown_connections 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1803c0 0x56447b182820 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b170680 0x56447b16d7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b16d270 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 --2- 172.21.15.28:0/269213330 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b1654a0 0x56447b16ff80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 >> 172.21.15.28:0/269213330 conn(0x56447b1619b0 msgr2=0x56447b158310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51ba000640 0 client.26473 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 shutdown_connections 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.984+0000 7f51d9e11740 1 -- 172.21.15.28:0/269213330 wait complete. 2024-05-20T20:10:51.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:51.985+0000 7f51d9e11740 20 client.26473 trim_cache size 0 max 16384 2024-05-20T20:10:52.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.026+0000 7f51d9e11740 1 Processor -- start 2024-05-20T20:10:52.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.026+0000 7f51d9e11740 1 -- start start 2024-05-20T20:10:52.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.026+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b1654a0 0x56447b187ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.026+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b165dc0 0x56447b188420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.026+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b16a4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51cfe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b1654a0 0x56447b187ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51cfe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b1654a0 0x56447b187ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49060/0 (socket says 172.21.15.28:49060) 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51d5000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b16a4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51cfe00640 1 -- 172.21.15.28:0/377573147 learned_addr learned my addr 172.21.15.28:0/377573147 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f51d01b4cf0 con 0x56447b170680 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51cf400640 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b165dc0 0x56447b188420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f51d01b6830 con 0x56447b165dc0 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.028+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f51d01b53c0 con 0x56447b1654a0 2024-05-20T20:10:52.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.030+0000 7f51cfe00640 1 -- 172.21.15.28:0/377573147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b165dc0 msgr2=0x56447b188420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.030+0000 7f51cfe00640 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b165dc0 0x56447b188420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.030+0000 7f51cfe00640 1 -- 172.21.15.28:0/377573147 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 msgr2=0x56447b16a4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.030+0000 7f51cfe00640 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b16a4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.030+0000 7f51cfe00640 1 -- 172.21.15.28:0/377573147 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56447b1576b0 con 0x56447b1654a0 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.030+0000 7f51cf400640 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b165dc0 0x56447b188420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.030+0000 7f51d5000640 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b16a4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.031+0000 7f51cfe00640 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b1654a0 0x56447b187ee0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f51bc0058c0 tx=0x7f51bc0cc040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ed19997f22f6210d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.031+0000 7f51c4c00640 1 -- 172.21.15.28:0/377573147 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51bc01d040 con 0x56447b1654a0 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.031+0000 7f51c4c00640 1 -- 172.21.15.28:0/377573147 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51bc0a90e0 con 0x56447b1654a0 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.031+0000 7f51c4c00640 1 -- 172.21.15.28:0/377573147 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51bc01d040 con 0x56447b1654a0 2024-05-20T20:10:52.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b1654a0 msgr2=0x56447b187ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b1654a0 0x56447b187ee0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f51bc0058c0 tx=0x7f51bc0cc040 comp rx=0 tx=0).stop 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 shutdown_connections 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b165dc0 0x56447b188420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b16a4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 --2- 172.21.15.28:0/377573147 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b1654a0 0x56447b187ee0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 >> 172.21.15.28:0/377573147 conn(0x56447b1619b0 msgr2=0x56447b158310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.033+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 shutdown_connections 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.034+0000 7f51d9e11740 1 -- 172.21.15.28:0/377573147 wait complete. 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.035+0000 7f51d9e11740 1 -- start start 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.035+0000 7f51d9e11740 1 Processor -- start 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.036+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b1654a0 0x56447b1873e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.036+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b187920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:52.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.036+0000 7f51d9e11740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b1e8e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.036+0000 7f51d9e11740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f51d01b6160 con 0x56447b170680 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.036+0000 7f51d9e11740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f51d01b5a90 con 0x56447b1654a0 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.036+0000 7f51d9e11740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f51d01b2cf0 con 0x56447b165dc0 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.036+0000 7f51d9e11740 10 client.0 _ll_register_callbacks cb 0x7f51c8009b20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.037+0000 7f51cfe00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b1654a0 0x56447b1873e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.037+0000 7f51cf400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b187920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.038+0000 7f51cfe00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b1654a0 0x56447b1873e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42200/0 (socket says 172.21.15.28:42200) 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.038+0000 7f51cfe00640 1 -- 172.21.15.28:0/3415958753 learned_addr learned my addr 172.21.15.28:0/3415958753 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.038+0000 7f51d5000640 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b1e8e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.039+0000 7f51cfe00640 1 -- 172.21.15.28:0/3415958753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 msgr2=0x56447b187920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.039+0000 7f51cfe00640 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b187920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.039+0000 7f51cfe00640 1 -- 172.21.15.28:0/3415958753 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 msgr2=0x56447b1e8e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:52.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.039+0000 7f51cfe00640 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b1e8e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.045 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.039+0000 7f51cfe00640 1 -- 172.21.15.28:0/3415958753 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56447b1e93c0 con 0x56447b1654a0 2024-05-20T20:10:52.045 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.040+0000 7f51ba000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:52.045 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.040+0000 7f51cfe00640 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b1654a0 0x56447b1873e0 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f51bc0a9560 tx=0x7f51bc0a8e00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dd47ef65464f974f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:52.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.040+0000 7f51ba000640 1 -- 172.21.15.28:0/3415958753 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51bc0c8060 con 0x56447b1654a0 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.041+0000 7f51ba000640 1 -- 172.21.15.28:0/3415958753 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f51bc0a5bc0 con 0x56447b1654a0 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.041+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f51bc0a90e0 con 0x56447b1654a0 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.041+0000 7f51ba000640 1 -- 172.21.15.28:0/3415958753 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f51bc0c2030 con 0x56447b1654a0 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.041+0000 7f51d9e11740 10 client.26549 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.041+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56447b1ea6b0 con 0x56447b1654a0 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.042+0000 7f51ba000640 1 -- 172.21.15.28:0/3415958753 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51bc0a90e0 con 0x56447b1654a0 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.042+0000 7f51ba000640 1 -- 172.21.15.28:0/3415958753 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f51bc0dfcc0 con 0x56447b1654a0 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.043+0000 7f51d9e11740 10 client.26549 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.043+0000 7f51d9e11740 10 client.26549 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:10:52.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.043+0000 7f51d9e11740 20 client.26549 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:52.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.043+0000 7f51d9e11740 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1eacb0 0x56447b1ed0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:52.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.043+0000 7f51d9e11740 4 client.26549 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:10:52.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.043+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56447b0c92c0 con 0x56447b1eacb0 2024-05-20T20:10:52.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.044+0000 7f51cf400640 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1eacb0 0x56447b1ed0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:52.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.045+0000 7f51ba000640 10 client.26549 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:52.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.045+0000 7f51cf400640 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1eacb0 0x56447b1ed0d0 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e9e33ca763bd00f7 server_cookie=748718c2d0d8b3 in_seq=0 out_seq=0 2024-05-20T20:10:52.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.081+0000 7f51ba000640 1 -- 172.21.15.28:0/3415958753 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x56447b0c92c0 con 0x56447b1eacb0 2024-05-20T20:10:52.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.081+0000 7f51ba000640 10 client.26549 handle_command_reply: tid=0 2024-05-20T20:10:52.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 client.26549 shutdown 2024-05-20T20:10:52.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b1654a0 msgr2=0x56447b1873e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:52.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b1654a0 0x56447b1873e0 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f51bc0a9560 tx=0x7f51bc0a8e00 comp rx=0 tx=0).stop 2024-05-20T20:10:52.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 shutdown_connections 2024-05-20T20:10:52.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56447b1eacb0 0x56447b1ed0d0 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51ba000640 0 client.26549 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:10:52.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56447b170680 0x56447b1e8e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56447b165dc0 0x56447b187920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 --2- 172.21.15.28:0/3415958753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56447b1654a0 0x56447b1873e0 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.084 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.082+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 >> 172.21.15.28:0/3415958753 conn(0x56447b1619b0 msgr2=0x56447b17b5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:52.084 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.083+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 shutdown_connections 2024-05-20T20:10:52.084 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.083+0000 7f51d9e11740 1 -- 172.21.15.28:0/3415958753 wait complete. 2024-05-20T20:10:52.085 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:10:52.085 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:10:52.085 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:10:52.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.083+0000 7f51d9e11740 20 client.26549 trim_cache size 0 max 16384 2024-05-20T20:10:52.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f51b40b2b60 msgr2=0x7f51b40b5020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:52.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f51b40b2b60 0x7f51b40b5020 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f51c8005fe0 tx=0x7f51c802ae80 comp rx=0 tx=0).stop 2024-05-20T20:10:52.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51d0109a50 msgr2=0x7f51d01a97b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:52.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51d0109a50 0x7f51d01a97b0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f51bc00ca30 tx=0x7f51bc00cef0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 shutdown_connections 2024-05-20T20:10:52.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f51d0115ba0 0x7f51d01a3960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f51b40b2b60 0x7f51b40b5020 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51d010a370 0x7f51d01a9cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 --2- 172.21.15.28:0/13028591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f51d0109a50 0x7f51d01a97b0 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:52.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.086+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 >> 172.21.15.28:0/13028591 conn(0x7f51d006db20 msgr2=0x7f51d0071670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:52.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.089+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 shutdown_connections 2024-05-20T20:10:52.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:10:52.089+0000 7f51d7000640 1 -- 172.21.15.28:0/13028591 wait complete. 2024-05-20T20:10:52.981 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set '65623019', total seconds elapsed: 38.4 2024-05-20T20:10:53.310 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:52 smithi028 ceph-mon[31178]: pgmap v160: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 150 MiB/s wr, 358 op/s 2024-05-20T20:10:53.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:52 smithi154 ceph-mon[36314]: pgmap v160: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 150 MiB/s wr, 358 op/s 2024-05-20T20:10:53.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:52 smithi033 ceph-mon[36179]: pgmap v160: 97 pgs: 97 active+clean; 25 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 150 MiB/s wr, 358 op/s 2024-05-20T20:10:55.138 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:54 smithi028 ceph-mon[31178]: pgmap v161: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 185 MiB/s wr, 544 op/s 2024-05-20T20:10:55.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:54 smithi154 ceph-mon[36314]: pgmap v161: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 185 MiB/s wr, 544 op/s 2024-05-20T20:10:55.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:54 smithi033 ceph-mon[36179]: pgmap v161: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 185 MiB/s wr, 544 op/s 2024-05-20T20:10:55.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.714+0000 7fd464e00700 1 -- 172.21.15.28:0/3813772638 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46011cbf0 msgr2=0x7fd46011efe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.714+0000 7fd464e00700 1 --2- 172.21.15.28:0/3813772638 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46011cbf0 0x7fd46011efe0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fd45800b0a0 tx=0x7fd45800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.714+0000 7fd464e00700 1 -- 172.21.15.28:0/3813772638 shutdown_connections 2024-05-20T20:10:55.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.714+0000 7fd464e00700 1 --2- 172.21.15.28:0/3813772638 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46011cbf0 0x7fd46011efe0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.714+0000 7fd464e00700 1 --2- 172.21.15.28:0/3813772638 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46006f080 0x7fd46006f460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.714+0000 7fd464e00700 1 --2- 172.21.15.28:0/3813772638 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46006f9a0 0x7fd46011c6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.714+0000 7fd464e00700 1 -- 172.21.15.28:0/3813772638 >> 172.21.15.28:0/3813772638 conn(0x7fd46006e760 msgr2=0x7fd46010c550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.719+0000 7fd464e00700 1 -- 172.21.15.28:0/3813772638 shutdown_connections 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.719+0000 7fd464e00700 1 -- 172.21.15.28:0/3813772638 wait complete. 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.720+0000 7fd464e00700 1 Processor -- start 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.721+0000 7fd464e00700 1 -- start start 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.722+0000 7fd464e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46006f080 0x7fd46011c180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.722+0000 7fd464e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46006f9a0 0x7fd460115210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.722+0000 7fd464e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46011cbf0 0x7fd460115750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.722+0000 7fd464e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd460122570 con 0x7fd46011cbf0 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.722+0000 7fd464e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601223f0 con 0x7fd46006f9a0 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.722+0000 7fd464e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601226f0 con 0x7fd46006f080 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.723+0000 7fd45de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46006f080 0x7fd46011c180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.723+0000 7fd45de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46006f080 0x7fd46011c180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49094/0 (socket says 172.21.15.28:49094) 2024-05-20T20:10:55.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.723+0000 7fd45de00700 1 -- 172.21.15.28:0/1246640004 learned_addr learned my addr 172.21.15.28:0/1246640004 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.723+0000 7fd45d400700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46006f9a0 0x7fd460115210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.723+0000 7fd45e800700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46011cbf0 0x7fd460115750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.724+0000 7fd45de00700 1 -- 172.21.15.28:0/1246640004 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46006f9a0 msgr2=0x7fd460115210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.724+0000 7fd45de00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46006f9a0 0x7fd460115210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.724+0000 7fd45de00700 1 -- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46011cbf0 msgr2=0x7fd460115750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.724+0000 7fd45de00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46011cbf0 0x7fd460115750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.724+0000 7fd45de00700 1 -- 172.21.15.28:0/1246640004 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd460115d40 con 0x7fd46006f080 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.724+0000 7fd45de00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46006f080 0x7fd46011c180 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fd44c00d980 tx=0x7fd44c00dc90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.725+0000 7fd456a00700 1 -- 172.21.15.28:0/1246640004 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd44c016070 con 0x7fd46006f080 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.725+0000 7fd456a00700 1 -- 172.21.15.28:0/1246640004 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd44c004830 con 0x7fd46006f080 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.725+0000 7fd456a00700 1 -- 172.21.15.28:0/1246640004 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd44c0054c0 con 0x7fd46006f080 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.725+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd460116030 con 0x7fd46006f080 2024-05-20T20:10:55.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.725+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd460119fd0 con 0x7fd46006f080 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.740+0000 7fd46f7ccb80 1 Processor -- start 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.741+0000 7fd456a00700 1 -- 172.21.15.28:0/1246640004 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd44c004220 con 0x7fd46006f080 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.742+0000 7fd46f7ccb80 1 -- start start 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.744+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fec5290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.744+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec57d0 0x55727febf260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.744+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727febf880 0x55727febfce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.744+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55727fe1de40 con 0x55727fec4eb0 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.744+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55727fe0d9e0 con 0x55727febf880 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.744+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55727fe11a20 con 0x55727fec57d0 2024-05-20T20:10:55.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.745+0000 7fd45d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727febf880 0x55727febfce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.745+0000 7fd45d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727febf880 0x55727febfce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42226/0 (socket says 172.21.15.28:42226) 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.745+0000 7fd45d400700 1 -- 172.21.15.28:0/2116971554 learned_addr learned my addr 172.21.15.28:0/2116971554 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.745+0000 7fd45de00700 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec57d0 0x55727febf260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.746+0000 7fd456a00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4440b2af0 0x7fd4440b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.746+0000 7fd45d400700 1 -- 172.21.15.28:0/2116971554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec57d0 msgr2=0x55727febf260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.746+0000 7fd45d400700 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec57d0 0x55727febf260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.746+0000 7fd45d400700 1 -- 172.21.15.28:0/2116971554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 msgr2=0x55727fec5290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.746+0000 7fd45d400700 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fec5290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.746+0000 7fd45d400700 1 -- 172.21.15.28:0/2116971554 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55727fdd6420 con 0x55727febf880 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.746+0000 7fd456a00700 1 -- 172.21.15.28:0/1246640004 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd44c0e2240 con 0x7fd46006f080 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.747+0000 7fd45e800700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4440b2af0 0x7fd4440b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.748+0000 7fd45d400700 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727febf880 0x55727febfce0 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fd450009cb0 tx=0x7fd450011440 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a834eb1ab39d3ee3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.749+0000 7fd45e800700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4440b2af0 0x7fd4440b4fb0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fd45800b070 tx=0x7fd458009780 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.749+0000 7fd44b400700 1 -- 172.21.15.28:0/2116971554 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd450011d20 con 0x55727febf880 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.749+0000 7fd44b400700 1 -- 172.21.15.28:0/2116971554 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd450004500 con 0x55727febf880 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.749+0000 7fd44b400700 1 -- 172.21.15.28:0/2116971554 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd450005310 con 0x55727febf880 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.750+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2116971554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727febf880 msgr2=0x55727febfce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.750+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727febf880 0x55727febfce0 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fd450009cb0 tx=0x7fd450011440 comp rx=0 tx=0).stop 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.750+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2116971554 shutdown_connections 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.750+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727febf880 0x55727febfce0 secure :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fd450009cb0 tx=0x7fd450011440 comp rx=0 tx=0).stop 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.750+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fec5290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.750+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2116971554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec57d0 0x55727febf260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.750+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2116971554 >> 172.21.15.28:0/2116971554 conn(0x55727fdedbe0 msgr2=0x55727feba010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.757+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2116971554 shutdown_connections 2024-05-20T20:10:55.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.757+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2116971554 wait complete. 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.758+0000 7fd46f7ccb80 1 -- start start 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.759+0000 7fd46f7ccb80 1 Processor -- start 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.760+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fecb7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.760+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec57d0 0x55727fec8a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.760+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fec9410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.761+0000 7fd45d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec57d0 0x55727fec8a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.761+0000 7fd45d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec57d0 0x55727fec8a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42236/0 (socket says 172.21.15.28:42236) 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.761+0000 7fd45de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fecb7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.761+0000 7fd45de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fecb7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35308/0 (socket says 172.21.15.28:35308) 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.764+0000 7fd45d400700 1 -- 172.21.15.28:0/98753093 learned_addr learned my addr 172.21.15.28:0/98753093 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.764+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55727fd13fa0 con 0x55727fec4eb0 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.764+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55727fdae760 con 0x55727fec57d0 2024-05-20T20:10:55.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.764+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55727fecc240 con 0x55727fec8fd0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.764+0000 7fd46f7ccb80 10 client.0 _ll_register_callbacks cb 0x55727fdb7c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.764+0000 7fd45e800700 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fec9410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.765+0000 7fd45d400700 1 -- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 msgr2=0x55727fec9410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.765+0000 7fd45d400700 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fec9410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.765+0000 7fd45d400700 1 -- 172.21.15.28:0/98753093 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 msgr2=0x55727fecb7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.765+0000 7fd45d400700 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fecb7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.765+0000 7fd45d400700 1 -- 172.21.15.28:0/98753093 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55727fd5d8d0 con 0x55727fec57d0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.765+0000 7fd45d400700 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec57d0 0x55727fec8a40 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7fd450009c80 tx=0x7fd450004480 comp rx=0 tx=0).ready entity=mon.2 client_cookie=843e6486277bf3ed server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.766+0000 7fd43fe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.766+0000 7fd43fe00700 1 -- 172.21.15.28:0/98753093 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd450028060 con 0x55727fec57d0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.766+0000 7fd43fe00700 1 -- 172.21.15.28:0/98753093 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd450011b20 con 0x55727fec57d0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.767+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd450004500 con 0x55727fec57d0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.768+0000 7fd43fe00700 1 -- 172.21.15.28:0/98753093 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd450011d20 con 0x55727fec57d0 2024-05-20T20:10:55.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.769+0000 7fd46f7ccb80 10 client.26561 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.769+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55727fdde060 con 0x55727fec57d0 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.771+0000 7fd43fe00700 1 -- 172.21.15.28:0/98753093 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd450004500 con 0x55727fec57d0 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.771+0000 7fd43fe00700 1 -- 172.21.15.28:0/98753093 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd450018070 con 0x55727fec57d0 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.773+0000 7fd46f7ccb80 10 client.26561 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.773+0000 7fd46f7ccb80 10 client.26561 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.773+0000 7fd46f7ccb80 20 client.26561 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.773+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727fed92c0 0x55727fedb6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.773+0000 7fd46f7ccb80 4 client.26561 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.773+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55727fe1fd90 con 0x55727fed92c0 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.773+0000 7fd45de00700 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727fed92c0 0x55727fedb6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.775+0000 7fd45de00700 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727fed92c0 0x55727fedb6b0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=4f888e85de691cea server_cookie=39e356c1911cd71c in_seq=0 out_seq=0 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.775+0000 7fd43fe00700 10 client.26561 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:10:55.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.776+0000 7fd43fe00700 1 -- 172.21.15.28:0/98753093 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55727fe1fd90 con 0x55727fed92c0 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.776+0000 7fd43fe00700 10 client.26561 handle_command_reply: tid=0 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.776+0000 7fd46f7ccb80 1 client.26561 shutdown 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec57d0 msgr2=0x55727fec8a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec57d0 0x55727fec8a40 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7fd450009c80 tx=0x7fd450004480 comp rx=0 tx=0).stop 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 shutdown_connections 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727fed92c0 0x55727fedb6b0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fec9410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd43fe00700 0 client.26561 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec4eb0 0x55727fecb7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/98753093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec57d0 0x55727fec8a40 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.777+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 >> 172.21.15.28:0/98753093 conn(0x55727fdedbe0 msgr2=0x55727febe250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.778+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 shutdown_connections 2024-05-20T20:10:55.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.778+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/98753093 wait complete. 2024-05-20T20:10:55.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.778+0000 7fd46f7ccb80 20 client.26561 trim_cache size 0 max 16384 2024-05-20T20:10:55.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 Processor -- start 2024-05-20T20:10:55.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 -- start start 2024-05-20T20:10:55.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727febf260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fed5820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601c5620 con 0x55727fec57d0 2024-05-20T20:10:55.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601c7160 con 0x55727fec4eb0 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.807+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601c5cf0 con 0x55727fec8fd0 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fed5820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fed5820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:35312/0 (socket says 172.21.15.28:35312) 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727febf260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45d400700 1 -- 172.21.15.28:0/2406017378 learned_addr learned my addr 172.21.15.28:0/2406017378 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45de00700 1 -- 172.21.15.28:0/2406017378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 msgr2=0x55727fee0da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45de00700 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.808+0000 7fd45de00700 1 -- 172.21.15.28:0/2406017378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 msgr2=0x55727fed5820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.809+0000 7fd45de00700 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fed5820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.809+0000 7fd45de00700 1 -- 172.21.15.28:0/2406017378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55727fec54e0 con 0x55727fec4eb0 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.809+0000 7fd45d400700 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fed5820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.809+0000 7fd45de00700 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727febf260 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7fd44c0df940 tx=0x7fd44c0e4610 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c26433537753316 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:55.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.809+0000 7fd44aa00700 1 -- 172.21.15.28:0/2406017378 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd44c0a5390 con 0x55727fec4eb0 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.809+0000 7fd44aa00700 1 -- 172.21.15.28:0/2406017378 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd44c004510 con 0x55727fec4eb0 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.809+0000 7fd44aa00700 1 -- 172.21.15.28:0/2406017378 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd44c005660 con 0x55727fec4eb0 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2406017378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 msgr2=0x55727febf260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727febf260 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7fd44c0df940 tx=0x7fd44c0e4610 comp rx=0 tx=0).stop 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2406017378 shutdown_connections 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727febf260 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/2406017378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fed5820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2406017378 >> 172.21.15.28:0/2406017378 conn(0x55727fdedbe0 msgr2=0x55727feb3680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.810+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2406017378 shutdown_connections 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.811+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/2406017378 wait complete. 2024-05-20T20:10:55.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.811+0000 7fd46f7ccb80 1 -- start start 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.811+0000 7fd46f7ccb80 1 Processor -- start 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727fed1c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fee06e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd46f7ccb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601c6a90 con 0x55727fec57d0 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601c63c0 con 0x55727fec4eb0 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd46f7ccb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd4601c35e0 con 0x55727fec8fd0 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd46f7ccb80 10 client.0 _ll_register_callbacks cb 0x55727fdb7c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd45e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd45e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49130/0 (socket says 172.21.15.28:49130) 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd45e800700 1 -- 172.21.15.28:0/1238898609 learned_addr learned my addr 172.21.15.28:0/1238898609 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd45de00700 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727fed1c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.812+0000 7fd45d400700 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fee06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.813+0000 7fd45de00700 1 -- 172.21.15.28:0/1238898609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 msgr2=0x55727fee0cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.813+0000 7fd45de00700 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.813+0000 7fd45de00700 1 -- 172.21.15.28:0/1238898609 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 msgr2=0x55727fee06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.813+0000 7fd45de00700 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fee06e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.813+0000 7fd45de00700 1 -- 172.21.15.28:0/1238898609 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55727fee1500 con 0x55727fec4eb0 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.814+0000 7fd45de00700 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727fed1c90 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x55727febbc20 tx=0x7fd44c004f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cc9c14560389453a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.814+0000 7fd43fe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.814+0000 7fd43fe00700 1 -- 172.21.15.28:0/1238898609 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd44c0a94d0 con 0x55727fec4eb0 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.814+0000 7fd43fe00700 1 -- 172.21.15.28:0/1238898609 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd44c0a5ac0 con 0x55727fec4eb0 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.814+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55727fe19570 con 0x55727fec4eb0 2024-05-20T20:10:55.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.814+0000 7fd43fe00700 1 -- 172.21.15.28:0/1238898609 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd44c0c9050 con 0x55727fec4eb0 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 10 client.26567 fetch_fsmap learned FSMap version 29 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55727ff43d80 con 0x55727fec4eb0 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd43fe00700 1 -- 172.21.15.28:0/1238898609 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55727fe19570 con 0x55727fec4eb0 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd43fe00700 1 -- 172.21.15.28:0/1238898609 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd44c0a97c0 con 0x55727fec4eb0 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 10 client.26567 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 10 client.26567 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 20 client.26567 populate_metadata read hostname 'smithi028' 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727ff44310 0x55727ff46700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 4 client.26567 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.815+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55727fc90250 con 0x55727ff44310 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.816+0000 7fd45d400700 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727ff44310 0x55727ff46700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.817+0000 7fd45d400700 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727ff44310 0x55727ff46700 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=8a23394bf482cd35 server_cookie=cc88da5b274f1198 in_seq=0 out_seq=0 2024-05-20T20:10:55.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.817+0000 7fd43fe00700 10 client.26567 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:10:55.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd43fe00700 1 -- 172.21.15.28:0/1238898609 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55727fc90250 con 0x55727ff44310 2024-05-20T20:10:55.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd43fe00700 10 client.26567 handle_command_reply: tid=0 2024-05-20T20:10:55.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 client.26567 shutdown 2024-05-20T20:10:55.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 msgr2=0x55727fed1c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727fed1c90 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x55727febbc20 tx=0x7fd44c004f10 comp rx=0 tx=0).stop 2024-05-20T20:10:55.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 shutdown_connections 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55727ff44310 0x55727ff46700 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55727fec8fd0 0x55727fee0cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55727fec4eb0 0x55727fed1c90 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 --2- 172.21.15.28:0/1238898609 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55727fec57d0 0x55727fee06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 >> 172.21.15.28:0/1238898609 conn(0x55727fdedbe0 msgr2=0x55727fec29c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.818+0000 7fd43fe00700 0 client.26567 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.819+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 shutdown_connections 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.819+0000 7fd46f7ccb80 1 -- 172.21.15.28:0/1238898609 wait complete. 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.819+0000 7fd46f7ccb80 20 client.26567 trim_cache size 0 max 16384 2024-05-20T20:10:55.826 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:10:55.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4440b2af0 msgr2=0x7fd4440b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4440b2af0 0x7fd4440b4fb0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7fd45800b070 tx=0x7fd458009780 comp rx=0 tx=0).stop 2024-05-20T20:10:55.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46006f080 msgr2=0x7fd46011c180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:10:55.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46006f080 0x7fd46011c180 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fd44c00d980 tx=0x7fd44c00dc90 comp rx=0 tx=0).stop 2024-05-20T20:10:55.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 shutdown_connections 2024-05-20T20:10:55.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46011cbf0 0x7fd460115750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46006f080 0x7fd46011c180 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4440b2af0 0x7fd4440b4fb0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 --2- 172.21.15.28:0/1246640004 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46006f9a0 0x7fd460115210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:10:55.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 >> 172.21.15.28:0/1246640004 conn(0x7fd46006e760 msgr2=0x7fd46011de10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:10:55.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 shutdown_connections 2024-05-20T20:10:55.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:10:55.820+0000 7fd464e00700 1 -- 172.21.15.28:0/1246640004 wait complete. 2024-05-20T20:10:57.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:56 smithi028 ceph-mon[31178]: pgmap v162: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 113 MiB/s wr, 478 op/s 2024-05-20T20:10:57.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:56 smithi154 ceph-mon[36314]: pgmap v162: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 113 MiB/s wr, 478 op/s 2024-05-20T20:10:57.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:56 smithi033 ceph-mon[36179]: pgmap v162: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 113 MiB/s wr, 478 op/s 2024-05-20T20:10:57.703 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:10:58.603 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:10:59.120 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:10:58 smithi028 ceph-mon[31178]: pgmap v163: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 67 MiB/s wr, 372 op/s 2024-05-20T20:10:59.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:10:58 smithi154 ceph-mon[36314]: pgmap v163: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 67 MiB/s wr, 372 op/s 2024-05-20T20:10:59.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:10:58 smithi033 ceph-mon[36179]: pgmap v163: 97 pgs: 97 active+clean; 26 GiB data, 5.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 67 MiB/s wr, 372 op/s 2024-05-20T20:11:01.057 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:01 smithi028 ceph-mon[31178]: pgmap v164: 97 pgs: 97 active+clean; 27 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 192 MiB/s wr, 1.02k op/s 2024-05-20T20:11:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:01 smithi154 ceph-mon[36314]: pgmap v164: 97 pgs: 97 active+clean; 27 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 192 MiB/s wr, 1.02k op/s 2024-05-20T20:11:01.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:01 smithi033 ceph-mon[36179]: pgmap v164: 97 pgs: 97 active+clean; 27 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 192 MiB/s wr, 1.02k op/s 2024-05-20T20:11:02.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.061+0000 7f3a72e00700 1 -- 172.21.15.28:0/1646899840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c102ec0 msgr2=0x7f3a6c106f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.061+0000 7f3a72e00700 1 --2- 172.21.15.28:0/1646899840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c106f10 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7f3a58009990 tx=0x7f3a58009ca0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.061+0000 7f3a72e00700 1 -- 172.21.15.28:0/1646899840 shutdown_connections 2024-05-20T20:11:02.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.061+0000 7f3a72e00700 1 --2- 172.21.15.28:0/1646899840 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a6c1075f0 0x7f3a6c10f1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.061+0000 7f3a72e00700 1 --2- 172.21.15.28:0/1646899840 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102510 0x7f3a6c1028f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.061+0000 7f3a72e00700 1 --2- 172.21.15.28:0/1646899840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c106f10 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.061+0000 7f3a72e00700 1 -- 172.21.15.28:0/1646899840 >> 172.21.15.28:0/1646899840 conn(0x7f3a6c0fdda0 msgr2=0x7f3a6c1001c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:02.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.062+0000 7f3a72e00700 1 -- 172.21.15.28:0/1646899840 shutdown_connections 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.062+0000 7f3a72e00700 1 -- 172.21.15.28:0/1646899840 wait complete. 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.064+0000 7f3a72e00700 1 Processor -- start 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.064+0000 7f3a72e00700 1 -- start start 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a6c102510 0x7f3a6c10eef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c10af90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c1075f0 0x7f3a6c10b4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c111460 con 0x7f3a6c102ec0 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1112e0 con 0x7f3a6c1075f0 2024-05-20T20:11:02.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1115e0 con 0x7f3a6c102510 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a71000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c10af90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c1075f0 0x7f3a6c10b4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c1075f0 0x7f3a6c10b4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53384/0 (socket says 172.21.15.28:53384) 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a72400700 1 -- 172.21.15.28:0/2012001958 learned_addr learned my addr 172.21.15.28:0/2012001958 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a6c102510 0x7f3a6c10eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.065+0000 7f3a71000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c10af90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38256/0 (socket says 172.21.15.28:38256) 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.068+0000 7f3a71000700 1 -- 172.21.15.28:0/2012001958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a6c102510 msgr2=0x7f3a6c10eef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.068+0000 7f3a71000700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a6c102510 0x7f3a6c10eef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.068+0000 7f3a71000700 1 -- 172.21.15.28:0/2012001958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c1075f0 msgr2=0x7f3a6c10b4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.068+0000 7f3a71000700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c1075f0 0x7f3a6c10b4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.068+0000 7f3a71000700 1 -- 172.21.15.28:0/2012001958 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3a6c10bc90 con 0x7f3a6c102ec0 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.068+0000 7f3a72400700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c1075f0 0x7f3a6c10b4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.068+0000 7f3a71000700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c10af90 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f3a580098e0 tx=0x7f3a580057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.069+0000 7f3a6a000700 1 -- 172.21.15.28:0/2012001958 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a58027070 con 0x7f3a6c102ec0 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.069+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3a6c1ac6f0 con 0x7f3a6c102ec0 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.069+0000 7f3a6a000700 1 -- 172.21.15.28:0/2012001958 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3a58022470 con 0x7f3a6c102ec0 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.069+0000 7f3a6a000700 1 -- 172.21.15.28:0/2012001958 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a58017650 con 0x7f3a6c102ec0 2024-05-20T20:11:02.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.069+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3a6c1acb40 con 0x7f3a6c102ec0 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.078+0000 7f3a6a000700 1 -- 172.21.15.28:0/2012001958 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3a5802a070 con 0x7f3a6c102ec0 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.082+0000 7f3a6a000700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a540b2a60 0x7f3a540b4f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.083+0000 7f3a6a000700 1 -- 172.21.15.28:0/2012001958 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f3a580e34c0 con 0x7f3a6c102ec0 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.083+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a540b2a60 0x7f3a540b4f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.084+0000 7f3a7edf8b80 1 Processor -- start 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.084+0000 7f3a7edf8b80 1 -- start start 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3a600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e344b0 0x560cd4e34910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560cd4d92e40 con 0x560cd4e34e50 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x560cd4d829e0 con 0x560cd4e344b0 2024-05-20T20:11:02.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x560cd4d86a20 con 0x560cd4e3a220 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a72400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.085+0000 7f3a71a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3a600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.086+0000 7f3a71000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e344b0 0x560cd4e34910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.086+0000 7f3a71000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e344b0 0x560cd4e34910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53392/0 (socket says 172.21.15.28:53392) 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.086+0000 7f3a71000700 1 -- 172.21.15.28:0/4036044760 learned_addr learned my addr 172.21.15.28:0/4036044760 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.086+0000 7f3a71000700 1 -- 172.21.15.28:0/4036044760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 msgr2=0x560cd4e3a600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.087+0000 7f3a71a00700 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3a600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41232/0 (socket says 172.21.15.28:41232) 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.087+0000 7f3a71000700 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3a600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.087+0000 7f3a71000700 1 -- 172.21.15.28:0/4036044760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 msgr2=0x560cd4e40f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.087+0000 7f3a71000700 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.088+0000 7f3a71000700 1 -- 172.21.15.28:0/4036044760 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560cd4d4b420 con 0x560cd4e344b0 2024-05-20T20:11:02.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.088+0000 7f3a72400700 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.088+0000 7f3a71a00700 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3a600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.088+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a540b2a60 0x7f3a540b4f20 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f3a60006fd0 tx=0x7f3a60008040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.089+0000 7f3a71000700 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e344b0 0x560cd4e34910 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f3a580a6c70 tx=0x7f3a580a6380 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4a0e17d49e45f37c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.089+0000 7f3a5ea00700 1 -- 172.21.15.28:0/4036044760 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a580aabf0 con 0x560cd4e344b0 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.090+0000 7f3a5ea00700 1 -- 172.21.15.28:0/4036044760 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3a580aefa0 con 0x560cd4e344b0 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.090+0000 7f3a5ea00700 1 -- 172.21.15.28:0/4036044760 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a580aae50 con 0x560cd4e344b0 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/4036044760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e344b0 msgr2=0x560cd4e34910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e344b0 0x560cd4e34910 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7f3a580a6c70 tx=0x7f3a580a6380 comp rx=0 tx=0).stop 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/4036044760 shutdown_connections 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3a600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/4036044760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e344b0 0x560cd4e34910 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/4036044760 >> 172.21.15.28:0/4036044760 conn(0x560cd4d62be0 msgr2=0x560cd4e2f1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/4036044760 shutdown_connections 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.091+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/4036044760 wait complete. 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.092+0000 7f3a7edf8b80 1 -- start start 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.092+0000 7f3a7edf8b80 1 Processor -- start 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.093+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.093+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3e060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.093+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e3ea20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.093+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560cd4c88fa0 con 0x560cd4e34e50 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.093+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x560cd4d23760 con 0x560cd4e3e5a0 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.093+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x560cd4e3ac80 con 0x560cd4e3a220 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.093+0000 7f3a7edf8b80 10 client.0 _ll_register_callbacks cb 0x560cd4d2cc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:02.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.094+0000 7f3a71a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.094+0000 7f3a72400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e3ea20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.094+0000 7f3a72400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e3ea20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53404/0 (socket says 172.21.15.28:53404) 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.094+0000 7f3a72400700 1 -- 172.21.15.28:0/2017122212 learned_addr learned my addr 172.21.15.28:0/2017122212 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.094+0000 7f3a71000700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3e060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.094+0000 7f3a72400700 1 -- 172.21.15.28:0/2017122212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 msgr2=0x560cd4e3e060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.096+0000 7f3a72400700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3e060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.096+0000 7f3a72400700 1 -- 172.21.15.28:0/2017122212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 msgr2=0x560cd4e40d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.096+0000 7f3a72400700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.096+0000 7f3a72400700 1 -- 172.21.15.28:0/2017122212 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560cd4cd28d0 con 0x560cd4e3e5a0 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.096+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.096+0000 7f3a71000700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3e060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:02.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.097+0000 7f3a72400700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e3ea20 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f3a64007ed0 tx=0x7f3a6400e7d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3aa9f229ec0c3d9d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.097+0000 7f3a53400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.097+0000 7f3a53400700 1 -- 172.21.15.28:0/2017122212 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a6401a070 con 0x560cd4e3e5a0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.097+0000 7f3a53400700 1 -- 172.21.15.28:0/2017122212 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f3a64016620 con 0x560cd4e3e5a0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.097+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3a580aefa0 con 0x560cd4e3e5a0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.098+0000 7f3a53400700 1 -- 172.21.15.28:0/2017122212 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3a64025050 con 0x560cd4e3e5a0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.098+0000 7f3a7edf8b80 10 client.26579 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.098+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560cd4d53060 con 0x560cd4e3e5a0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a53400700 1 -- 172.21.15.28:0/2017122212 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3a580aefa0 con 0x560cd4e3e5a0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a53400700 1 -- 172.21.15.28:0/2017122212 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3a64015920 con 0x560cd4e3e5a0 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a7edf8b80 10 client.26579 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a7edf8b80 10 client.26579 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:11:02.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a7edf8b80 20 client.26579 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4e4e300 0x560cd4e506f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a7edf8b80 4 client.26579 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560cd4d94d90 con 0x560cd4e4e300 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.099+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4e4e300 0x560cd4e506f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.100+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4e4e300 0x560cd4e506f0 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=b447f82f70d0d258 server_cookie=98303bc7ac8b0777 in_seq=0 out_seq=0 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.100+0000 7f3a53400700 10 client.26579 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a53400700 1 -- 172.21.15.28:0/2017122212 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x560cd4d94d90 con 0x560cd4e4e300 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a53400700 10 client.26579 handle_command_reply: tid=0 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 client.26579 shutdown 2024-05-20T20:11:02.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 msgr2=0x560cd4e3ea20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e3ea20 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f3a64007ed0 tx=0x7f3a6400e7d0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 shutdown_connections 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4e4e300 0x560cd4e506f0 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e3ea20 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e40d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a53400700 0 client.26579 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2017122212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e3e060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.101+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 >> 172.21.15.28:0/2017122212 conn(0x560cd4d62be0 msgr2=0x560cd4e334a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.102+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 shutdown_connections 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.102+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2017122212 wait complete. 2024-05-20T20:11:02.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.102+0000 7f3a7edf8b80 20 client.26579 trim_cache size 0 max 16384 2024-05-20T20:11:02.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.136+0000 7f3a7edf8b80 1 Processor -- start 2024-05-20T20:11:02.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.136+0000 7f3a7edf8b80 1 -- start start 2024-05-20T20:11:02.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.136+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e348b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.136+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e4a6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.137+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e55f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.137+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1b4190 con 0x560cd4e34e50 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.137+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1b5cd0 con 0x560cd4e3e5a0 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.137+0000 7f3a72400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e55f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.137+0000 7f3a72400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e55f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53416/0 (socket says 172.21.15.28:53416) 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.137+0000 7f3a72400700 1 -- 172.21.15.28:0/2896880070 learned_addr learned my addr 172.21.15.28:0/2896880070 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.137+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2896880070 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1b4860 con 0x560cd4e3a220 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.138+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e348b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.138+0000 7f3a72400700 1 -- 172.21.15.28:0/2896880070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 msgr2=0x560cd4e4a6f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.138+0000 7f3a72400700 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e4a6f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.138+0000 7f3a72400700 1 -- 172.21.15.28:0/2896880070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 msgr2=0x560cd4e348b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.138+0000 7f3a72400700 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e348b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.138+0000 7f3a72400700 1 -- 172.21.15.28:0/2896880070 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560cd4e50cb0 con 0x560cd4e3e5a0 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.138+0000 7f3a71a00700 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e348b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.139+0000 7f3a72400700 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e55f40 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f3a6400efb0 tx=0x7f3a64015f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fad820c61b42e052 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.139+0000 7f3a5e000700 1 -- 172.21.15.28:0/2896880070 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a6401a030 con 0x560cd4e3e5a0 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.139+0000 7f3a5e000700 1 -- 172.21.15.28:0/2896880070 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3a6402e070 con 0x560cd4e3e5a0 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.141+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2896880070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 msgr2=0x560cd4e55f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.141+0000 7f3a5e000700 1 -- 172.21.15.28:0/2896880070 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a6401a030 con 0x560cd4e3e5a0 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.141+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e55f40 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f3a6400efb0 tx=0x7f3a64015f10 comp rx=0 tx=0).stop 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.141+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2896880070 shutdown_connections 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.142+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e55f40 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.142+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e348b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.142+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/2896880070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e3a220 0x560cd4e4a6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.142+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2896880070 >> 172.21.15.28:0/2896880070 conn(0x560cd4d62be0 msgr2=0x560cd4e38090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:02.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.144+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2896880070 shutdown_connections 2024-05-20T20:11:02.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.144+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/2896880070 wait complete. 2024-05-20T20:11:02.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.144+0000 7f3a7edf8b80 1 -- start start 2024-05-20T20:11:02.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.145+0000 7f3a7edf8b80 1 Processor -- start 2024-05-20T20:11:02.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.145+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e46ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.145+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e38b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.145+0000 7f3a7edf8b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e390c0 0x560cd4e39520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1b5600 con 0x560cd4e34e50 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1b4f30 con 0x560cd4e3e5a0 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a7edf8b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3a6c1b2130 con 0x560cd4e390c0 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a7edf8b80 10 client.0 _ll_register_callbacks cb 0x560cd4d2cc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a71000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e38b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a71000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e38b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53432/0 (socket says 172.21.15.28:53432) 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a71000700 1 -- 172.21.15.28:0/974783803 learned_addr learned my addr 172.21.15.28:0/974783803 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:02.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.146+0000 7f3a72400700 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e390c0 0x560cd4e39520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.147+0000 7f3a72400700 1 -- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 msgr2=0x560cd4e38b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.147+0000 7f3a72400700 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e38b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.147+0000 7f3a72400700 1 -- 172.21.15.28:0/974783803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 msgr2=0x560cd4e46ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.147+0000 7f3a72400700 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e46ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.148+0000 7f3a72400700 1 -- 172.21.15.28:0/974783803 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560cd4eb8050 con 0x560cd4e390c0 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.148+0000 7f3a71000700 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e38b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.149+0000 7f3a72400700 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e390c0 0x560cd4e39520 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f3a64015e90 tx=0x7f3a6400e7d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=907c0d69bebb7e64 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.149+0000 7f3a53400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.150+0000 7f3a53400700 1 -- 172.21.15.28:0/974783803 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3a64036070 con 0x560cd4e390c0 2024-05-20T20:11:02.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.150+0000 7f3a53400700 1 -- 172.21.15.28:0/974783803 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f3a64003c00 con 0x560cd4e390c0 2024-05-20T20:11:02.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.150+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3a6402e070 con 0x560cd4e390c0 2024-05-20T20:11:02.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.151+0000 7f3a53400700 1 -- 172.21.15.28:0/974783803 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3a64040050 con 0x560cd4e390c0 2024-05-20T20:11:02.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.151+0000 7f3a7edf8b80 10 client.26506 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:02.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.151+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560cd4eb92a0 con 0x560cd4e390c0 2024-05-20T20:11:02.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.152+0000 7f3a53400700 1 -- 172.21.15.28:0/974783803 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3a6402e070 con 0x560cd4e390c0 2024-05-20T20:11:02.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.153+0000 7f3a53400700 1 -- 172.21.15.28:0/974783803 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3a6401db00 con 0x560cd4e390c0 2024-05-20T20:11:02.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.153+0000 7f3a7edf8b80 10 client.26506 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:02.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.153+0000 7f3a7edf8b80 10 client.26506 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:11:02.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.153+0000 7f3a7edf8b80 20 client.26506 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:02.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.153+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4eb9840 0x560cd4ebbc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:02.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.154+0000 7f3a7edf8b80 4 client.26506 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:11:02.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.154+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x560cd4d8e570 con 0x560cd4eb9840 2024-05-20T20:11:02.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.154+0000 7f3a71a00700 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4eb9840 0x560cd4ebbc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:02.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.155+0000 7f3a71a00700 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4eb9840 0x560cd4ebbc30 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=70cd2a2317dfd311 server_cookie=9445123d344f21e9 in_seq=0 out_seq=0 2024-05-20T20:11:02.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.155+0000 7f3a53400700 10 client.26506 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:11:02.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.155+0000 7f3a53400700 1 -- 172.21.15.28:0/974783803 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x560cd4d8e570 con 0x560cd4eb9840 2024-05-20T20:11:02.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.155+0000 7f3a53400700 10 client.26506 handle_command_reply: tid=0 2024-05-20T20:11:02.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.156+0000 7f3a7edf8b80 1 client.26506 shutdown 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.156+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e390c0 msgr2=0x560cd4e39520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.156+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e390c0 0x560cd4e39520 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f3a64015e90 tx=0x7f3a6400e7d0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.157+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 shutdown_connections 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.157+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x560cd4eb9840 0x560cd4ebbc30 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.157+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560cd4e390c0 0x560cd4e39520 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.157+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560cd4e34e50 0x560cd4e46ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.157+0000 7f3a53400700 0 client.26506 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.157+0000 7f3a7edf8b80 1 --2- 172.21.15.28:0/974783803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560cd4e3e5a0 0x560cd4e38b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.157+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 >> 172.21.15.28:0/974783803 conn(0x560cd4d62be0 msgr2=0x560cd4e3d1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.158+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 shutdown_connections 2024-05-20T20:11:02.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.158+0000 7f3a7edf8b80 1 -- 172.21.15.28:0/974783803 wait complete. 2024-05-20T20:11:02.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.158+0000 7f3a7edf8b80 20 client.26506 trim_cache size 0 max 16384 2024-05-20T20:11:02.162 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:11:02.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a540b2a60 msgr2=0x7f3a540b4f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a540b2a60 0x7f3a540b4f20 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f3a60006fd0 tx=0x7f3a60008040 comp rx=0 tx=0).stop 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102ec0 msgr2=0x7f3a6c10af90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c10af90 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f3a580098e0 tx=0x7f3a580057c0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 shutdown_connections 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3a540b2a60 0x7f3a540b4f20 secure :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f3a60006fd0 tx=0x7f3a60008040 comp rx=0 tx=0).stop 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3a6c1075f0 0x7f3a6c10b4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a6c102510 0x7f3a6c10eef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 --2- 172.21.15.28:0/2012001958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3a6c102ec0 0x7f3a6c10af90 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:02.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 >> 172.21.15.28:0/2012001958 conn(0x7f3a6c0fdda0 msgr2=0x7f3a6c100120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:02.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 shutdown_connections 2024-05-20T20:11:02.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:02.162+0000 7f3a72e00700 1 -- 172.21.15.28:0/2012001958 wait complete. 2024-05-20T20:11:03.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:03 smithi028 ceph-mon[31178]: pgmap v165: 97 pgs: 97 active+clean; 27 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 180 MiB/s wr, 951 op/s 2024-05-20T20:11:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:03 smithi154 ceph-mon[36314]: pgmap v165: 97 pgs: 97 active+clean; 27 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 180 MiB/s wr, 951 op/s 2024-05-20T20:11:03.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:03 smithi033 ceph-mon[36179]: pgmap v165: 97 pgs: 97 active+clean; 27 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 180 MiB/s wr, 951 op/s 2024-05-20T20:11:04.000 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:11:04.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:04 smithi028 ceph-mon[31178]: pgmap v166: 97 pgs: 97 active+clean; 27 GiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 259 MiB/s wr, 1.52k op/s 2024-05-20T20:11:04.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:04 smithi154 ceph-mon[36314]: pgmap v166: 97 pgs: 97 active+clean; 27 GiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 259 MiB/s wr, 1.52k op/s 2024-05-20T20:11:04.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:04 smithi033 ceph-mon[36179]: pgmap v166: 97 pgs: 97 active+clean; 27 GiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 259 MiB/s wr, 1.52k op/s 2024-05-20T20:11:04.708 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:05.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.347+0000 7fd82ae00640 1 -- 172.21.15.28:0/3657135093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8241138d0 msgr2=0x7fd824115cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.347+0000 7fd82ae00640 1 --2- 172.21.15.28:0/3657135093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8241138d0 0x7fd824115cc0 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7fd81c009cd0 tx=0x7fd81c02a8e0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.349+0000 7fd82ae00640 1 -- 172.21.15.28:0/3657135093 shutdown_connections 2024-05-20T20:11:05.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.349+0000 7fd82ae00640 1 --2- 172.21.15.28:0/3657135093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8241138d0 0x7fd824115cc0 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.349+0000 7fd82ae00640 1 --2- 172.21.15.28:0/3657135093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8240778a0 0x7fd824113390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.349+0000 7fd82ae00640 1 --2- 172.21.15.28:0/3657135093 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd824076f80 0x7fd824077360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.349+0000 7fd82ae00640 1 -- 172.21.15.28:0/3657135093 >> 172.21.15.28:0/3657135093 conn(0x7fd824100740 msgr2=0x7fd824102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:05.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.349+0000 7fd82ae00640 1 -- 172.21.15.28:0/3657135093 shutdown_connections 2024-05-20T20:11:05.351 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.350+0000 7fd82ae00640 1 -- 172.21.15.28:0/3657135093 wait complete. 2024-05-20T20:11:05.351 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.351+0000 7fd82ae00640 1 Processor -- start 2024-05-20T20:11:05.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.351+0000 7fd82ae00640 1 -- start start 2024-05-20T20:11:05.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.352+0000 7fd82ae00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd824076f80 0x7fd8240741f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.352+0000 7fd82ae00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8240778a0 0x7fd824074730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.352+0000 7fd82ae00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8241138d0 0x7fd82406d4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.352+0000 7fd82ae00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd824119340 con 0x7fd824076f80 2024-05-20T20:11:05.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.352+0000 7fd82ae00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd8241191c0 con 0x7fd8240778a0 2024-05-20T20:11:05.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.352+0000 7fd82ae00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd8241194c0 con 0x7fd8241138d0 2024-05-20T20:11:05.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.353+0000 7fd828e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8241138d0 0x7fd82406d4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.353+0000 7fd828e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8241138d0 0x7fd82406d4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41268/0 (socket says 172.21.15.28:41268) 2024-05-20T20:11:05.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.353+0000 7fd828e00640 1 -- 172.21.15.28:0/1982015692 learned_addr learned my addr 172.21.15.28:0/1982015692 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:05.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.353+0000 7fd823400640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8240778a0 0x7fd824074730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.354+0000 7fd828e00640 1 -- 172.21.15.28:0/1982015692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8240778a0 msgr2=0x7fd824074730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.354+0000 7fd828e00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8240778a0 0x7fd824074730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.354+0000 7fd828e00640 1 -- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd824076f80 msgr2=0x7fd8240741f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.354+0000 7fd828e00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd824076f80 0x7fd8240741f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.354+0000 7fd828e00640 1 -- 172.21.15.28:0/1982015692 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd82406dd60 con 0x7fd8241138d0 2024-05-20T20:11:05.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.354+0000 7fd828e00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8241138d0 0x7fd82406d4a0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fd81c031e40 tx=0x7fd81c003970 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:05.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.354+0000 7fd820c00640 1 -- 172.21.15.28:0/1982015692 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd81c004040 con 0x7fd8241138d0 2024-05-20T20:11:05.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.355+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd82406dff0 con 0x7fd8241138d0 2024-05-20T20:11:05.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.355+0000 7fd820c00640 1 -- 172.21.15.28:0/1982015692 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd81c004690 con 0x7fd8241138d0 2024-05-20T20:11:05.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.355+0000 7fd820c00640 1 -- 172.21.15.28:0/1982015692 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd81c004040 con 0x7fd8241138d0 2024-05-20T20:11:05.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.355+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd824072000 con 0x7fd8241138d0 2024-05-20T20:11:05.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.363+0000 7fd82dbf1740 1 Processor -- start 2024-05-20T20:11:05.368 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.367+0000 7fd820c00640 1 -- 172.21.15.28:0/1982015692 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd81c030d60 con 0x7fd8241138d0 2024-05-20T20:11:05.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.368+0000 7fd82dbf1740 1 -- start start 2024-05-20T20:11:05.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f4c90 0x56240b6f5070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f55b0 0x56240b6ff930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6ffe70 0x56240b702260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd82dbf1740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56240b665b90 con 0x56240b6ffe70 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd82dbf1740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56240b655730 con 0x56240b6f4c90 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd82dbf1740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56240b659770 con 0x56240b6f55b0 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd823400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f4c90 0x56240b6f5070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd823400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f4c90 0x56240b6f5070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53464/0 (socket says 172.21.15.28:53464) 2024-05-20T20:11:05.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.369+0000 7fd823400640 1 -- 172.21.15.28:0/1436128881 learned_addr learned my addr 172.21.15.28:0/1436128881 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:05.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.370+0000 7fd820c00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8080b2c80 0x7fd8080b5140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.370+0000 7fd823400640 1 -- 172.21.15.28:0/1436128881 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f55b0 msgr2=0x56240b6ff930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:05.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.370+0000 7fd823400640 1 --2- 172.21.15.28:0/1436128881 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f55b0 0x56240b6ff930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.370+0000 7fd823400640 1 -- 172.21.15.28:0/1436128881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6ffe70 msgr2=0x56240b702260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:05.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.370+0000 7fd823400640 1 --2- 172.21.15.28:0/1436128881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6ffe70 0x56240b702260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.370+0000 7fd823400640 1 -- 172.21.15.28:0/1436128881 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56240b5ee8b0 con 0x56240b6f4c90 2024-05-20T20:11:05.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.371+0000 7fd820c00640 1 -- 172.21.15.28:0/1982015692 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd81c02e030 con 0x7fd8241138d0 2024-05-20T20:11:05.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.371+0000 7fd823400640 1 --2- 172.21.15.28:0/1436128881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f4c90 0x56240b6f5070 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fd8100058c0 tx=0x7fd810011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=81b28d71dc0047e3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:05.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.372+0000 7fd819600640 1 -- 172.21.15.28:0/1436128881 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd810009da0 con 0x56240b6f4c90 2024-05-20T20:11:05.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.372+0000 7fd819600640 1 -- 172.21.15.28:0/1436128881 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd8100054d0 con 0x56240b6f4c90 2024-05-20T20:11:05.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.372+0000 7fd819600640 1 -- 172.21.15.28:0/1436128881 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd810009da0 con 0x56240b6f4c90 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.372+0000 7fd828e00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8080b2c80 0x7fd8080b5140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1436128881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f4c90 msgr2=0x56240b6f5070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1436128881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f4c90 0x56240b6f5070 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fd8100058c0 tx=0x7fd810011810 comp rx=0 tx=0).stop 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1436128881 shutdown_connections 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1436128881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6ffe70 0x56240b702260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1436128881 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f55b0 0x56240b6ff930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1436128881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f4c90 0x56240b6f5070 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1436128881 >> 172.21.15.28:0/1436128881 conn(0x56240b6f11a0 msgr2=0x56240b6f1580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:05.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1436128881 shutdown_connections 2024-05-20T20:11:05.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.374+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1436128881 wait complete. 2024-05-20T20:11:05.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.375+0000 7fd82dbf1740 1 -- start start 2024-05-20T20:11:05.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.375+0000 7fd82dbf1740 1 Processor -- start 2024-05-20T20:11:05.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.376+0000 7fd828e00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8080b2c80 0x7fd8080b5140 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fd81c0c7040 tx=0x7fd81c03d040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:05.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.376+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b6ff8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.376+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b6fcb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.376+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 0x56240b6fd2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.376+0000 7fd82dbf1740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56240b33b790 con 0x56240b6f4c90 2024-05-20T20:11:05.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.377+0000 7fd82dbf1740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56240b5fdbc0 con 0x56240b6f55b0 2024-05-20T20:11:05.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.377+0000 7fd82dbf1740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56240b702aa0 con 0x56240b6ffe70 2024-05-20T20:11:05.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.377+0000 7fd82dbf1740 10 client.0 _ll_register_callbacks cb 0x56240b5db400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:05.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.378+0000 7fd823e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b6fcb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.378+0000 7fd823e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b6fcb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53478/0 (socket says 172.21.15.28:53478) 2024-05-20T20:11:05.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.378+0000 7fd823e00640 1 -- 172.21.15.28:0/1651048259 learned_addr learned my addr 172.21.15.28:0/1651048259 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:05.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.379+0000 7fd823400640 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 0x56240b6fd2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.379+0000 7fd823400640 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b6ff8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.380+0000 7fd823e00640 1 -- 172.21.15.28:0/1651048259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 msgr2=0x56240b6fd2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:05.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.380+0000 7fd823e00640 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 0x56240b6fd2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.380+0000 7fd823e00640 1 -- 172.21.15.28:0/1651048259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 msgr2=0x56240b6ff8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:05.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.380+0000 7fd823e00640 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b6ff8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.380+0000 7fd823e00640 1 -- 172.21.15.28:0/1651048259 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56240b36a210 con 0x56240b6f55b0 2024-05-20T20:11:05.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.380+0000 7fd823e00640 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b6fcb90 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fd814009e80 tx=0x7fd81400b4b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fdc4a470fb01321a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:05.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.381+0000 7fd80e000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:05.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.381+0000 7fd80e000640 1 -- 172.21.15.28:0/1651048259 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd814015020 con 0x56240b6f55b0 2024-05-20T20:11:05.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.381+0000 7fd80e000640 1 -- 172.21.15.28:0/1651048259 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd814011620 con 0x56240b6f55b0 2024-05-20T20:11:05.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.381+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd8100054d0 con 0x56240b6f55b0 2024-05-20T20:11:05.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.382+0000 7fd80e000640 1 -- 172.21.15.28:0/1651048259 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd81400be30 con 0x56240b6f55b0 2024-05-20T20:11:05.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.382+0000 7fd82dbf1740 10 client.26597 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:05.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.382+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56240b436620 con 0x56240b6f55b0 2024-05-20T20:11:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd80e000640 1 -- 172.21.15.28:0/1651048259 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd8100054d0 con 0x56240b6f55b0 2024-05-20T20:11:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd80e000640 1 -- 172.21.15.28:0/1651048259 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd814022020 con 0x56240b6f55b0 2024-05-20T20:11:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd82dbf1740 10 client.26597 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:05.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd82dbf1740 10 client.26597 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd82dbf1740 20 client.26597 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b720990 0x56240b722da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd82dbf1740 4 client.26597 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.383+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56240b667ae0 con 0x56240b720990 2024-05-20T20:11:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.384+0000 7fd823400640 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b720990 0x56240b722da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.385+0000 7fd823400640 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b720990 0x56240b722da0 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c8078aa92e25ed85 server_cookie=60c25cb76da21d64 in_seq=0 out_seq=0 2024-05-20T20:11:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.386+0000 7fd80e000640 10 client.26597 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:05.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.386+0000 7fd80e000640 1 -- 172.21.15.28:0/1651048259 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56240b667ae0 con 0x56240b720990 2024-05-20T20:11:05.387 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.386+0000 7fd80e000640 10 client.26597 handle_command_reply: tid=0 2024-05-20T20:11:05.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.386+0000 7fd82dbf1740 1 client.26597 shutdown 2024-05-20T20:11:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.394+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 msgr2=0x56240b6fcb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.394+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b6fcb90 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fd814009e80 tx=0x7fd81400b4b0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.402 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.398+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 shutdown_connections 2024-05-20T20:11:05.402 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.398+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b720990 0x56240b722da0 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.402 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.398+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b6fcb90 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.402 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.398+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 0x56240b6fd2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.398+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/1651048259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b6ff8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.398+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 >> 172.21.15.28:0/1651048259 conn(0x56240b6f11a0 msgr2=0x56240b6f6ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:05.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.398+0000 7fd80e000640 0 client.26597 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:05.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.405+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 shutdown_connections 2024-05-20T20:11:05.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.405+0000 7fd82dbf1740 1 -- 172.21.15.28:0/1651048259 wait complete. 2024-05-20T20:11:05.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.405+0000 7fd82dbf1740 20 client.26597 trim_cache size 0 max 16384 2024-05-20T20:11:05.434 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:05 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:11:05.473 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.473+0000 7fd82dbf1740 1 Processor -- start 2024-05-20T20:11:05.474 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.473+0000 7fd82dbf1740 1 -- start start 2024-05-20T20:11:05.474 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.473+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f4c90 0x56240b6f8160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.475 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.474+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f55b0 0x56240b6ff3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.475 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.474+0000 7fd823400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f4c90 0x56240b6f8160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.475 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.474+0000 7fd823400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f4c90 0x56240b6f8160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41282/0 (socket says 172.21.15.28:41282) 2024-05-20T20:11:05.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.474+0000 7fd823400640 1 -- 172.21.15.28:0/393633643 learned_addr learned my addr 172.21.15.28:0/393633643 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:05.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.475+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6ffe70 0x56240b7286b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.475+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd82410dfc0 con 0x56240b6f55b0 2024-05-20T20:11:05.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.475+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd8241b66c0 con 0x56240b6ffe70 2024-05-20T20:11:05.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.475+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd82410e690 con 0x56240b6f4c90 2024-05-20T20:11:05.477 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.476+0000 7fd823e00640 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f55b0 0x56240b6ff3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.477 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.477+0000 7fd823e00640 1 -- 172.21.15.28:0/393633643 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f4c90 msgr2=0x56240b6f8160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:05.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.477+0000 7fd823e00640 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f4c90 0x56240b6f8160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.477+0000 7fd823e00640 1 -- 172.21.15.28:0/393633643 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6ffe70 msgr2=0x56240b7286b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:05.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.477+0000 7fd823e00640 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6ffe70 0x56240b7286b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.477+0000 7fd823e00640 1 -- 172.21.15.28:0/393633643 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56240b6fd640 con 0x56240b6f55b0 2024-05-20T20:11:05.479 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.478+0000 7fd823e00640 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f55b0 0x56240b6ff3d0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7fd81401ad80 tx=0x7fd81400b9b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=85534f01ab84ef3c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:05.483 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.480+0000 7fd818c00640 1 -- 172.21.15.28:0/393633643 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd814015070 con 0x56240b6f55b0 2024-05-20T20:11:05.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.480+0000 7fd818c00640 1 -- 172.21.15.28:0/393633643 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd814006c50 con 0x56240b6f55b0 2024-05-20T20:11:05.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.481+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f55b0 msgr2=0x56240b6ff3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.482+0000 7fd818c00640 1 -- 172.21.15.28:0/393633643 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd814015070 con 0x56240b6f55b0 2024-05-20T20:11:05.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.482+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f55b0 0x56240b6ff3d0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7fd81401ad80 tx=0x7fd81400b9b0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.482+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 shutdown_connections 2024-05-20T20:11:05.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.483+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6ffe70 0x56240b7286b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.483+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f55b0 0x56240b6ff3d0 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.483+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/393633643 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6f4c90 0x56240b6f8160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.483+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 >> 172.21.15.28:0/393633643 conn(0x56240b6f11a0 msgr2=0x56240b6f1580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:05.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.484+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 shutdown_connections 2024-05-20T20:11:05.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.484+0000 7fd82dbf1740 1 -- 172.21.15.28:0/393633643 wait complete. 2024-05-20T20:11:05.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.484+0000 7fd82dbf1740 1 -- start start 2024-05-20T20:11:05.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.485+0000 7fd82dbf1740 1 Processor -- start 2024-05-20T20:11:05.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.486+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b7083a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.487+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b703710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.487+0000 7fd82dbf1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 0x56240b703c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.487+0000 7fd823e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b703710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.488+0000 7fd823e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b703710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53504/0 (socket says 172.21.15.28:53504) 2024-05-20T20:11:05.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.488+0000 7fd823e00640 1 -- 172.21.15.28:0/3962905749 learned_addr learned my addr 172.21.15.28:0/3962905749 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:05.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.489+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd8241b5ff0 con 0x56240b6f4c90 2024-05-20T20:11:05.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.489+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd8241b5920 con 0x56240b6f55b0 2024-05-20T20:11:05.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.489+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd8240507e0 con 0x56240b6ffe70 2024-05-20T20:11:05.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.489+0000 7fd82dbf1740 10 client.0 _ll_register_callbacks cb 0x7fd814009a10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:05.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.489+0000 7fd823400640 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b7083a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.490+0000 7fd823e00640 1 -- 172.21.15.28:0/3962905749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 msgr2=0x56240b703c50 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:05.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.490+0000 7fd823e00640 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 0x56240b703c50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.490+0000 7fd823e00640 1 -- 172.21.15.28:0/3962905749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 msgr2=0x56240b7083a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:05.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.490+0000 7fd823e00640 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b7083a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.490+0000 7fd823e00640 1 -- 172.21.15.28:0/3962905749 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56240b704460 con 0x56240b6f55b0 2024-05-20T20:11:05.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.490+0000 7fd823400640 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b7083a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:05.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.491+0000 7fd823e00640 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b703710 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fd81400b000 tx=0x7fd814019780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ae6b1e7cbb934596 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:05.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.491+0000 7fd80e000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:05.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.499+0000 7fd80e000640 1 -- 172.21.15.28:0/3962905749 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd814015040 con 0x56240b6f55b0 2024-05-20T20:11:05.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.499+0000 7fd80e000640 1 -- 172.21.15.28:0/3962905749 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fd814019dc0 con 0x56240b6f55b0 2024-05-20T20:11:05.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.499+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd814006c50 con 0x56240b6f55b0 2024-05-20T20:11:05.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.500+0000 7fd80e000640 1 -- 172.21.15.28:0/3962905749 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd81401a3f0 con 0x56240b6f55b0 2024-05-20T20:11:05.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.500+0000 7fd82dbf1740 10 client.26600 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:05.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.500+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56240b78b560 con 0x56240b6f55b0 2024-05-20T20:11:05.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.503+0000 7fd80e000640 1 -- 172.21.15.28:0/3962905749 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd814006c50 con 0x56240b6f55b0 2024-05-20T20:11:05.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.504+0000 7fd80e000640 1 -- 172.21.15.28:0/3962905749 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd814022070 con 0x56240b6f55b0 2024-05-20T20:11:05.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.504+0000 7fd82dbf1740 10 client.26600 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.504+0000 7fd82dbf1740 10 client.26600 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.504+0000 7fd82dbf1740 20 client.26600 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.504+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b78bb60 0x56240b78df50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.504+0000 7fd82dbf1740 4 client.26600 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.504+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56240b6612c0 con 0x56240b78bb60 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.516+0000 7fd823400640 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b78bb60 0x56240b78df50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.517+0000 7fd823400640 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b78bb60 0x56240b78df50 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a5dce080f8bd9f9d server_cookie=3cc459518a7747b0 in_seq=0 out_seq=0 2024-05-20T20:11:05.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.517+0000 7fd80e000640 10 client.26600 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:05.557 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.556+0000 7fd80e000640 1 -- 172.21.15.28:0/3962905749 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x56240b6612c0 con 0x56240b78bb60 2024-05-20T20:11:05.557 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.556+0000 7fd80e000640 10 client.26600 handle_command_reply: tid=0 2024-05-20T20:11:05.557 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.556+0000 7fd82dbf1740 1 client.26600 shutdown 2024-05-20T20:11:05.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.557+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 msgr2=0x56240b703710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.557+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b703710 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fd81400b000 tx=0x7fd814019780 comp rx=0 tx=0).stop 2024-05-20T20:11:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.558+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 shutdown_connections 2024-05-20T20:11:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.558+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56240b78bb60 0x56240b78df50 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.558+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56240b6f55b0 0x56240b703710 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.558+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56240b6ffe70 0x56240b703c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.558+0000 7fd82dbf1740 1 --2- 172.21.15.28:0/3962905749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56240b6f4c90 0x56240b7083a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.558+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 >> 172.21.15.28:0/3962905749 conn(0x56240b6f11a0 msgr2=0x56240b6f0310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:05.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.558+0000 7fd80e000640 0 client.26600 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:11:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:11:05.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:11:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.559+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 shutdown_connections 2024-05-20T20:11:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.559+0000 7fd82dbf1740 1 -- 172.21.15.28:0/3962905749 wait complete. 2024-05-20T20:11:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.559+0000 7fd82dbf1740 20 client.26600 trim_cache size 0 max 16384 2024-05-20T20:11:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.562+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8080b2c80 msgr2=0x7fd8080b5140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.562+0000 7fd82ae00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8080b2c80 0x7fd8080b5140 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fd81c0c7040 tx=0x7fd81c03d040 comp rx=0 tx=0).stop 2024-05-20T20:11:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.562+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8241138d0 msgr2=0x7fd82406d4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:05.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.562+0000 7fd82ae00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8241138d0 0x7fd82406d4a0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fd81c031e40 tx=0x7fd81c003970 comp rx=0 tx=0).stop 2024-05-20T20:11:05.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 shutdown_connections 2024-05-20T20:11:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd8240778a0 0x7fd824074730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8080b2c80 0x7fd8080b5140 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd8241138d0 0x7fd82406d4a0 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 --2- 172.21.15.28:0/1982015692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd824076f80 0x7fd8240741f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 >> 172.21.15.28:0/1982015692 conn(0x7fd824100740 msgr2=0x7fd824102b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 shutdown_connections 2024-05-20T20:11:05.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:05.564+0000 7fd82ae00640 1 -- 172.21.15.28:0/1982015692 wait complete. 2024-05-20T20:11:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:05 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:11:05.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:05 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:11:06.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:06 smithi028 ceph-mon[31178]: pgmap v167: 97 pgs: 97 active+clean; 27 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 259 MiB/s wr, 1.58k op/s 2024-05-20T20:11:06.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:06 smithi154 ceph-mon[36314]: pgmap v167: 97 pgs: 97 active+clean; 27 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 259 MiB/s wr, 1.58k op/s 2024-05-20T20:11:06.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:06 smithi033 ceph-mon[36179]: pgmap v167: 97 pgs: 97 active+clean; 27 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 259 MiB/s wr, 1.58k op/s 2024-05-20T20:11:07.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:07 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:11:07.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:07 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:11:07.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:07 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:11:07.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:07 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:11:07.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:07 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:11:07.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:07 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:11:08.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:07 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:11:08.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:07 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:11:08.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:07 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:11:08.672 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:08 smithi028 ceph-mon[31178]: pgmap v168: 97 pgs: 97 active+clean; 27 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 238 MiB/s wr, 1.46k op/s 2024-05-20T20:11:08.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:08 smithi154 ceph-mon[36314]: pgmap v168: 97 pgs: 97 active+clean; 27 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 238 MiB/s wr, 1.46k op/s 2024-05-20T20:11:08.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 -- 172.21.15.28:0/1875744418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a8009a70 msgr2=0x7f22a80a6690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1875744418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a8009a70 0x7f22a80a6690 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f229c0098f0 tx=0x7f229c009c00 comp rx=0 tx=0).stop 2024-05-20T20:11:08.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 -- 172.21.15.28:0/1875744418 shutdown_connections 2024-05-20T20:11:08.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1875744418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f22a80a6bd0 0x7f22a80a8fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1875744418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f22a8009150 0x7f22a8009530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1875744418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a8009a70 0x7f22a80a6690 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 -- 172.21.15.28:0/1875744418 >> 172.21.15.28:0/1875744418 conn(0x7f22a8093a20 msgr2=0x7f22a8095e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:08.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 -- 172.21.15.28:0/1875744418 shutdown_connections 2024-05-20T20:11:08.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.873+0000 7f22b4c00700 1 -- 172.21.15.28:0/1875744418 wait complete. 2024-05-20T20:11:08.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.875+0000 7f22b4c00700 1 Processor -- start 2024-05-20T20:11:08.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:08 smithi033 ceph-mon[36179]: pgmap v168: 97 pgs: 97 active+clean; 27 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 238 MiB/s wr, 1.46k op/s 2024-05-20T20:11:08.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.876+0000 7f22b4c00700 1 -- start start 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.876+0000 7f22b4c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f22a8009150 0x7f22a81387b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.876+0000 7f22b4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f22a8009a70 0x7f22a8138cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22b4c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a80a6bd0 0x7f22a8132970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22b4c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f22a80ac590 con 0x7f22a8009150 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22b4c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f22a80ac410 con 0x7f22a80a6bd0 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22b4c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f22a80ac710 con 0x7f22a8009a70 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22af400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f22a8009150 0x7f22a81387b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22afe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a80a6bd0 0x7f22a8132970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22afe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a80a6bd0 0x7f22a8132970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58416/0 (socket says 172.21.15.28:58416) 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22afe00700 1 -- 172.21.15.28:0/1733479813 learned_addr learned my addr 172.21.15.28:0/1733479813 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22aea00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f22a8009a70 0x7f22a8138cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22afe00700 1 -- 172.21.15.28:0/1733479813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f22a8009a70 msgr2=0x7f22a8138cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22afe00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f22a8009a70 0x7f22a8138cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.877+0000 7f22afe00700 1 -- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f22a8009150 msgr2=0x7f22a81387b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.878+0000 7f22afe00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f22a8009150 0x7f22a81387b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.878+0000 7f22afe00700 1 -- 172.21.15.28:0/1733479813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f22a81330d0 con 0x7f22a80a6bd0 2024-05-20T20:11:08.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.878+0000 7f22af400700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f22a8009150 0x7f22a81387b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:08.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.878+0000 7f22afe00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a80a6bd0 0x7f22a8132970 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f22b0052080 tx=0x7f22b0075cf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:08.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.879+0000 7f22a3e00700 1 -- 172.21.15.28:0/1733479813 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22b0084070 con 0x7f22a80a6bd0 2024-05-20T20:11:08.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.879+0000 7f22a3e00700 1 -- 172.21.15.28:0/1733479813 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f22b006be20 con 0x7f22a80a6bd0 2024-05-20T20:11:08.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.879+0000 7f22a3e00700 1 -- 172.21.15.28:0/1733479813 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22b007f830 con 0x7f22a80a6bd0 2024-05-20T20:11:08.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.879+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f22a81333c0 con 0x7f22a80a6bd0 2024-05-20T20:11:08.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.879+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f22a813f680 con 0x7f22a80a6bd0 2024-05-20T20:11:08.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.885+0000 7f22a3e00700 1 -- 172.21.15.28:0/1733479813 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f22b0087070 con 0x7f22a80a6bd0 2024-05-20T20:11:08.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.887+0000 7f22a3e00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f22980b2b20 0x7f22980b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.888+0000 7f22a3e00700 1 -- 172.21.15.28:0/1733479813 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f22b0150570 con 0x7f22a80a6bd0 2024-05-20T20:11:08.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.888+0000 7f22af400700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f22980b2b20 0x7f22980b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.888+0000 7f22c0a94b80 1 Processor -- start 2024-05-20T20:11:08.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.888+0000 7f22c0a94b80 1 -- start start 2024-05-20T20:11:08.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f92ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f8cd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f8d7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560350eeae40 con 0x560350f927c0 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x560350eda9e0 con 0x560350f930e0 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x560350edea20 con 0x560350f8d390 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22aea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f8cd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22aea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f8cd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58428/0 (socket says 172.21.15.28:58428) 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22aea00700 1 -- 172.21.15.28:0/3255146810 learned_addr learned my addr 172.21.15.28:0/3255146810 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.889+0000 7f22afe00700 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f8d7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22af400700 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f92ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22aea00700 1 -- 172.21.15.28:0/3255146810 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 msgr2=0x560350f8d7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:08.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22aea00700 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f8d7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22aea00700 1 -- 172.21.15.28:0/3255146810 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 msgr2=0x560350f92ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22aea00700 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f92ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22aea00700 1 -- 172.21.15.28:0/3255146810 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560350ea3420 con 0x560350f930e0 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22afe00700 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f8d7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22af400700 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f92ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22af400700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f22980b2b20 0x7f22980b4fe0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f22a8133ab0 tx=0x7f22a4009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.890+0000 7f22aea00700 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f8cd70 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f229c0098f0 tx=0x7f229c009f50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e9c67fb9a6c09a0d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.891+0000 7f22a0c00700 1 -- 172.21.15.28:0/3255146810 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f229c026070 con 0x560350f930e0 2024-05-20T20:11:08.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.891+0000 7f22a0c00700 1 -- 172.21.15.28:0/3255146810 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f229c0054f0 con 0x560350f930e0 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.891+0000 7f22a0c00700 1 -- 172.21.15.28:0/3255146810 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f229c0039d0 con 0x560350f930e0 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 -- 172.21.15.28:0/3255146810 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 msgr2=0x560350f8cd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f8cd70 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f229c0098f0 tx=0x7f229c009f50 comp rx=0 tx=0).stop 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 -- 172.21.15.28:0/3255146810 shutdown_connections 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f8d7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f92ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/3255146810 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f8cd70 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.894 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 -- 172.21.15.28:0/3255146810 >> 172.21.15.28:0/3255146810 conn(0x560350ebabe0 msgr2=0x560350f87810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:08.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 -- 172.21.15.28:0/3255146810 shutdown_connections 2024-05-20T20:11:08.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.892+0000 7f22c0a94b80 1 -- 172.21.15.28:0/3255146810 wait complete. 2024-05-20T20:11:08.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.893+0000 7f22c0a94b80 1 -- start start 2024-05-20T20:11:08.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.893+0000 7f22c0a94b80 1 Processor -- start 2024-05-20T20:11:08.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.894+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f98dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.894+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f961b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.894+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f966f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.895+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560350de0fa0 con 0x560350f927c0 2024-05-20T20:11:08.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.895+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x560350e7b760 con 0x560350f930e0 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.895+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x560350f99ba0 con 0x560350f8d390 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.895+0000 7f22aea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f961b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.896+0000 7f22c0a94b80 10 client.0 _ll_register_callbacks cb 0x560350e84c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.896+0000 7f22afe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f966f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.896+0000 7f22afe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f966f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58434/0 (socket says 172.21.15.28:58434) 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.896+0000 7f22afe00700 1 -- 172.21.15.28:0/4070273169 learned_addr learned my addr 172.21.15.28:0/4070273169 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.896+0000 7f22afe00700 1 -- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 msgr2=0x560350f98dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.896+0000 7f22af400700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f98dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.897+0000 7f22afe00700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f98dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.897+0000 7f22afe00700 1 -- 172.21.15.28:0/4070273169 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 msgr2=0x560350f961b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.897+0000 7f22afe00700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f961b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.897+0000 7f22afe00700 1 -- 172.21.15.28:0/4070273169 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560350e2a8d0 con 0x560350f930e0 2024-05-20T20:11:08.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.898+0000 7f22afe00700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f966f0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f22b0113100 tx=0x7f22b0117fb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=86e5fb212dfed074 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:08.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.898+0000 7f2295600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:08.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.898+0000 7f22aea00700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f961b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:08.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.898+0000 7f2295600700 1 -- 172.21.15.28:0/4070273169 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22b0131060 con 0x560350f930e0 2024-05-20T20:11:08.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.898+0000 7f22af400700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f98dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:08.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.898+0000 7f2295600700 1 -- 172.21.15.28:0/4070273169 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f22b014def0 con 0x560350f930e0 2024-05-20T20:11:08.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.899+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f229c0054f0 con 0x560350f930e0 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.899+0000 7f2295600700 1 -- 172.21.15.28:0/4070273169 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f22b008e040 con 0x560350f930e0 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.899+0000 7f22c0a94b80 10 client.26612 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.899+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560350eab060 con 0x560350f930e0 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f2295600700 1 -- 172.21.15.28:0/4070273169 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f229c0054f0 con 0x560350f930e0 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f2295600700 1 -- 172.21.15.28:0/4070273169 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f22b0137070 con 0x560350f930e0 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f22c0a94b80 10 client.26612 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f22c0a94b80 10 client.26612 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f22c0a94b80 20 client.26612 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560350fa67a0 0x560350fa8b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f22c0a94b80 4 client.26612 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:08.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x560350eecd90 con 0x560350fa67a0 2024-05-20T20:11:08.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.900+0000 7f22af400700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560350fa67a0 0x560350fa8b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.901+0000 7f22af400700 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560350fa67a0 0x560350fa8b90 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=965397f03aec62c3 server_cookie=2038c36b5c68e247 in_seq=0 out_seq=0 2024-05-20T20:11:08.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.901+0000 7f2295600700 10 client.26612 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:11:08.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.902+0000 7f2295600700 1 -- 172.21.15.28:0/4070273169 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x560350eecd90 con 0x560350fa67a0 2024-05-20T20:11:08.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.902+0000 7f2295600700 10 client.26612 handle_command_reply: tid=0 2024-05-20T20:11:08.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.902+0000 7f22c0a94b80 1 client.26612 shutdown 2024-05-20T20:11:08.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.904+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 msgr2=0x560350f966f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.904+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f966f0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f22b0113100 tx=0x7f22b0117fb0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.905+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 shutdown_connections 2024-05-20T20:11:08.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.905+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f930e0 0x560350f966f0 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.905+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f98dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.905+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560350fa67a0 0x560350fa8b90 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.905+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/4070273169 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f961b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.905+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 >> 172.21.15.28:0/4070273169 conn(0x560350ebabe0 msgr2=0x560350f86fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:08.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.905+0000 7f2295600700 0 client.26612 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:11:08.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.906+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 shutdown_connections 2024-05-20T20:11:08.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.907+0000 7f22c0a94b80 1 -- 172.21.15.28:0/4070273169 wait complete. 2024-05-20T20:11:08.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.907+0000 7f22c0a94b80 20 client.26612 trim_cache size 0 max 16384 2024-05-20T20:11:08.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.934+0000 7f22c0a94b80 1 Processor -- start 2024-05-20T20:11:08.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.935+0000 7f22c0a94b80 1 -- start start 2024-05-20T20:11:08.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f8d390 0x560350f8cd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f908b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f930e0 0x560350f93c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f22a8146bc0 con 0x560350f927c0 2024-05-20T20:11:08.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f22a8148700 con 0x560350f8d390 2024-05-20T20:11:08.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f22a8147290 con 0x560350f930e0 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22af400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f8d390 0x560350f8cd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22afe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f930e0 0x560350f93c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22afe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f930e0 0x560350f93c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37052/0 (socket says 172.21.15.28:37052) 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.936+0000 7f22afe00700 1 -- 172.21.15.28:0/1071309743 learned_addr learned my addr 172.21.15.28:0/1071309743 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.937+0000 7f22afe00700 1 -- 172.21.15.28:0/1071309743 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f8d390 msgr2=0x560350f8cd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.937+0000 7f22afe00700 1 --2- 172.21.15.28:0/1071309743 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f8d390 0x560350f8cd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.937+0000 7f22afe00700 1 -- 172.21.15.28:0/1071309743 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 msgr2=0x560350f908b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.937+0000 7f22afe00700 1 --2- 172.21.15.28:0/1071309743 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f908b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.937+0000 7f22afe00700 1 -- 172.21.15.28:0/1071309743 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x560350fa9210 con 0x560350f930e0 2024-05-20T20:11:08.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.937+0000 7f22afe00700 1 --2- 172.21.15.28:0/1071309743 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f930e0 0x560350f93c90 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f22b0114a70 tx=0x7f22b01137f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f9edd1c6f11b4b06 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:08.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.937+0000 7f2297e00700 1 -- 172.21.15.28:0/1071309743 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22b0117a60 con 0x560350f930e0 2024-05-20T20:11:08.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.938+0000 7f2297e00700 1 -- 172.21.15.28:0/1071309743 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f22b0117c00 con 0x560350f930e0 2024-05-20T20:11:08.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.938+0000 7f2297e00700 1 -- 172.21.15.28:0/1071309743 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22b00ff340 con 0x560350f930e0 2024-05-20T20:11:08.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.938+0000 7f22c0a94b80 1 -- 172.21.15.28:0/1071309743 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f930e0 msgr2=0x560350f93c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.938+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/1071309743 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f930e0 0x560350f93c90 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f22b0114a70 tx=0x7f22b01137f0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.939+0000 7f22c0a94b80 1 -- 172.21.15.28:0/1071309743 shutdown_connections 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.939+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/1071309743 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f930e0 0x560350f93c90 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.939+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/1071309743 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f8d390 0x560350f8cd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.939+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/1071309743 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f927c0 0x560350f908b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.939+0000 7f22c0a94b80 1 -- 172.21.15.28:0/1071309743 >> 172.21.15.28:0/1071309743 conn(0x560350ebabe0 msgr2=0x560350f93580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.939+0000 7f22c0a94b80 1 -- 172.21.15.28:0/1071309743 shutdown_connections 2024-05-20T20:11:08.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.939+0000 7f22c0a94b80 1 -- 172.21.15.28:0/1071309743 wait complete. 2024-05-20T20:11:08.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.940+0000 7f22c0a94b80 1 -- start start 2024-05-20T20:11:08.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.940+0000 7f22c0a94b80 1 Processor -- start 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.941+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f9de90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.941+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f927c0 0x560350f9e3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.941+0000 7f22c0a94b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f930e0 0x560350f9e910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.941+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f22a8148030 con 0x560350f930e0 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.941+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f22a8147960 con 0x560350f927c0 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.941+0000 7f22c0a94b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f22a8144b60 con 0x560350f8d390 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.941+0000 7f22c0a94b80 10 client.0 _ll_register_callbacks cb 0x560350e84c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22aea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f927c0 0x560350f9e3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f930e0 0x560350f9e910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f930e0 0x560350f9e910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43688/0 (socket says 172.21.15.28:43688) 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 -- 172.21.15.28:0/2188248477 learned_addr learned my addr 172.21.15.28:0/2188248477 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:08.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22af400700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f9de90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22aea00700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f927c0 0x560350f9e3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58446/0 (socket says 172.21.15.28:58446) 2024-05-20T20:11:08.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 -- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 msgr2=0x560350f9de90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:08.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f9de90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 -- 172.21.15.28:0/2188248477 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f927c0 msgr2=0x560350f9e3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:08.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f927c0 0x560350f9e3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.942+0000 7f22afe00700 1 -- 172.21.15.28:0/2188248477 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560350fa41c0 con 0x560350f930e0 2024-05-20T20:11:08.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.943+0000 7f22aea00700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f927c0 0x560350f9e3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:08.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.943+0000 7f22afe00700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f930e0 0x560350f9e910 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f22b0052c80 tx=0x7f22b0117fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=43eca6a8bbb1b27a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:08.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.943+0000 7f2295600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:11:08.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.943+0000 7f2295600700 1 -- 172.21.15.28:0/2188248477 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f22b0117a60 con 0x560350f930e0 2024-05-20T20:11:08.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.944+0000 7f2295600700 1 -- 172.21.15.28:0/2188248477 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f22b00ff2c0 con 0x560350f930e0 2024-05-20T20:11:08.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.944+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f22b0117c00 con 0x560350f930e0 2024-05-20T20:11:08.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.944+0000 7f2295600700 1 -- 172.21.15.28:0/2188248477 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f22b0150040 con 0x560350f930e0 2024-05-20T20:11:08.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.944+0000 7f22c0a94b80 10 client.16362 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:08.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.945+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560351022950 con 0x560350f930e0 2024-05-20T20:11:08.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.945+0000 7f2295600700 1 -- 172.21.15.28:0/2188248477 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f22b0117c00 con 0x560350f930e0 2024-05-20T20:11:08.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.945+0000 7f2295600700 1 -- 172.21.15.28:0/2188248477 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f22b0137070 con 0x560350f930e0 2024-05-20T20:11:08.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.946+0000 7f22c0a94b80 10 client.16362 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:08.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.946+0000 7f22c0a94b80 10 client.16362 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:11:08.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.946+0000 7f22c0a94b80 20 client.16362 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:08.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.946+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560351022b80 0x560351024fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:08.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.946+0000 7f22c0a94b80 4 client.16362 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:11:08.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.946+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x560350ee6570 con 0x560351022b80 2024-05-20T20:11:08.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.946+0000 7f22af400700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560351022b80 0x560351024fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:08.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.947+0000 7f22af400700 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560351022b80 0x560351024fa0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=8b805523feac630b server_cookie=7df705c3d15cc01c in_seq=0 out_seq=0 2024-05-20T20:11:08.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.947+0000 7f2295600700 10 client.16362 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:11:08.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f2295600700 1 -- 172.21.15.28:0/2188248477 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x560350ee6570 con 0x560351022b80 2024-05-20T20:11:08.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f2295600700 10 client.16362 handle_command_reply: tid=0 2024-05-20T20:11:08.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 client.16362 shutdown 2024-05-20T20:11:08.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f930e0 msgr2=0x560350f9e910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f930e0 0x560350f9e910 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f22b0052c80 tx=0x7f22b0117fb0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 shutdown_connections 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x560351022b80 0x560351024fa0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x560350f930e0 0x560350f9e910 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x560350f8d390 0x560350f9de90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 --2- 172.21.15.28:0/2188248477 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x560350f927c0 0x560350f9e3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 >> 172.21.15.28:0/2188248477 conn(0x560350ebabe0 msgr2=0x560350f912e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:08.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.948+0000 7f2295600700 0 client.16362 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:11:08.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.949+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 shutdown_connections 2024-05-20T20:11:08.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.949+0000 7f22c0a94b80 1 -- 172.21.15.28:0/2188248477 wait complete. 2024-05-20T20:11:08.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.949+0000 7f22c0a94b80 20 client.16362 trim_cache size 0 max 16384 2024-05-20T20:11:08.951 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:11:08.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.950+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f22980b2b20 msgr2=0x7f22980b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.950+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f22980b2b20 0x7f22980b4fe0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f22a8133ab0 tx=0x7f22a4009260 comp rx=0 tx=0).stop 2024-05-20T20:11:08.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.950+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a80a6bd0 msgr2=0x7f22a8132970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:08.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.950+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a80a6bd0 0x7f22a8132970 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f22b0052080 tx=0x7f22b0075cf0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.951+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 shutdown_connections 2024-05-20T20:11:08.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.951+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f22980b2b20 0x7f22980b4fe0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.951+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f22a80a6bd0 0x7f22a8132970 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.951+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f22a8009150 0x7f22a81387b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.951+0000 7f22b4c00700 1 --2- 172.21.15.28:0/1733479813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f22a8009a70 0x7f22a8138cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:08.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.951+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 >> 172.21.15.28:0/1733479813 conn(0x7f22a8093a20 msgr2=0x7f22a80a7840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:08.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.954+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 shutdown_connections 2024-05-20T20:11:08.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:08.954+0000 7f22b4c00700 1 -- 172.21.15.28:0/1733479813 wait complete. 2024-05-20T20:11:09.937 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:11:10.734 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:11.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:10 smithi154 ceph-mon[36314]: pgmap v169: 97 pgs: 97 active+clean; 28 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 354 MiB/s wr, 2.37k op/s 2024-05-20T20:11:11.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:10 smithi033 ceph-mon[36179]: pgmap v169: 97 pgs: 97 active+clean; 28 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 354 MiB/s wr, 2.37k op/s 2024-05-20T20:11:11.129 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:10 smithi028 ceph-mon[31178]: pgmap v169: 97 pgs: 97 active+clean; 28 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 354 MiB/s wr, 2.37k op/s 2024-05-20T20:11:13.047 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:12 smithi028 ceph-mon[31178]: pgmap v170: 97 pgs: 97 active+clean; 28 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 229 MiB/s wr, 1.72k op/s 2024-05-20T20:11:13.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:12 smithi154 ceph-mon[36314]: pgmap v170: 97 pgs: 97 active+clean; 28 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 229 MiB/s wr, 1.72k op/s 2024-05-20T20:11:13.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:12 smithi033 ceph-mon[36179]: pgmap v170: 97 pgs: 97 active+clean; 28 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 229 MiB/s wr, 1.72k op/s 2024-05-20T20:11:14.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.042+0000 7f285f800700 1 -- 172.21.15.28:0/1957633877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28581035f0 msgr2=0x7f2858103a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.042+0000 7f285f800700 1 --2- 172.21.15.28:0/1957633877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28581035f0 0x7f2858103a50 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f284401a9a0 tx=0x7f284401acb0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.043+0000 7f285f800700 1 -- 172.21.15.28:0/1957633877 shutdown_connections 2024-05-20T20:11:14.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.043+0000 7f285f800700 1 --2- 172.21.15.28:0/1957633877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2858103f90 0x7f2858110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.043+0000 7f285f800700 1 --2- 172.21.15.28:0/1957633877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2858108500 0x7f28581088e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.043+0000 7f285f800700 1 --2- 172.21.15.28:0/1957633877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28581035f0 0x7f2858103a50 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.043+0000 7f285f800700 1 -- 172.21.15.28:0/1957633877 >> 172.21.15.28:0/1957633877 conn(0x7f285807a6a0 msgr2=0x7f285807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:14.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.044+0000 7f285f800700 1 -- 172.21.15.28:0/1957633877 shutdown_connections 2024-05-20T20:11:14.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.044+0000 7f285f800700 1 -- 172.21.15.28:0/1957633877 wait complete. 2024-05-20T20:11:14.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.046+0000 7f285f800700 1 Processor -- start 2024-05-20T20:11:14.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.046+0000 7f285f800700 1 -- start start 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.047+0000 7f285f800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28581035f0 0x7f2858071190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.047+0000 7f285f800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2858103f90 0x7f28580716d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.047+0000 7f285f800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2858108500 0x7f2858076960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.047+0000 7f285f800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f285806f730 con 0x7f2858103f90 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.047+0000 7f285f800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f285806f5b0 con 0x7f2858108500 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.047+0000 7f285f800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f285806f8b0 con 0x7f28581035f0 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.049+0000 7f285ce00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28581035f0 0x7f2858071190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.049+0000 7f285ce00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28581035f0 0x7f2858071190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37068/0 (socket says 172.21.15.28:37068) 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.049+0000 7f285ce00700 1 -- 172.21.15.28:0/2828581401 learned_addr learned my addr 172.21.15.28:0/2828581401 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.050+0000 7f285ce00700 1 -- 172.21.15.28:0/2828581401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2858108500 msgr2=0x7f2858076960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.050+0000 7f285ce00700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2858108500 0x7f2858076960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.050+0000 7f285ce00700 1 -- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2858103f90 msgr2=0x7f28580716d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.050+0000 7f285ce00700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2858103f90 0x7f28580716d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.050+0000 7f285ce00700 1 -- 172.21.15.28:0/2828581401 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2858077260 con 0x7f28581035f0 2024-05-20T20:11:14.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.051+0000 7f285ce00700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28581035f0 0x7f2858071190 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f284c00d980 tx=0x7f284c00dd40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:14.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.056+0000 7f2855600700 1 -- 172.21.15.28:0/2828581401 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f284c017030 con 0x7f28581035f0 2024-05-20T20:11:14.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.056+0000 7f2855600700 1 -- 172.21.15.28:0/2828581401 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f284c0044d0 con 0x7f28581035f0 2024-05-20T20:11:14.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.056+0000 7f2855600700 1 -- 172.21.15.28:0/2828581401 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f284c0039c0 con 0x7f28581035f0 2024-05-20T20:11:14.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.063+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2858077550 con 0x7f28581035f0 2024-05-20T20:11:14.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.064+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f285819bf70 con 0x7f28581035f0 2024-05-20T20:11:14.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.083+0000 7f286a24cb80 1 Processor -- start 2024-05-20T20:11:14.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.083+0000 7f2855600700 1 -- 172.21.15.28:0/2828581401 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f284c01a070 con 0x7f28581035f0 2024-05-20T20:11:14.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.083+0000 7f286a24cb80 1 -- start start 2024-05-20T20:11:14.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4da1610 0x5595c4da19f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4d9bb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4d9c140 0x5595c4d9c5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f286a24cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5595c4cf9e40 con 0x5595c4d9c140 2024-05-20T20:11:14.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f286a24cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5595c4ce99e0 con 0x5595c4da1610 2024-05-20T20:11:14.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f286a24cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5595c4ceda20 con 0x5595c4da1f30 2024-05-20T20:11:14.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f285ce00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4d9bb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f285ce00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4d9bb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37078/0 (socket says 172.21.15.28:37078) 2024-05-20T20:11:14.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.086+0000 7f285ce00700 1 -- 172.21.15.28:0/249153625 learned_addr learned my addr 172.21.15.28:0/249153625 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:14.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.087+0000 7f285ce00700 1 -- 172.21.15.28:0/249153625 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4da1610 msgr2=0x5595c4da19f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.087+0000 7f285ce00700 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4da1610 0x5595c4da19f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.087+0000 7f285ce00700 1 -- 172.21.15.28:0/249153625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4d9c140 msgr2=0x5595c4d9c5c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.087+0000 7f2857e00700 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4d9c140 0x5595c4d9c5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.087+0000 7f285ce00700 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4d9c140 0x5595c4d9c5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.088+0000 7f285ce00700 1 -- 172.21.15.28:0/249153625 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5595c4cb2420 con 0x5595c4da1f30 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.088+0000 7f285ce00700 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4d9bb20 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f284c0de710 tx=0x7f284c0e96e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=278ff6921acd38a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.088+0000 7f284a000700 1 -- 172.21.15.28:0/249153625 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f284c0f4070 con 0x5595c4da1f30 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.088+0000 7f284a000700 1 -- 172.21.15.28:0/249153625 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f284c0e9e30 con 0x5595c4da1f30 2024-05-20T20:11:14.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.089+0000 7f2855600700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28400b2b40 0x7f28400b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.089+0000 7f2855600700 1 -- 172.21.15.28:0/2828581401 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f284c0e10d0 con 0x7f28581035f0 2024-05-20T20:11:14.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.090+0000 7f284a000700 1 -- 172.21.15.28:0/249153625 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f284c0ef7f0 con 0x5595c4da1f30 2024-05-20T20:11:14.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.092+0000 7f286a24cb80 1 -- 172.21.15.28:0/249153625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 msgr2=0x5595c4d9bb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.092+0000 7f286a24cb80 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4d9bb20 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f284c0de710 tx=0x7f284c0e96e0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.093+0000 7f286a24cb80 1 -- 172.21.15.28:0/249153625 shutdown_connections 2024-05-20T20:11:14.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.094+0000 7f286a24cb80 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4d9c140 0x5595c4d9c5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.094+0000 7f286a24cb80 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4da1610 0x5595c4da19f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.095+0000 7f285d800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28400b2b40 0x7f28400b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.095+0000 7f286a24cb80 1 --2- 172.21.15.28:0/249153625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4d9bb20 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.096+0000 7f286a24cb80 1 -- 172.21.15.28:0/249153625 >> 172.21.15.28:0/249153625 conn(0x5595c4cc9be0 msgr2=0x5595c4d96690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:14.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.097+0000 7f286a24cb80 1 -- 172.21.15.28:0/249153625 shutdown_connections 2024-05-20T20:11:14.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.098+0000 7f286a24cb80 1 -- 172.21.15.28:0/249153625 wait complete. 2024-05-20T20:11:14.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.099+0000 7f285d800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28400b2b40 0x7f28400b5000 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f285000e3f0 tx=0x7f285000b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:14.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.101+0000 7f286a24cb80 1 -- start start 2024-05-20T20:11:14.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.104+0000 7f286a24cb80 1 Processor -- start 2024-05-20T20:11:14.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.107+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4da7bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.110+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1610 0x5595c4da4f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1f30 0x5595c4da5490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f286a24cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5595c4beffa0 con 0x5595c4da1f30 2024-05-20T20:11:14.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f2857e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1610 0x5595c4da4f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f285ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4da7bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f285ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4da7bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58494/0 (socket says 172.21.15.28:58494) 2024-05-20T20:11:14.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f285ce00700 1 -- 172.21.15.28:0/2874579015 learned_addr learned my addr 172.21.15.28:0/2874579015 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:14.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5595c4c8a760 con 0x5595c4d9c140 2024-05-20T20:11:14.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.111+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5595c4da8a80 con 0x5595c4da1610 2024-05-20T20:11:14.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.112+0000 7f286a24cb80 10 client.0 _ll_register_callbacks cb 0x5595c4c93c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:14.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.112+0000 7f285d800700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1f30 0x5595c4da5490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.113+0000 7f285d800700 1 -- 172.21.15.28:0/2874579015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1610 msgr2=0x5595c4da4f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:14.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.114+0000 7f285d800700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1610 0x5595c4da4f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.114+0000 7f285d800700 1 -- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 msgr2=0x5595c4da7bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:14.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.114+0000 7f285d800700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4da7bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.114+0000 7f285d800700 1 -- 172.21.15.28:0/2874579015 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5595c4c398d0 con 0x5595c4da1f30 2024-05-20T20:11:14.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.114+0000 7f285ce00700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4da7bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:14.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.115+0000 7f2857e00700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1610 0x5595c4da4f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:14.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.115+0000 7f285d800700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1f30 0x5595c4da5490 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f2850015ab0 tx=0x7f2850004ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=195b71c81fa8d91f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:14.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.116+0000 7f283ea00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:11:14.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.116+0000 7f283ea00700 1 -- 172.21.15.28:0/2874579015 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2850023020 con 0x5595c4da1f30 2024-05-20T20:11:14.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.116+0000 7f283ea00700 1 -- 172.21.15.28:0/2874579015 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2850013a50 con 0x5595c4da1f30 2024-05-20T20:11:14.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.116+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f284c0e9e30 con 0x5595c4da1f30 2024-05-20T20:11:14.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.117+0000 7f283ea00700 1 -- 172.21.15.28:0/2874579015 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f285002e050 con 0x5595c4da1f30 2024-05-20T20:11:14.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.117+0000 7f286a24cb80 10 client.16368 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:14.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.117+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5595c4cba060 con 0x5595c4da1f30 2024-05-20T20:11:14.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.118+0000 7f283ea00700 1 -- 172.21.15.28:0/2874579015 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f284c0e9e30 con 0x5595c4da1f30 2024-05-20T20:11:14.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.118+0000 7f283ea00700 1 -- 172.21.15.28:0/2874579015 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f285001e030 con 0x5595c4da1f30 2024-05-20T20:11:14.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.118+0000 7f286a24cb80 10 client.16368 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:14.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.118+0000 7f286a24cb80 10 client.16368 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:11:14.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.120+0000 7f286a24cb80 20 client.16368 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:14.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.120+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4db56c0 0x5595c4db7ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.120+0000 7f286a24cb80 4 client.16368 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:14.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.120+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5595c4cfbd90 con 0x5595c4db56c0 2024-05-20T20:11:14.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.120+0000 7f285ce00700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4db56c0 0x5595c4db7ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.121+0000 7f285ce00700 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4db56c0 0x5595c4db7ab0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=4a742e4df9489f84 server_cookie=afbdeb67fc5f0585 in_seq=0 out_seq=0 2024-05-20T20:11:14.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.121+0000 7f283ea00700 10 client.16368 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:11:14.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.122+0000 7f283ea00700 1 -- 172.21.15.28:0/2874579015 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5595c4cfbd90 con 0x5595c4db56c0 2024-05-20T20:11:14.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.122+0000 7f283ea00700 10 client.16368 handle_command_reply: tid=0 2024-05-20T20:11:14.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.123+0000 7f286a24cb80 1 client.16368 shutdown 2024-05-20T20:11:14.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.123+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1f30 msgr2=0x5595c4da5490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.123+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1f30 0x5595c4da5490 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f2850015ab0 tx=0x7f2850004ad0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 shutdown_connections 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4db56c0 0x5595c4db7ab0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1f30 0x5595c4da5490 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4da7bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2874579015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1610 0x5595c4da4f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f283ea00700 0 client.16368 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 >> 172.21.15.28:0/2874579015 conn(0x5595c4cc9be0 msgr2=0x5595c4d95d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:14.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 shutdown_connections 2024-05-20T20:11:14.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.124+0000 7f286a24cb80 1 -- 172.21.15.28:0/2874579015 wait complete. 2024-05-20T20:11:14.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.125+0000 7f286a24cb80 20 client.16368 trim_cache size 0 max 16384 2024-05-20T20:11:14.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.154+0000 7f286a24cb80 1 Processor -- start 2024-05-20T20:11:14.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.158+0000 7f286a24cb80 1 -- start start 2024-05-20T20:11:14.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.159+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4d9bb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.159+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4d98250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.159+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4da53a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.159+0000 7f286a24cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f28581b4580 con 0x5595c4da1610 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.159+0000 7f286a24cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f28581b60c0 con 0x5595c4d9c140 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.159+0000 7f286a24cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f28581b4c50 con 0x5595c4da1f30 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.161+0000 7f285d800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4da53a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.161+0000 7f285d800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4da53a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37100/0 (socket says 172.21.15.28:37100) 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.162+0000 7f285d800700 1 -- 172.21.15.28:0/2900632139 learned_addr learned my addr 172.21.15.28:0/2900632139 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.163+0000 7f285d800700 1 -- 172.21.15.28:0/2900632139 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 msgr2=0x5595c4d9bb20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.164+0000 7f285d800700 1 --2- 172.21.15.28:0/2900632139 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4d9bb20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.164+0000 7f285d800700 1 -- 172.21.15.28:0/2900632139 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 msgr2=0x5595c4d98250 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.164+0000 7f285d800700 1 --2- 172.21.15.28:0/2900632139 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4d98250 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.164+0000 7f285d800700 1 -- 172.21.15.28:0/2900632139 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5595c4d9f9d0 con 0x5595c4da1f30 2024-05-20T20:11:14.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.166+0000 7f285d800700 1 --2- 172.21.15.28:0/2900632139 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4da53a0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f285001ff20 tx=0x7f2850009920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7fbec6718cb37e3c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:14.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.168+0000 7f2849600700 1 -- 172.21.15.28:0/2900632139 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2850038030 con 0x5595c4da1f30 2024-05-20T20:11:14.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.169+0000 7f2849600700 1 -- 172.21.15.28:0/2900632139 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2850005c60 con 0x5595c4da1f30 2024-05-20T20:11:14.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.169+0000 7f2849600700 1 -- 172.21.15.28:0/2900632139 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2850023050 con 0x5595c4da1f30 2024-05-20T20:11:14.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.170+0000 7f286a24cb80 1 -- 172.21.15.28:0/2900632139 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 msgr2=0x5595c4da53a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.170+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2900632139 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4da53a0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f285001ff20 tx=0x7f2850009920 comp rx=0 tx=0).stop 2024-05-20T20:11:14.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.171+0000 7f286a24cb80 1 -- 172.21.15.28:0/2900632139 shutdown_connections 2024-05-20T20:11:14.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.171+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2900632139 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4da53a0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.171+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2900632139 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4d9bb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.171+0000 7f286a24cb80 1 --2- 172.21.15.28:0/2900632139 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4d98250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.171+0000 7f286a24cb80 1 -- 172.21.15.28:0/2900632139 >> 172.21.15.28:0/2900632139 conn(0x5595c4cc9be0 msgr2=0x5595c4da3840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:14.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.171+0000 7f286a24cb80 1 -- 172.21.15.28:0/2900632139 shutdown_connections 2024-05-20T20:11:14.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.171+0000 7f286a24cb80 1 -- 172.21.15.28:0/2900632139 wait complete. 2024-05-20T20:11:14.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.172+0000 7f286a24cb80 1 -- start start 2024-05-20T20:11:14.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.172+0000 7f286a24cb80 1 Processor -- start 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.176+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4dae080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.176+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4dae5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.176+0000 7f286a24cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4dbc860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.176+0000 7f286a24cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f28581b59f0 con 0x5595c4da1610 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.176+0000 7f286a24cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f28581b5320 con 0x5595c4d9c140 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.176+0000 7f286a24cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f28581b2520 con 0x5595c4da1f30 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.177+0000 7f286a24cb80 10 client.0 _ll_register_callbacks cb 0x5595c4c93c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.178+0000 7f2857e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4dae5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.178+0000 7f285d800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4dbc860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.178+0000 7f285ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4dae080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.178+0000 7f285ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4dae080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58510/0 (socket says 172.21.15.28:58510) 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.178+0000 7f285ce00700 1 -- 172.21.15.28:0/3011438202 learned_addr learned my addr 172.21.15.28:0/3011438202 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.179+0000 7f285d800700 1 -- 172.21.15.28:0/3011438202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 msgr2=0x5595c4dae080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.180+0000 7f285d800700 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4dae080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.180+0000 7f285d800700 1 -- 172.21.15.28:0/3011438202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 msgr2=0x5595c4dae5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.180+0000 7f285d800700 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4dae5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.180+0000 7f285d800700 1 -- 172.21.15.28:0/3011438202 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5595c4dbd040 con 0x5595c4da1f30 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.180+0000 7f2857e00700 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4dae5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.181+0000 7f285d800700 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4dbc860 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f2850015ab0 tx=0x7f2850004de0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=69049041b124eb66 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.181+0000 7f283ea00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.181+0000 7f283ea00700 1 -- 172.21.15.28:0/3011438202 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2850042060 con 0x5595c4da1f30 2024-05-20T20:11:14.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.182+0000 7f283ea00700 1 -- 172.21.15.28:0/3011438202 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f28500136a0 con 0x5595c4da1f30 2024-05-20T20:11:14.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.182+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2850005c60 con 0x5595c4da1f30 2024-05-20T20:11:14.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.182+0000 7f283ea00700 1 -- 172.21.15.28:0/3011438202 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2850038070 con 0x5595c4da1f30 2024-05-20T20:11:14.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.182+0000 7f286a24cb80 10 client.26548 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:14.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.182+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5595c4e1fe10 con 0x5595c4da1f30 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.183+0000 7f283ea00700 1 -- 172.21.15.28:0/3011438202 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2850005c60 con 0x5595c4da1f30 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.183+0000 7f283ea00700 1 -- 172.21.15.28:0/3011438202 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f285001e040 con 0x5595c4da1f30 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.183+0000 7f286a24cb80 10 client.26548 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.184+0000 7f286a24cb80 10 client.26548 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.184+0000 7f286a24cb80 20 client.26548 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.184+0000 7f286a24cb80 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4e20350 0x5595c4e22740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.184+0000 7f286a24cb80 4 client.26548 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.184+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5595c4cf5570 con 0x5595c4e20350 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.184+0000 7f285ce00700 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4e20350 0x5595c4e22740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.185+0000 7f285ce00700 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4e20350 0x5595c4e22740 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=8061b5519c096d4a server_cookie=a127bdf0ebb1ab83 in_seq=0 out_seq=0 2024-05-20T20:11:14.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.185+0000 7f283ea00700 10 client.26548 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:11:14.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.188+0000 7f283ea00700 1 -- 172.21.15.28:0/3011438202 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5595c4cf5570 con 0x5595c4e20350 2024-05-20T20:11:14.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.188+0000 7f283ea00700 10 client.26548 handle_command_reply: tid=0 2024-05-20T20:11:14.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.189+0000 7f286a24cb80 1 client.26548 shutdown 2024-05-20T20:11:14.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 msgr2=0x5595c4dbc860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4dbc860 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f2850015ab0 tx=0x7f2850004de0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 shutdown_connections 2024-05-20T20:11:14.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5595c4e20350 0x5595c4e22740 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5595c4da1f30 0x5595c4dbc860 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5595c4d9c140 0x5595c4dae080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 --2- 172.21.15.28:0/3011438202 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5595c4da1610 0x5595c4dae5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 >> 172.21.15.28:0/3011438202 conn(0x5595c4cc9be0 msgr2=0x5595c4da40d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:14.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.192+0000 7f283ea00700 0 client.26548 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:11:14.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.195+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 shutdown_connections 2024-05-20T20:11:14.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.195+0000 7f286a24cb80 1 -- 172.21.15.28:0/3011438202 wait complete. 2024-05-20T20:11:14.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.195+0000 7f286a24cb80 20 client.26548 trim_cache size 0 max 16384 2024-05-20T20:11:14.196 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:11:14.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.199+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28400b2b40 msgr2=0x7f28400b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.199+0000 7f285f800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28400b2b40 0x7f28400b5000 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f285000e3f0 tx=0x7f285000b040 comp rx=0 tx=0).stop 2024-05-20T20:11:14.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.199+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28581035f0 msgr2=0x7f2858071190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:14.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.199+0000 7f285f800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28581035f0 0x7f2858071190 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f284c00d980 tx=0x7f284c00dd40 comp rx=0 tx=0).stop 2024-05-20T20:11:14.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.204+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 shutdown_connections 2024-05-20T20:11:14.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.204+0000 7f285f800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28400b2b40 0x7f28400b5000 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.204+0000 7f285f800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2858108500 0x7f2858076960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.204+0000 7f285f800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28581035f0 0x7f2858071190 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.204+0000 7f285f800700 1 --2- 172.21.15.28:0/2828581401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2858103f90 0x7f28580716d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:14.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.204+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 >> 172.21.15.28:0/2828581401 conn(0x7f285807a6a0 msgr2=0x7f2858101740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:14.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.210+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 shutdown_connections 2024-05-20T20:11:14.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:14.210+0000 7f285f800700 1 -- 172.21.15.28:0/2828581401 wait complete. 2024-05-20T20:11:15.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:14 smithi028 ceph-mon[31178]: pgmap v171: 97 pgs: 97 active+clean; 28 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 300 MiB/s wr, 2.37k op/s 2024-05-20T20:11:15.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:14 smithi154 ceph-mon[36314]: pgmap v171: 97 pgs: 97 active+clean; 28 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 300 MiB/s wr, 2.37k op/s 2024-05-20T20:11:15.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:14 smithi033 ceph-mon[36179]: pgmap v171: 97 pgs: 97 active+clean; 28 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 300 MiB/s wr, 2.37k op/s 2024-05-20T20:11:15.299 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:11:16.111 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:17.156 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:16 smithi028 ceph-mon[31178]: pgmap v172: 97 pgs: 97 active+clean; 28 GiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 269 MiB/s wr, 2.24k op/s 2024-05-20T20:11:17.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:16 smithi154 ceph-mon[36314]: pgmap v172: 97 pgs: 97 active+clean; 28 GiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 269 MiB/s wr, 2.24k op/s 2024-05-20T20:11:17.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:16 smithi033 ceph-mon[36179]: pgmap v172: 97 pgs: 97 active+clean; 28 GiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 269 MiB/s wr, 2.24k op/s 2024-05-20T20:11:18.995 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:18 smithi028 ceph-mon[31178]: pgmap v173: 97 pgs: 97 active+clean; 28 GiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 235 MiB/s wr, 1.98k op/s 2024-05-20T20:11:18.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:18.992+0000 7fdbda400700 1 -- 172.21.15.28:0/3247749494 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102440 msgr2=0x7fdbd41028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:18.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:18.994+0000 7fdbda400700 1 --2- 172.21.15.28:0/3247749494 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102440 0x7fdbd41028c0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7fdbcc009f90 tx=0x7fdbcc00b330 comp rx=0 tx=0).stop 2024-05-20T20:11:18.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:18.995+0000 7fdbda400700 1 -- 172.21.15.28:0/3247749494 shutdown_connections 2024-05-20T20:11:18.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:18.995+0000 7fdbda400700 1 --2- 172.21.15.28:0/3247749494 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102e00 0x7fdbd410f2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:18.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:18.995+0000 7fdbda400700 1 --2- 172.21.15.28:0/3247749494 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdbd4108520 0x7fdbd4108900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:18.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:18.995+0000 7fdbda400700 1 --2- 172.21.15.28:0/3247749494 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102440 0x7fdbd41028c0 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:18.996 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:18.996+0000 7fdbda400700 1 -- 172.21.15.28:0/3247749494 >> 172.21.15.28:0/3247749494 conn(0x7fdbd40793f0 msgr2=0x7fdbd4079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.004+0000 7fdbda400700 1 -- 172.21.15.28:0/3247749494 shutdown_connections 2024-05-20T20:11:19.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.004+0000 7fdbda400700 1 -- 172.21.15.28:0/3247749494 wait complete. 2024-05-20T20:11:19.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.008+0000 7fdbda400700 1 Processor -- start 2024-05-20T20:11:19.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.008+0000 7fdbda400700 1 -- start start 2024-05-20T20:11:19.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.010+0000 7fdbda400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102440 0x7fdbd419e7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.010+0000 7fdbda400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102e00 0x7fdbd419ece0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.011+0000 7fdbda400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdbd4108520 0x7fdbd4198960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.012+0000 7fdbda400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd4112620 con 0x7fdbd4108520 2024-05-20T20:11:19.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.012+0000 7fdbda400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41124a0 con 0x7fdbd4102e00 2024-05-20T20:11:19.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.012+0000 7fdbda400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41127a0 con 0x7fdbd4102440 2024-05-20T20:11:19.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.012+0000 7fdbd3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102440 0x7fdbd419e7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.013+0000 7fdbd3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102440 0x7fdbd419e7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39960/0 (socket says 172.21.15.28:39960) 2024-05-20T20:11:19.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.013+0000 7fdbd3400700 1 -- 172.21.15.28:0/1670277472 learned_addr learned my addr 172.21.15.28:0/1670277472 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.015+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102e00 0x7fdbd419ece0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.015+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdbd4108520 0x7fdbd4198960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.016+0000 7fdbd3400700 1 -- 172.21.15.28:0/1670277472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102e00 msgr2=0x7fdbd419ece0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.016+0000 7fdbd3400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102e00 0x7fdbd419ece0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.016+0000 7fdbd3400700 1 -- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdbd4108520 msgr2=0x7fdbd4198960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.016+0000 7fdbd3400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdbd4108520 0x7fdbd4198960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.016+0000 7fdbd3400700 1 -- 172.21.15.28:0/1670277472 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdbd41990c0 con 0x7fdbd4102440 2024-05-20T20:11:19.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.018+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102e00 0x7fdbd419ece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:19.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.019+0000 7fdbd3400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102440 0x7fdbd419e7a0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fdbc000d950 tx=0x7fdbc000dc60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.019+0000 7fdbcbe00700 1 -- 172.21.15.28:0/1670277472 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbc0016070 con 0x7fdbd4102440 2024-05-20T20:11:19.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.020+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdbd41993b0 con 0x7fdbd4102440 2024-05-20T20:11:19.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.020+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdbd406b220 con 0x7fdbd4102440 2024-05-20T20:11:19.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.021+0000 7fdbcbe00700 1 -- 172.21.15.28:0/1670277472 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdbc0009de0 con 0x7fdbd4102440 2024-05-20T20:11:19.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.021+0000 7fdbcbe00700 1 -- 172.21.15.28:0/1670277472 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbc0016070 con 0x7fdbd4102440 2024-05-20T20:11:19.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.039+0000 7f28c7e00640 1 -- 172.21.15.28:0/2447404634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c81079b0 msgr2=0x7f28c8107d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 --2- 172.21.15.28:0/2447404634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c81079b0 0x7f28c8107d90 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f28b8009930 tx=0x7f28b802a480 comp rx=0 tx=0).stop 2024-05-20T20:11:19.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 -- 172.21.15.28:0/2447404634 shutdown_connections 2024-05-20T20:11:19.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 --2- 172.21.15.28:0/2447404634 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 0x7f28c810f620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 --2- 172.21.15.28:0/2447404634 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28c8100960 0x7f28c8100de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 --2- 172.21.15.28:0/2447404634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c81079b0 0x7f28c8107d90 secure :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f28b8009930 tx=0x7f28b802a480 comp rx=0 tx=0).stop 2024-05-20T20:11:19.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 -- 172.21.15.28:0/2447404634 >> 172.21.15.28:0/2447404634 conn(0x7f28c80778b0 msgr2=0x7f28c80fec70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 -- 172.21.15.28:0/2447404634 shutdown_connections 2024-05-20T20:11:19.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.040+0000 7f28c7e00640 1 -- 172.21.15.28:0/2447404634 wait complete. 2024-05-20T20:11:19.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.043+0000 7f28c7e00640 1 Processor -- start 2024-05-20T20:11:19.045 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.044+0000 7f28c7e00640 1 -- start start 2024-05-20T20:11:19.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.046+0000 7f28c7e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c8100960 0x7f28c81a4380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.046+0000 7f28c7e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 0x7f28c81a48c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.046+0000 7f28c7e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28c810e940 0x7f28c810edc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.048+0000 7f28c7e00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f28c8112b40 con 0x7f28c8100960 2024-05-20T20:11:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.048+0000 7f28c7e00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f28c81129c0 con 0x7f28c8101320 2024-05-20T20:11:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.048+0000 7f28c7e00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f28c8112cc0 con 0x7f28c810e940 2024-05-20T20:11:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.048+0000 7f28c6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 0x7f28c81a48c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.048+0000 7f28c6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 0x7f28c81a48c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58472/0 (socket says 172.21.15.28:58472) 2024-05-20T20:11:19.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.048+0000 7f28c6000640 1 -- 172.21.15.28:0/1462667316 learned_addr learned my addr 172.21.15.28:0/1462667316 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.050+0000 7f28c7400640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28c810e940 0x7f28c810edc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.050+0000 7f28c6a00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c8100960 0x7f28c81a4380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.051+0000 7f28c6000640 1 -- 172.21.15.28:0/1462667316 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28c810e940 msgr2=0x7f28c810edc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.052+0000 7f28c6000640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28c810e940 0x7f28c810edc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.052+0000 7f28c6000640 1 -- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c8100960 msgr2=0x7f28c81a4380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.052+0000 7f28c6000640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c8100960 0x7f28c81a4380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.052+0000 7f28c6000640 1 -- 172.21.15.28:0/1462667316 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f28c810f4f0 con 0x7f28c8101320 2024-05-20T20:11:19.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.053+0000 7f28c7400640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28c810e940 0x7f28c810edc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:19.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.053+0000 7f28c6000640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 0x7f28c81a48c0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f28b400cab0 tx=0x7f28b400cf70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.054+0000 7f28bf400640 1 -- 172.21.15.28:0/1462667316 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28b4007d30 con 0x7f28c8101320 2024-05-20T20:11:19.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.054+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f28c81ac2f0 con 0x7f28c8101320 2024-05-20T20:11:19.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.054+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f28c81ac6a0 con 0x7f28c8101320 2024-05-20T20:11:19.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.054+0000 7f28bf400640 1 -- 172.21.15.28:0/1462667316 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f28b40035f0 con 0x7f28c8101320 2024-05-20T20:11:19.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.054+0000 7f28bf400640 1 -- 172.21.15.28:0/1462667316 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28b400f690 con 0x7f28c8101320 2024-05-20T20:11:19.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.046+0000 7fdbcbe00700 1 -- 172.21.15.28:0/1670277472 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fdbc0019070 con 0x7fdbd4102440 2024-05-20T20:11:19.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.048+0000 7fdbe4d74b80 1 Processor -- start 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.048+0000 7fdbe4d74b80 1 -- start start 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.049+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c6de00 0x55f406c6e1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.049+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6e720 0x55f406c68330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.049+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c68950 0x55f406c68db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.049+0000 7fdbe4d74b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55f406bbde40 con 0x55f406c6e720 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.049+0000 7fdbe4d74b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55f406bad9e0 con 0x55f406c6de00 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.049+0000 7fdbe4d74b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55f406bb1a20 con 0x55f406c68950 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.049+0000 7fdbd2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c68950 0x55f406c68db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.050+0000 7fdbd2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c68950 0x55f406c68db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39980/0 (socket says 172.21.15.28:39980) 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.050+0000 7fdbd2a00700 1 -- 172.21.15.28:0/3498134450 learned_addr learned my addr 172.21.15.28:0/3498134450 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.050+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c6de00 0x55f406c6e1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.052+0000 7fdbd3e00700 1 -- 172.21.15.28:0/3498134450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c68950 msgr2=0x55f406c68db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.052+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c68950 0x55f406c68db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.052+0000 7fdbd3e00700 1 -- 172.21.15.28:0/3498134450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6e720 msgr2=0x55f406c68330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.052+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6e720 0x55f406c68330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.052+0000 7fdbd3e00700 1 -- 172.21.15.28:0/3498134450 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f406b76420 con 0x55f406c6de00 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.052+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c6de00 0x55f406c6e1e0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7fdbc4009c20 tx=0x7fdbc4009f30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=658e38a6a21b440b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.053+0000 7fdbc8c00700 1 -- 172.21.15.28:0/3498134450 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbc4011cb0 con 0x55f406c6de00 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.053+0000 7fdbc8c00700 1 -- 172.21.15.28:0/3498134450 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdbc4011e50 con 0x55f406c6de00 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.053+0000 7fdbc8c00700 1 -- 172.21.15.28:0/3498134450 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbc4005280 con 0x55f406c6de00 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.055+0000 7fdbcbe00700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdbb80b2c70 0x7fdbb80b5130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.060+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/3498134450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c6de00 msgr2=0x55f406c6e1e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.060+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c6de00 0x55f406c6e1e0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7fdbc4009c20 tx=0x7fdbc4009f30 comp rx=0 tx=0).stop 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.062+0000 7fdbcbe00700 1 -- 172.21.15.28:0/1670277472 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fdbc00e1cd0 con 0x7fdbd4102440 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.062+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/3498134450 shutdown_connections 2024-05-20T20:11:19.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.062+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c68950 0x55f406c68db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.062+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c6de00 0x55f406c6e1e0 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.062+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/3498134450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6e720 0x55f406c68330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.062+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/3498134450 >> 172.21.15.28:0/3498134450 conn(0x55f406b8dbe0 msgr2=0x55f406c5a690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.063+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/3498134450 shutdown_connections 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.064+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/3498134450 wait complete. 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.065+0000 7fdbe4d74b80 1 -- start start 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.065+0000 7fdbe4d74b80 1 Processor -- start 2024-05-20T20:11:19.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.068+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c71e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.069+0000 7fdbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c71e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.070+0000 7fdbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c71e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58488/0 (socket says 172.21.15.28:58488) 2024-05-20T20:11:19.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.070+0000 7fdbd3400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdbb80b2c70 0x7fdbb80b5130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.073+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c72340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.070+0000 7f28bf400640 1 -- 172.21.15.28:0/1462667316 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f28b400f830 con 0x7f28c8101320 2024-05-20T20:11:19.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.073+0000 7f28bf400640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28ac0b2aa0 0x7f28ac0b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.073+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c72880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.073+0000 7fdbe4d74b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55f406ab3fa0 con 0x55f406c6de00 2024-05-20T20:11:19.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.073+0000 7fdbe4d74b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55f406b4e760 con 0x55f406c68950 2024-05-20T20:11:19.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.073+0000 7fdbe4d74b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55f406c5ff20 con 0x55f406c6e720 2024-05-20T20:11:19.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.074+0000 7fdbe4d74b80 10 client.0 _ll_register_callbacks cb 0x55f406b57c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:19.078 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.078+0000 7f28bf400640 1 -- 172.21.15.28:0/1462667316 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f28b40e3390 con 0x7f28c8101320 2024-05-20T20:11:19.078 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.079+0000 7f28c6a00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28ac0b2aa0 0x7f28ac0b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.081 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.081+0000 7f28c6a00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28ac0b2aa0 0x7f28ac0b4f60 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f28b8034040 tx=0x7f28b8005fd0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.075+0000 7fdbd3e00700 1 -- 172.21.15.28:0/1352122610 learned_addr learned my addr 172.21.15.28:0/1352122610 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.076+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c72880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.077+0000 7fdbd3e00700 1 -- 172.21.15.28:0/1352122610 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 msgr2=0x55f406c71e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.077+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c71e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.077+0000 7fdbd3e00700 1 -- 172.21.15.28:0/1352122610 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 msgr2=0x55f406c72340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.077+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c72340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.077+0000 7fdbd3e00700 1 -- 172.21.15.28:0/1352122610 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f406afd8d0 con 0x55f406c6e720 2024-05-20T20:11:19.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.077+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c72880 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7fdbc4009550 tx=0x7fdbc4009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2d8b7d638b7cdb47 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.077+0000 7fdbbd600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.078+0000 7fdbbd600700 1 -- 172.21.15.28:0/1352122610 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbc402d060 con 0x55f406c6e720 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.078+0000 7fdbbd600700 1 -- 172.21.15.28:0/1352122610 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fdbc4005320 con 0x55f406c6e720 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.078+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdbc4011e50 con 0x55f406c6e720 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.078+0000 7fdbbd600700 1 -- 172.21.15.28:0/1352122610 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fdbc4005520 con 0x55f406c6e720 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.079+0000 7fdbe4d74b80 10 client.26563 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.079+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55f406b7e060 con 0x55f406c6e720 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.079+0000 7fdbbd600700 1 -- 172.21.15.28:0/1352122610 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdbc4011e50 con 0x55f406c6e720 2024-05-20T20:11:19.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.082+0000 7fdbbd600700 1 -- 172.21.15.28:0/1352122610 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fdbc4021070 con 0x55f406c6e720 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.082+0000 7fdbe4d74b80 10 client.26563 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.082+0000 7fdbe4d74b80 10 client.26563 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.082+0000 7fdbe4d74b80 20 client.26563 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.082+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406c8a630 0x55f406c8ca20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.082+0000 7fdbe4d74b80 4 client.26563 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.082+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55f406bbfd90 con 0x55f406c8a630 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.083+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406c8a630 0x55f406c8ca20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.083+0000 7fdbd3400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdbb80b2c70 0x7fdbb80b5130 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fdbc0009520 tx=0x7fdbc0000bc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.083+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406c8a630 0x55f406c8ca20 crc :-1 s=READY pgs=139 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=628ddb4e943c3d87 server_cookie=26571ab227d77814 in_seq=0 out_seq=0 2024-05-20T20:11:19.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.083+0000 7fdbbd600700 10 client.26563 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.084+0000 7fdbbd600700 1 -- 172.21.15.28:0/1352122610 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55f406bbfd90 con 0x55f406c8a630 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.084+0000 7fdbbd600700 10 client.26563 handle_command_reply: tid=0 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.084+0000 7fdbe4d74b80 1 client.26563 shutdown 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.085+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 msgr2=0x55f406c72880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.085+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c72880 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7fdbc4009550 tx=0x7fdbc4009310 comp rx=0 tx=0).stop 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.086+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 shutdown_connections 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.086+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406c8a630 0x55f406c8ca20 crc :-1 s=READY pgs=139 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.086+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c72880 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.086+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c71e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.086+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1352122610 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c72340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.086+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 >> 172.21.15.28:0/1352122610 conn(0x55f406b8dbe0 msgr2=0x55f406c59d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.086+0000 7fdbbd600700 0 client.26563 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:19.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.087+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 shutdown_connections 2024-05-20T20:11:19.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.087+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1352122610 wait complete. 2024-05-20T20:11:19.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.087+0000 7fdbe4d74b80 20 client.26563 trim_cache size 0 max 16384 2024-05-20T20:11:19.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.094+0000 7f28d09e5740 1 Processor -- start 2024-05-20T20:11:19.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.098+0000 7f28d09e5740 1 -- start start 2024-05-20T20:11:19.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.101+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21be250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.101+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c8b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.101+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21cb440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.101+0000 7f28d09e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5561e212eb90 con 0x5561e21be790 2024-05-20T20:11:19.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.101+0000 7f28d09e5740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5561e211e730 con 0x5561e21c9050 2024-05-20T20:11:19.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.101+0000 7f28d09e5740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5561e2122770 con 0x5561e21bde70 2024-05-20T20:11:19.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.102+0000 7f28c6a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21be250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.103+0000 7f28c6a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21be250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39992/0 (socket says 172.21.15.28:39992) 2024-05-20T20:11:19.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.103+0000 7f28c6a00640 1 -- 172.21.15.28:0/3075701786 learned_addr learned my addr 172.21.15.28:0/3075701786 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.103+0000 7f28c7400640 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21cb440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.104+0000 7f28c6000640 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c8b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.104+0000 7f28c7400640 1 -- 172.21.15.28:0/3075701786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 msgr2=0x5561e21be250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.104+0000 7f28c7400640 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21be250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.104+0000 7f28c7400640 1 -- 172.21.15.28:0/3075701786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 msgr2=0x5561e21c8b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.105+0000 7f28c7400640 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c8b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.105+0000 7f28c7400640 1 -- 172.21.15.28:0/3075701786 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5561e20b78b0 con 0x5561e21c9050 2024-05-20T20:11:19.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.105+0000 7f28c6000640 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c8b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:19.106 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.106+0000 7f28c7400640 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21cb440 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f28c000f6d0 tx=0x7f28c000fb90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7da1026934c8781b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.108+0000 7f28b3e00640 1 -- 172.21.15.28:0/3075701786 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28c0004d20 con 0x5561e21c9050 2024-05-20T20:11:19.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.108+0000 7f28b3e00640 1 -- 172.21.15.28:0/3075701786 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f28c0003e20 con 0x5561e21c9050 2024-05-20T20:11:19.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.108+0000 7f28b3e00640 1 -- 172.21.15.28:0/3075701786 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28c0004380 con 0x5561e21c9050 2024-05-20T20:11:19.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.109+0000 7f28d09e5740 1 -- 172.21.15.28:0/3075701786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 msgr2=0x5561e21cb440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.109+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21cb440 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f28c000f6d0 tx=0x7f28c000fb90 comp rx=0 tx=0).stop 2024-05-20T20:11:19.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.109+0000 7f28d09e5740 1 -- 172.21.15.28:0/3075701786 shutdown_connections 2024-05-20T20:11:19.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.110+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21cb440 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.110+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c8b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.110+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3075701786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21be250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.110+0000 7f28d09e5740 1 -- 172.21.15.28:0/3075701786 >> 172.21.15.28:0/3075701786 conn(0x5561e21ba380 msgr2=0x5561e21ba760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.110+0000 7f28d09e5740 1 -- 172.21.15.28:0/3075701786 shutdown_connections 2024-05-20T20:11:19.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.111+0000 7f28d09e5740 1 -- 172.21.15.28:0/3075701786 wait complete. 2024-05-20T20:11:19.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.112+0000 7f28d09e5740 1 -- start start 2024-05-20T20:11:19.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.112+0000 7f28d09e5740 1 Processor -- start 2024-05-20T20:11:19.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.114+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21c89a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.114+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c5c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.114+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21c61f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.115+0000 7f28c6a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21c89a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.115+0000 7f28c6a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21c89a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:40002/0 (socket says 172.21.15.28:40002) 2024-05-20T20:11:19.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.115+0000 7f28c7400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21c61f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.116+0000 7f28c7400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21c61f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58516/0 (socket says 172.21.15.28:58516) 2024-05-20T20:11:19.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.116+0000 7f28d09e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5561e1e04790 con 0x5561e21be790 2024-05-20T20:11:19.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.116+0000 7f28d09e5740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5561e20c6bc0 con 0x5561e21c9050 2024-05-20T20:11:19.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.117+0000 7f28d09e5740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5561e21cbc80 con 0x5561e21bde70 2024-05-20T20:11:19.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.117+0000 7f28c6000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c5c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.117+0000 7f28c6000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c5c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60428/0 (socket says 172.21.15.28:60428) 2024-05-20T20:11:19.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.117+0000 7f28c6000640 1 -- 172.21.15.28:0/3692712328 learned_addr learned my addr 172.21.15.28:0/3692712328 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.118+0000 7f28d09e5740 10 client.0 _ll_register_callbacks cb 0x5561e20a4400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:19.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.119+0000 7f28c7400640 1 -- 172.21.15.28:0/3692712328 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 msgr2=0x5561e21c89a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.119+0000 7f28c7400640 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21c89a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.119+0000 7f28c7400640 1 -- 172.21.15.28:0/3692712328 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 msgr2=0x5561e21c5c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.119+0000 7f28c7400640 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c5c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.119+0000 7f28c7400640 1 -- 172.21.15.28:0/3692712328 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5561e1e33210 con 0x5561e21c9050 2024-05-20T20:11:19.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.121+0000 7f28c7400640 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21c61f0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f28c0000f80 tx=0x7f28c000db30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7be3cf2ae2deaac1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.121+0000 7f28b0c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:19.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.121+0000 7f28b0c00640 1 -- 172.21.15.28:0/3692712328 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28c0026060 con 0x5561e21c9050 2024-05-20T20:11:19.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.122+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f28c0003e20 con 0x5561e21c9050 2024-05-20T20:11:19.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.122+0000 7fdbe4d74b80 1 Processor -- start 2024-05-20T20:11:19.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.122+0000 7f28b0c00640 1 -- 172.21.15.28:0/3692712328 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f28c0004a40 con 0x5561e21c9050 2024-05-20T20:11:19.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.122+0000 7fdbe4d74b80 1 -- start start 2024-05-20T20:11:19.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.123+0000 7f28b0c00640 1 -- 172.21.15.28:0/3692712328 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f28c0026060 con 0x5561e21c9050 2024-05-20T20:11:19.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.123+0000 7f28d09e5740 10 client.26648 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:19.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.123+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5561e1eff620 con 0x5561e21c9050 2024-05-20T20:11:19.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.124+0000 7f28b0c00640 1 -- 172.21.15.28:0/3692712328 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f28c0003e20 con 0x5561e21c9050 2024-05-20T20:11:19.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.123+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c5a4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.127 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.124+0000 7f28b0c00640 1 -- 172.21.15.28:0/3692712328 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f28c0005050 con 0x5561e21c9050 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.124+0000 7fdbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c5a4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.124+0000 7fdbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c5a4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58532/0 (socket says 172.21.15.28:58532) 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.124+0000 7fdbd3e00700 1 -- 172.21.15.28:0/1761336111 learned_addr learned my addr 172.21.15.28:0/1761336111 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.124+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c5f220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.125+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c92050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.125+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41abd90 con 0x55f406c6de00 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.125+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41ad8d0 con 0x55f406c68950 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.125+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41ac460 con 0x55f406c6e720 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.126+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c5f220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.126+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c92050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.127+0000 7fdbd2a00700 1 -- 172.21.15.28:0/1761336111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 msgr2=0x55f406c92050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.128+0000 7f28d09e5740 10 client.26648 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:19.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.128+0000 7f28d09e5740 10 client.26648 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:19.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.128+0000 7f28d09e5740 20 client.26648 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:19.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.128+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e21d88f0 0x5561e21dace0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.128+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c92050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.128+0000 7fdbd2a00700 1 -- 172.21.15.28:0/1761336111 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 msgr2=0x55f406c5a4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.128+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c5a4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.128+0000 7f28d09e5740 4 client.26648 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:19.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.128+0000 7f28c6a00640 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e21d88f0 0x5561e21dace0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.129+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5561e2130ae0 con 0x5561e21d88f0 2024-05-20T20:11:19.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.128+0000 7fdbd2a00700 1 -- 172.21.15.28:0/1761336111 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f406c70390 con 0x55f406c6de00 2024-05-20T20:11:19.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.129+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c5a4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:19.131 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.129+0000 7f28c6a00640 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e21d88f0 0x5561e21dace0 crc :-1 s=READY pgs=140 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fb68ac52bc741d03 server_cookie=98b33e0adc91f85b in_seq=0 out_seq=0 2024-05-20T20:11:19.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.129+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c5f220 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7fdbcc009790 tx=0x7fdbcc00b680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e110a05490dd4eec server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.130+0000 7f28b0c00640 10 client.26648 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.131+0000 7f28b0c00640 1 -- 172.21.15.28:0/3692712328 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5561e2130ae0 con 0x5561e21d88f0 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.131+0000 7f28b0c00640 10 client.26648 handle_command_reply: tid=0 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.131+0000 7f28d09e5740 1 client.26648 shutdown 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 msgr2=0x5561e21c61f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21c61f0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f28c0000f80 tx=0x7f28c000db30 comp rx=0 tx=0).stop 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 shutdown_connections 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e21d88f0 0x5561e21dace0 crc :-1 s=READY pgs=140 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21c9050 0x5561e21c61f0 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21be790 0x5561e21c5c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28b0c00640 0 client.26648 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:19.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3692712328 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21c89a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.132+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 >> 172.21.15.28:0/3692712328 conn(0x5561e21ba380 msgr2=0x5561e21c0100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.133+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 shutdown_connections 2024-05-20T20:11:19.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.133+0000 7f28d09e5740 1 -- 172.21.15.28:0/3692712328 wait complete. 2024-05-20T20:11:19.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.133+0000 7f28d09e5740 20 client.26648 trim_cache size 0 max 16384 2024-05-20T20:11:19.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.134+0000 7fdbbfe00700 1 -- 172.21.15.28:0/1761336111 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbcc00b9f0 con 0x55f406c6de00 2024-05-20T20:11:19.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.134+0000 7fdbbfe00700 1 -- 172.21.15.28:0/1761336111 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdbcc00bb90 con 0x55f406c6de00 2024-05-20T20:11:19.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.135+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 msgr2=0x55f406c5f220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.135+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c5f220 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7fdbcc009790 tx=0x7fdbcc00b680 comp rx=0 tx=0).stop 2024-05-20T20:11:19.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.136+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 shutdown_connections 2024-05-20T20:11:19.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.136+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c92050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.136+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c5a4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.136+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/1761336111 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c5f220 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.136+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 >> 172.21.15.28:0/1761336111 conn(0x55f406b8dbe0 msgr2=0x55f406c5eb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.137+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 shutdown_connections 2024-05-20T20:11:19.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.137+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/1761336111 wait complete. 2024-05-20T20:11:19.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.138+0000 7fdbe4d74b80 1 -- start start 2024-05-20T20:11:19.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.139+0000 7fdbe4d74b80 1 Processor -- start 2024-05-20T20:11:19.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.139+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c66e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.139+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c621e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.140+0000 7fdbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c66e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.140+0000 7fdbd3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c66e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58546/0 (socket says 172.21.15.28:58546) 2024-05-20T20:11:19.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.141+0000 7fdbe4d74b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c62920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.141+0000 7fdbd2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c621e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.141+0000 7fdbd2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c621e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60440/0 (socket says 172.21.15.28:60440) 2024-05-20T20:11:19.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.141+0000 7fdbd2a00700 1 -- 172.21.15.28:0/669632594 learned_addr learned my addr 172.21.15.28:0/669632594 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.141+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41ad200 con 0x55f406c6de00 2024-05-20T20:11:19.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.142+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41acb30 con 0x55f406c68950 2024-05-20T20:11:19.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.142+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdbd41a9cf0 con 0x55f406c6e720 2024-05-20T20:11:19.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.143+0000 7fdbe4d74b80 10 client.0 _ll_register_callbacks cb 0x55f406b57c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:19.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.143+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c62920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.143+0000 7fdbd3e00700 1 -- 172.21.15.28:0/669632594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 msgr2=0x55f406c62920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.143+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c62920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.143+0000 7fdbd3e00700 1 -- 172.21.15.28:0/669632594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 msgr2=0x55f406c621e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.144+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c621e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.144+0000 7fdbd3e00700 1 -- 172.21.15.28:0/669632594 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f406cf3f20 con 0x55f406c68950 2024-05-20T20:11:19.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.144+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c621e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:19.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.144+0000 7fdbbd600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:19.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.145+0000 7fdbd3e00700 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c66e60 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7fdbc4028e20 tx=0x7fdbc4004690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ce0a5198d6c04510 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.148+0000 7fdbbd600700 1 -- 172.21.15.28:0/669632594 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdbc401e040 con 0x55f406c68950 2024-05-20T20:11:19.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.149+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdbcc00bb90 con 0x55f406c68950 2024-05-20T20:11:19.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.149+0000 7fdbbd600700 1 -- 172.21.15.28:0/669632594 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fdbc4028ab0 con 0x55f406c68950 2024-05-20T20:11:19.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.150+0000 7fdbbd600700 1 -- 172.21.15.28:0/669632594 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fdbc401e040 con 0x55f406c68950 2024-05-20T20:11:19.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.150+0000 7fdbe4d74b80 10 client.26654 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:19.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.150+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55f406cf5110 con 0x55f406c68950 2024-05-20T20:11:19.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbbd600700 1 -- 172.21.15.28:0/669632594 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdbcc00bb90 con 0x55f406c68950 2024-05-20T20:11:19.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbbd600700 1 -- 172.21.15.28:0/669632594 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fdbc4021040 con 0x55f406c68950 2024-05-20T20:11:19.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbe4d74b80 10 client.26654 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:19.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbe4d74b80 10 client.26654 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:19.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbe4d74b80 20 client.26654 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406cf56e0 0x55f406cf7b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbe4d74b80 4 client.26654 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.151+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55f406bb9570 con 0x55f406cf56e0 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.154+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406cf56e0 0x55f406cf7b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.155+0000 7fdbd2a00700 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406cf56e0 0x55f406cf7b00 crc :-1 s=READY pgs=141 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fd39d238494d83b7 server_cookie=b882e958d342c92e in_seq=0 out_seq=0 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.155+0000 7fdbbd600700 10 client.26654 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.155+0000 7fdbbd600700 1 -- 172.21.15.28:0/669632594 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55f406bb9570 con 0x55f406cf56e0 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.155+0000 7fdbbd600700 10 client.26654 handle_command_reply: tid=0 2024-05-20T20:11:19.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.155+0000 7fdbe4d74b80 1 client.26654 shutdown 2024-05-20T20:11:19.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.156+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 msgr2=0x55f406c66e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.156+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c66e60 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7fdbc4028e20 tx=0x7fdbc4004690 comp rx=0 tx=0).stop 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.157+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 shutdown_connections 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.157+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f406cf56e0 0x55f406cf7b00 crc :-1 s=READY pgs=141 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.157+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f406c6e720 0x55f406c62920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.157+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f406c68950 0x55f406c66e60 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.157+0000 7fdbe4d74b80 1 --2- 172.21.15.28:0/669632594 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f406c6de00 0x55f406c621e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.157+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 >> 172.21.15.28:0/669632594 conn(0x55f406b8dbe0 msgr2=0x55f406c72650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.157+0000 7fdbbd600700 0 client.26654 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.158+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 shutdown_connections 2024-05-20T20:11:19.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.158+0000 7fdbe4d74b80 1 -- 172.21.15.28:0/669632594 wait complete. 2024-05-20T20:11:19.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.158+0000 7fdbe4d74b80 20 client.26654 trim_cache size 0 max 16384 2024-05-20T20:11:19.159 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:11:19.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.160+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdbb80b2c70 msgr2=0x7fdbb80b5130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.160+0000 7fdbda400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdbb80b2c70 0x7fdbb80b5130 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fdbc0009520 tx=0x7fdbc0000bc0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.160+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102440 msgr2=0x7fdbd419e7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.160+0000 7fdbda400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102440 0x7fdbd419e7a0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fdbc000d950 tx=0x7fdbc000dc60 comp rx=0 tx=0).stop 2024-05-20T20:11:19.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 shutdown_connections 2024-05-20T20:11:19.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdbd4108520 0x7fdbd4198960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdbd4102440 0x7fdbd419e7a0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdbb80b2c70 0x7fdbb80b5130 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 --2- 172.21.15.28:0/1670277472 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdbd4102e00 0x7fdbd419ece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 >> 172.21.15.28:0/1670277472 conn(0x7fdbd40793f0 msgr2=0x7fdbd41005f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 shutdown_connections 2024-05-20T20:11:19.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:19.161+0000 7fdbda400700 1 -- 172.21.15.28:0/1670277472 wait complete. 2024-05-20T20:11:19.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.188+0000 7f28d09e5740 1 Processor -- start 2024-05-20T20:11:19.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.188+0000 7f28d09e5740 1 -- start start 2024-05-20T20:11:19.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.189+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21bde70 0x5561e21d4d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.189+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21b9230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.189+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21c9050 0x5561e21e0310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.189+0000 7f28d09e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f28a800e700 con 0x5561e21bde70 2024-05-20T20:11:19.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28c7400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21c9050 0x5561e21e0310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28c7400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21c9050 0x5561e21e0310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:40034/0 (socket says 172.21.15.28:40034) 2024-05-20T20:11:19.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28c7400640 1 -- 172.21.15.28:0/3851590762 learned_addr learned my addr 172.21.15.28:0/3851590762 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28d09e5740 1 -- 172.21.15.28:0/3851590762 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f28a8010240 con 0x5561e21be790 2024-05-20T20:11:19.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28d09e5740 1 -- 172.21.15.28:0/3851590762 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f28a800edd0 con 0x5561e21c9050 2024-05-20T20:11:19.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28c7400640 1 -- 172.21.15.28:0/3851590762 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 msgr2=0x5561e21b9230 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:11:19.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28c7400640 1 --2- 172.21.15.28:0/3851590762 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21b9230 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28c7400640 1 -- 172.21.15.28:0/3851590762 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21bde70 msgr2=0x5561e21d4d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.190+0000 7f28c7400640 1 --2- 172.21.15.28:0/3851590762 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21bde70 0x5561e21d4d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.191+0000 7f28c7400640 1 -- 172.21.15.28:0/3851590762 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5561e21baf50 con 0x5561e21c9050 2024-05-20T20:11:19.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.192+0000 7f28c7400640 1 --2- 172.21.15.28:0/3851590762 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21c9050 0x5561e21e0310 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f28c0010010 tx=0x7f28c000d840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3898124bad52eb52 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.194+0000 7f28b3400640 1 -- 172.21.15.28:0/3851590762 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28c0005690 con 0x5561e21c9050 2024-05-20T20:11:19.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.194+0000 7f28b3400640 1 -- 172.21.15.28:0/3851590762 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f28c001c070 con 0x5561e21c9050 2024-05-20T20:11:19.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.194+0000 7f28b3400640 1 -- 172.21.15.28:0/3851590762 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28c0010e20 con 0x5561e21c9050 2024-05-20T20:11:19.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.196+0000 7f28d09e5740 1 -- 172.21.15.28:0/3851590762 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21c9050 msgr2=0x5561e21e0310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:19.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.196+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3851590762 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21c9050 0x5561e21e0310 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f28c0010010 tx=0x7f28c000d840 comp rx=0 tx=0).stop 2024-05-20T20:11:19.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.197+0000 7f28d09e5740 1 -- 172.21.15.28:0/3851590762 shutdown_connections 2024-05-20T20:11:19.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.197+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3851590762 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21b9230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.197+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3851590762 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21c9050 0x5561e21e0310 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.197+0000 7f28d09e5740 1 --2- 172.21.15.28:0/3851590762 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21bde70 0x5561e21d4d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.198+0000 7f28d09e5740 1 -- 172.21.15.28:0/3851590762 >> 172.21.15.28:0/3851590762 conn(0x5561e21ba380 msgr2=0x5561e21b8b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:19.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.198+0000 7f28d09e5740 1 -- 172.21.15.28:0/3851590762 shutdown_connections 2024-05-20T20:11:19.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.199+0000 7f28d09e5740 1 -- 172.21.15.28:0/3851590762 wait complete. 2024-05-20T20:11:19.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.200+0000 7f28d09e5740 1 -- start start 2024-05-20T20:11:19.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.201+0000 7f28d09e5740 1 Processor -- start 2024-05-20T20:11:19.205 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.206+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21d1500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.205 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.206+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21ce890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.205 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.206+0000 7f28d09e5740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21c9050 0x5561e21cedd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.206+0000 7f28d09e5740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f28a800fb70 con 0x5561e21c9050 2024-05-20T20:11:19.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.206+0000 7f28d09e5740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f28a800f4a0 con 0x5561e21be790 2024-05-20T20:11:19.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.207+0000 7f28d09e5740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f28a800c6e0 con 0x5561e21bde70 2024-05-20T20:11:19.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.207+0000 7f28d09e5740 10 client.0 _ll_register_callbacks cb 0x7f28b8034070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.207+0000 7f28c6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21ce890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.207+0000 7f28c6000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21ce890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58566/0 (socket says 172.21.15.28:58566) 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.207+0000 7f28c6000640 1 -- 172.21.15.28:0/1325340309 learned_addr learned my addr 172.21.15.28:0/1325340309 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.209+0000 7f28c7400640 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21c9050 0x5561e21cedd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.209+0000 7f28c6000640 1 -- 172.21.15.28:0/1325340309 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 msgr2=0x5561e21d1500 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.209+0000 7f28c6000640 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21d1500 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.209+0000 7f28c6000640 1 -- 172.21.15.28:0/1325340309 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21c9050 msgr2=0x5561e21cedd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:19.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.209+0000 7f28c6000640 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21c9050 0x5561e21cedd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:19.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.210+0000 7f28c6000640 1 -- 172.21.15.28:0/1325340309 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5561e21cf4c0 con 0x5561e21be790 2024-05-20T20:11:19.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.211+0000 7f28c6000640 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21ce890 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f28b40aa320 tx=0x7f28b40aa1f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ce8a72c78cc4a884 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:19.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.211+0000 7f28b0c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:19.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.212+0000 7f28b0c00640 1 -- 172.21.15.28:0/1325340309 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f28b401d040 con 0x5561e21be790 2024-05-20T20:11:19.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.213+0000 7f28b0c00640 1 -- 172.21.15.28:0/1325340309 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f28b4090b40 con 0x5561e21be790 2024-05-20T20:11:19.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.213+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f28c001c070 con 0x5561e21be790 2024-05-20T20:11:19.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.214+0000 7f28b0c00640 1 -- 172.21.15.28:0/1325340309 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f28b40e0f20 con 0x5561e21be790 2024-05-20T20:11:19.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.214+0000 7f28d09e5740 10 client.26657 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:19.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.214+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5561e2243100 con 0x5561e21be790 2024-05-20T20:11:19.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28b0c00640 1 -- 172.21.15.28:0/1325340309 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f28c001c070 con 0x5561e21be790 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28b0c00640 1 -- 172.21.15.28:0/1325340309 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f28b400fbb0 con 0x5561e21be790 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28d09e5740 10 client.26657 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28d09e5740 10 client.26657 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28d09e5740 20 client.26657 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28d09e5740 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e2243880 0x5561e2245ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28d09e5740 4 client.26657 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.215+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5561e212a2c0 con 0x5561e2243880 2024-05-20T20:11:19.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.216+0000 7f28c6a00640 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e2243880 0x5561e2245ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:19.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.216+0000 7f28c6a00640 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e2243880 0x5561e2245ca0 crc :-1 s=READY pgs=142 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=90c05168c142a360 server_cookie=58a83f2b38f6e69d in_seq=0 out_seq=0 2024-05-20T20:11:19.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:19.216+0000 7f28b0c00640 10 client.26657 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:19.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:18 smithi154 ceph-mon[36314]: pgmap v173: 97 pgs: 97 active+clean; 28 GiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 235 MiB/s wr, 1.98k op/s 2024-05-20T20:11:19.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:18 smithi033 ceph-mon[36179]: pgmap v173: 97 pgs: 97 active+clean; 28 GiB data, 6.2 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 235 MiB/s wr, 1.98k op/s 2024-05-20T20:11:20.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.918+0000 7f28b0c00640 1 -- 172.21.15.28:0/1325340309 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5561e212a2c0 con 0x5561e2243880 2024-05-20T20:11:20.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.918+0000 7f28b0c00640 10 client.26657 handle_command_reply: tid=0 2024-05-20T20:11:20.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.918+0000 7f28d09e5740 1 client.26657 shutdown 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 msgr2=0x5561e21ce890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21ce890 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f28b40aa320 tx=0x7f28b40aa1f0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 shutdown_connections 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5561e2243880 0x5561e2245ca0 crc :-1 s=READY pgs=142 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5561e21be790 0x5561e21ce890 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28b0c00640 0 client.26657 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5561e21c9050 0x5561e21cedd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 --2- 172.21.15.28:0/1325340309 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5561e21bde70 0x5561e21d1500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.920+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 >> 172.21.15.28:0/1325340309 conn(0x5561e21ba380 msgr2=0x5561e21b93b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:20.921 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.921+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 shutdown_connections 2024-05-20T20:11:20.921 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.921+0000 7f28d09e5740 1 -- 172.21.15.28:0/1325340309 wait complete. 2024-05-20T20:11:20.921 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.921+0000 7f28d09e5740 20 client.26657 trim_cache size 0 max 16384 2024-05-20T20:11:20.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:11:20.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:11:20.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:11:20.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28ac0b2aa0 msgr2=0x7f28ac0b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:20.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28ac0b2aa0 0x7f28ac0b4f60 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f28b8034040 tx=0x7f28b8005fd0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 msgr2=0x7f28c81a48c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:20.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 0x7f28c81a48c0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f28b400cab0 tx=0x7f28b400cf70 comp rx=0 tx=0).stop 2024-05-20T20:11:20.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 shutdown_connections 2024-05-20T20:11:20.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f28c8101320 0x7f28c81a48c0 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f28ac0b2aa0 0x7f28ac0b4f60 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28c810e940 0x7f28c810edc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 --2- 172.21.15.28:0/1462667316 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f28c8100960 0x7f28c81a4380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:20.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 >> 172.21.15.28:0/1462667316 conn(0x7f28c80778b0 msgr2=0x7f28c8103380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:20.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.924+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 shutdown_connections 2024-05-20T20:11:20.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:20.925+0000 7f28c7e00640 1 -- 172.21.15.28:0/1462667316 wait complete. 2024-05-20T20:11:21.309 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:20 smithi028 ceph-mon[31178]: pgmap v174: 97 pgs: 97 active+clean; 28 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 310 MiB/s wr, 2.75k op/s 2024-05-20T20:11:21.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:20 smithi033 ceph-mon[36179]: pgmap v174: 97 pgs: 97 active+clean; 28 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 310 MiB/s wr, 2.75k op/s 2024-05-20T20:11:21.379 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:20 smithi154 ceph-mon[36314]: pgmap v174: 97 pgs: 97 active+clean; 28 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 310 MiB/s wr, 2.75k op/s 2024-05-20T20:11:23.317 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:23 smithi028 ceph-mon[31178]: pgmap v175: 97 pgs: 97 active+clean; 29 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 234 MiB/s wr, 2.26k op/s 2024-05-20T20:11:23.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:23 smithi154 ceph-mon[36314]: pgmap v175: 97 pgs: 97 active+clean; 29 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 234 MiB/s wr, 2.26k op/s 2024-05-20T20:11:23.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:23 smithi033 ceph-mon[36179]: pgmap v175: 97 pgs: 97 active+clean; 29 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 234 MiB/s wr, 2.26k op/s 2024-05-20T20:11:25.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:25 smithi028 ceph-mon[31178]: pgmap v176: 97 pgs: 97 active+clean; 29 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 276 MiB/s wr, 2.73k op/s 2024-05-20T20:11:25.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:25 smithi154 ceph-mon[36314]: pgmap v176: 97 pgs: 97 active+clean; 29 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 276 MiB/s wr, 2.73k op/s 2024-05-20T20:11:25.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:25 smithi033 ceph-mon[36179]: pgmap v176: 97 pgs: 97 active+clean; 29 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 276 MiB/s wr, 2.73k op/s 2024-05-20T20:11:26.008 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:11:26.009 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:11:26.985 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:27.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:27 smithi028 ceph-mon[31178]: pgmap v177: 97 pgs: 97 active+clean; 29 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 219 MiB/s wr, 2.21k op/s 2024-05-20T20:11:27.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:27 smithi154 ceph-mon[36314]: pgmap v177: 97 pgs: 97 active+clean; 29 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 219 MiB/s wr, 2.21k op/s 2024-05-20T20:11:27.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:27 smithi033 ceph-mon[36179]: pgmap v177: 97 pgs: 97 active+clean; 29 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 219 MiB/s wr, 2.21k op/s 2024-05-20T20:11:28.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:28 smithi028 ceph-mon[31178]: pgmap v178: 97 pgs: 97 active+clean; 29 GiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 193 MiB/s wr, 1.98k op/s 2024-05-20T20:11:28.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:28 smithi154 ceph-mon[36314]: pgmap v178: 97 pgs: 97 active+clean; 29 GiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 193 MiB/s wr, 1.98k op/s 2024-05-20T20:11:28.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:28 smithi033 ceph-mon[36179]: pgmap v178: 97 pgs: 97 active+clean; 29 GiB data, 6.5 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 193 MiB/s wr, 1.98k op/s 2024-05-20T20:11:31.109 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:30 smithi028 ceph-mon[31178]: pgmap v179: 97 pgs: 97 active+clean; 30 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 275 MiB/s wr, 2.58k op/s 2024-05-20T20:11:31.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:30 smithi154 ceph-mon[36314]: pgmap v179: 97 pgs: 97 active+clean; 30 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 275 MiB/s wr, 2.58k op/s 2024-05-20T20:11:31.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:30 smithi033 ceph-mon[36179]: pgmap v179: 97 pgs: 97 active+clean; 30 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 275 MiB/s wr, 2.58k op/s 2024-05-20T20:11:32.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.014+0000 7f1ba7e00700 1 -- 172.21.15.28:0/4174226910 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0108460 msgr2=0x7f1ba006bae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.014+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/4174226910 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0108460 0x7f1ba006bae0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f1b94009990 tx=0x7f1b94009ca0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.014+0000 7f1ba7e00700 1 -- 172.21.15.28:0/4174226910 shutdown_connections 2024-05-20T20:11:32.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.014+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/4174226910 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1ba0105f70 0x7f1ba0106400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.014+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/4174226910 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0108460 0x7f1ba006bae0 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.014+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/4174226910 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 0x7f1ba006c480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.014+0000 7f1ba7e00700 1 -- 172.21.15.28:0/4174226910 >> 172.21.15.28:0/4174226910 conn(0x7f1ba00793b0 msgr2=0x7f1ba00797c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:32.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.015+0000 7f1ba7e00700 1 -- 172.21.15.28:0/4174226910 shutdown_connections 2024-05-20T20:11:32.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.015+0000 7f1ba7e00700 1 -- 172.21.15.28:0/4174226910 wait complete. 2024-05-20T20:11:32.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.018+0000 7f1ba7e00700 1 Processor -- start 2024-05-20T20:11:32.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.019+0000 7f1ba7e00700 1 -- start start 2024-05-20T20:11:32.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.019+0000 7f1ba7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 0x7f1ba01a0260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.019+0000 7f1ba7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0105f70 0x7f1ba01a07a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.019+0000 7f1ba7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1ba0108460 0x7f1ba01a4b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.019+0000 7f1ba7e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1ba0119d90 con 0x7f1ba0108460 2024-05-20T20:11:32.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.019+0000 7f1ba7e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1ba0119c10 con 0x7f1ba0105f70 2024-05-20T20:11:32.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.019+0000 7f1ba7e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1ba0119f10 con 0x7f1ba006c020 2024-05-20T20:11:32.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.020+0000 7f1ba5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 0x7f1ba01a0260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.020+0000 7f1ba5400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 0x7f1ba01a0260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43688/0 (socket says 172.21.15.28:43688) 2024-05-20T20:11:32.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.020+0000 7f1ba5400700 1 -- 172.21.15.28:0/431398095 learned_addr learned my addr 172.21.15.28:0/431398095 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:32.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.020+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1ba0108460 0x7f1ba01a4b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.021+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0105f70 0x7f1ba01a07a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.021+0000 7f1ba5400700 1 -- 172.21.15.28:0/431398095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0105f70 msgr2=0x7f1ba01a07a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.022+0000 7f1ba5400700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0105f70 0x7f1ba01a07a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.022+0000 7f1ba5400700 1 -- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1ba0108460 msgr2=0x7f1ba01a4b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.022+0000 7f1ba5400700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1ba0108460 0x7f1ba01a4b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.022+0000 7f1ba5400700 1 -- 172.21.15.28:0/431398095 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1ba01a5270 con 0x7f1ba006c020 2024-05-20T20:11:32.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.022+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0105f70 0x7f1ba01a07a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:32.023 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.023+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1ba0108460 0x7f1ba01a4b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:32.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.023+0000 7f1ba5400700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 0x7f1ba01a0260 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f1b940047e0 tx=0x7f1b94005e80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:32.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.032+0000 7f1b9e000700 1 -- 172.21.15.28:0/431398095 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b94027070 con 0x7f1ba006c020 2024-05-20T20:11:32.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.032+0000 7f1b9e000700 1 -- 172.21.15.28:0/431398095 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b94003f20 con 0x7f1ba006c020 2024-05-20T20:11:32.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.032+0000 7f1b9e000700 1 -- 172.21.15.28:0/431398095 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b94022bf0 con 0x7f1ba006c020 2024-05-20T20:11:32.037 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.032+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1ba01a5500 con 0x7f1ba006c020 2024-05-20T20:11:32.038 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.033+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1ba01acde0 con 0x7f1ba006c020 2024-05-20T20:11:32.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.048+0000 7f1b9e000700 1 -- 172.21.15.28:0/431398095 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1b9402a070 con 0x7f1ba006c020 2024-05-20T20:11:32.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.057+0000 7f1bb269db80 1 Processor -- start 2024-05-20T20:11:32.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.058+0000 7f1bb269db80 1 -- start start 2024-05-20T20:11:32.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.059+0000 7f1b9e000700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b880b2b40 0x7f1b880b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.059+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afb9310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.060+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afb32e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.060+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb3900 0x55bf0afb3d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.060+0000 7f1bb269db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bf0af11ec0 con 0x55bf0afb9850 2024-05-20T20:11:32.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.060+0000 7f1bb269db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bf0af01a60 con 0x55bf0afb8f30 2024-05-20T20:11:32.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.060+0000 7f1bb269db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bf0af05aa0 con 0x55bf0afb3900 2024-05-20T20:11:32.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.061+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b880b2b40 0x7f1b880b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.061+0000 7f1b9e000700 1 -- 172.21.15.28:0/431398095 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f1b940e3430 con 0x7f1ba006c020 2024-05-20T20:11:32.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.062+0000 7f1ba5e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb3900 0x55bf0afb3d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.062+0000 7f1ba5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afb9310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.062+0000 7f1ba5e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb3900 0x55bf0afb3d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43698/0 (socket says 172.21.15.28:43698) 2024-05-20T20:11:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.062+0000 7f1ba5e00700 1 -- 172.21.15.28:0/2623791652 learned_addr learned my addr 172.21.15.28:0/2623791652 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.063+0000 7f1ba5400700 1 -- 172.21.15.28:0/2623791652 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb3900 msgr2=0x55bf0afb3d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.063+0000 7f1ba5400700 1 --2- 172.21.15.28:0/2623791652 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb3900 0x55bf0afb3d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.063+0000 7f1ba5400700 1 -- 172.21.15.28:0/2623791652 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 msgr2=0x55bf0afb32e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.063+0000 7f1ba5400700 1 --2- 172.21.15.28:0/2623791652 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afb32e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.063+0000 7f1ba5400700 1 -- 172.21.15.28:0/2623791652 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bf0aeca4a0 con 0x55bf0afb8f30 2024-05-20T20:11:32.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.064+0000 7f1ba5400700 1 --2- 172.21.15.28:0/2623791652 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afb9310 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f1b940e0b30 tx=0x7f1b940a9f40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9edbad6b55409211 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:32.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.065+0000 7f1b92a00700 1 -- 172.21.15.28:0/2623791652 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b940aa750 con 0x55bf0afb8f30 2024-05-20T20:11:32.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.065+0000 7f1b92a00700 1 -- 172.21.15.28:0/2623791652 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b940aeb00 con 0x55bf0afb8f30 2024-05-20T20:11:32.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.066+0000 7f1bb269db80 1 -- 172.21.15.28:0/2623791652 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 msgr2=0x55bf0afb9310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.066+0000 7f1bb269db80 1 --2- 172.21.15.28:0/2623791652 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afb9310 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f1b940e0b30 tx=0x7f1b940a9f40 comp rx=0 tx=0).stop 2024-05-20T20:11:32.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.067+0000 7f1bb269db80 1 -- 172.21.15.28:0/2623791652 shutdown_connections 2024-05-20T20:11:32.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.067+0000 7f1bb269db80 1 --2- 172.21.15.28:0/2623791652 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb3900 0x55bf0afb3d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.068+0000 7f1bb269db80 1 --2- 172.21.15.28:0/2623791652 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afb9310 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.068+0000 7f1bb269db80 1 --2- 172.21.15.28:0/2623791652 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afb32e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.068+0000 7f1bb269db80 1 -- 172.21.15.28:0/2623791652 >> 172.21.15.28:0/2623791652 conn(0x55bf0aee1c60 msgr2=0x55bf0afae090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:32.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.068+0000 7f1bb269db80 1 -- 172.21.15.28:0/2623791652 shutdown_connections 2024-05-20T20:11:32.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.069+0000 7f1bb269db80 1 -- 172.21.15.28:0/2623791652 wait complete. 2024-05-20T20:11:32.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.069+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b880b2b40 0x7f1b880b5000 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f1b98011920 tx=0x7f1b98007a80 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:32.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.069+0000 7f1bb269db80 1 -- start start 2024-05-20T20:11:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.070+0000 7f1bb269db80 1 Processor -- start 2024-05-20T20:11:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.071+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afbf420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.071+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb8f30 0x55bf0afbc7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.071+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afbcd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.071+0000 7f1bb269db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bf0ae08020 con 0x55bf0afb9850 2024-05-20T20:11:32.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.071+0000 7f1bb269db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bf0aea27e0 con 0x55bf0afb3900 2024-05-20T20:11:32.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.071+0000 7f1bb269db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bf0afd1090 con 0x55bf0afb8f30 2024-05-20T20:11:32.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.072+0000 7f1ba5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afbf420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.072+0000 7f1ba5400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afbf420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44892/0 (socket says 172.21.15.28:44892) 2024-05-20T20:11:32.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.072+0000 7f1ba5400700 1 -- 172.21.15.28:0/152230511 learned_addr learned my addr 172.21.15.28:0/152230511 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:32.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.072+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb8f30 0x55bf0afbc7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.073+0000 7f1ba4a00700 1 -- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 msgr2=0x55bf0afbf420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:32.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.073+0000 7f1bb269db80 10 client.0 _ll_register_callbacks cb 0x55bf0aeabcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:32.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.073+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afbf420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.073+0000 7f1ba4a00700 1 -- 172.21.15.28:0/152230511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 msgr2=0x55bf0afbcd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:32.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.073+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afbcd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.073+0000 7f1ba4a00700 1 -- 172.21.15.28:0/152230511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bf0ae51950 con 0x55bf0afb8f30 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.074+0000 7f1ba5400700 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afbf420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.074+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1b940aeb00 con 0x55bf0afb8f30 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.076+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb8f30 0x55bf0afbc7c0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f1ba006be00 tx=0x7f1b8c0045a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=677824c1638b56d0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.076+0000 7f1b87400700 10 client.26590 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.077+0000 7f1b87400700 1 -- 172.21.15.28:0/152230511 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b8c01a470 con 0x55bf0afb8f30 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.078+0000 7f1b87400700 1 -- 172.21.15.28:0/152230511 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f1b8c01a610 con 0x55bf0afb8f30 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.078+0000 7f1b87400700 1 -- 172.21.15.28:0/152230511 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1b8c01a870 con 0x55bf0afb8f30 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.078+0000 7f1bb269db80 10 client.26590 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:32.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.078+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x55bf0aed20e0 con 0x55bf0afb8f30 2024-05-20T20:11:32.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.079+0000 7f1b87400700 1 -- 172.21.15.28:0/152230511 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1b8c01aa10 con 0x55bf0afb8f30 2024-05-20T20:11:32.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.079+0000 7f1bb269db80 10 client.26590 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:32.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.079+0000 7f1bb269db80 10 client.26590 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:32.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.079+0000 7f1bb269db80 20 client.26590 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:32.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.079+0000 7f1bb269db80 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0afdd8c0 0x55bf0afdfd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.079+0000 7f1bb269db80 4 client.26590 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:32.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.079+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55bf0af13e10 con 0x55bf0afdd8c0 2024-05-20T20:11:32.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.082+0000 7f1ba5400700 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0afdd8c0 0x55bf0afdfd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.082+0000 7f1ba5400700 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0afdd8c0 0x55bf0afdfd00 crc :-1 s=READY pgs=143 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f131d50d3871064c server_cookie=2d9407356c7b2804 in_seq=0 out_seq=0 2024-05-20T20:11:32.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.082+0000 7f1b87400700 10 client.26590 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:32.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.085+0000 7f1b87400700 1 -- 172.21.15.28:0/152230511 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55bf0af13e10 con 0x55bf0afdd8c0 2024-05-20T20:11:32.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.085+0000 7f1b87400700 10 client.26590 handle_command_reply: tid=0 2024-05-20T20:11:32.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.087+0000 7f1bb269db80 1 client.26590 shutdown 2024-05-20T20:11:32.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.090+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb8f30 msgr2=0x55bf0afbc7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.090+0000 7f1bb269db80 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb8f30 0x55bf0afbc7c0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f1ba006be00 tx=0x7f1b8c0045a0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.091+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 shutdown_connections 2024-05-20T20:11:32.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.091+0000 7f1bb269db80 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0afdd8c0 0x55bf0afdfd00 crc :-1 s=READY pgs=143 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.092+0000 7f1bb269db80 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afbcd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.092+0000 7f1bb269db80 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afbf420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.092+0000 7f1bb269db80 1 --2- 172.21.15.28:0/152230511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb8f30 0x55bf0afbc7c0 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.092+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 >> 172.21.15.28:0/152230511 conn(0x55bf0aee1c60 msgr2=0x55bf0afb22d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:32.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.096+0000 7f1b87400700 0 client.26590 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:32.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.096+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 shutdown_connections 2024-05-20T20:11:32.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.097+0000 7f1bb269db80 1 -- 172.21.15.28:0/152230511 wait complete. 2024-05-20T20:11:32.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.097+0000 7f1bb269db80 20 client.26590 trim_cache size 0 max 16384 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 Processor -- start 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 -- start start 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afb32e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb8f30 0x55bf0afba340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb9850 0x55bf0afada60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1b8000d6c0 con 0x55bf0afb8f30 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1b8000f200 con 0x55bf0afb3900 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.155+0000 7f1bb269db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1b8000dd90 con 0x55bf0afb9850 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.156+0000 7f1ba5e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb9850 0x55bf0afada60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.156+0000 7f1ba5e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb9850 0x55bf0afada60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43716/0 (socket says 172.21.15.28:43716) 2024-05-20T20:11:32.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.156+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3433202325 learned_addr learned my addr 172.21.15.28:0/3433202325 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.156+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb8f30 0x55bf0afba340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.157+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3433202325 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 msgr2=0x55bf0afb32e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.157+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afb32e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.157+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3433202325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb8f30 msgr2=0x55bf0afba340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.157+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb8f30 0x55bf0afba340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.157+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3433202325 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bf0afa7c20 con 0x55bf0afb9850 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.157+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb8f30 0x55bf0afba340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.157+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb9850 0x55bf0afada60 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f1b98011590 tx=0x7f1b98015040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=73f3d303497788b2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:32.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.158+0000 7f1b92000700 1 -- 172.21.15.28:0/3433202325 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b98011df0 con 0x55bf0afb9850 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.158+0000 7f1b92000700 1 -- 172.21.15.28:0/3433202325 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b98023070 con 0x55bf0afb9850 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.158+0000 7f1b92000700 1 -- 172.21.15.28:0/3433202325 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b9801d620 con 0x55bf0afb9850 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.159+0000 7f1bb269db80 1 -- 172.21.15.28:0/3433202325 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb9850 msgr2=0x55bf0afada60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.159+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb9850 0x55bf0afada60 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f1b98011590 tx=0x7f1b98015040 comp rx=0 tx=0).stop 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.159+0000 7f1bb269db80 1 -- 172.21.15.28:0/3433202325 shutdown_connections 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.159+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afb9850 0x55bf0afada60 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.159+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb3900 0x55bf0afb32e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.159+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3433202325 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb8f30 0x55bf0afba340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.159+0000 7f1bb269db80 1 -- 172.21.15.28:0/3433202325 >> 172.21.15.28:0/3433202325 conn(0x55bf0aee1c60 msgr2=0x55bf0afb9c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:32.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.161+0000 7f1bb269db80 1 -- 172.21.15.28:0/3433202325 shutdown_connections 2024-05-20T20:11:32.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.162+0000 7f1bb269db80 1 -- 172.21.15.28:0/3433202325 wait complete. 2024-05-20T20:11:32.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.162+0000 7f1bb269db80 1 -- start start 2024-05-20T20:11:32.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.163+0000 7f1bb269db80 1 Processor -- start 2024-05-20T20:11:32.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.164+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afd6a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.165+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afd32b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.165+0000 7f1bb269db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afd37f0 0x55bf0afd3c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.166+0000 7f1bb269db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1b8000eb30 con 0x55bf0afb9850 2024-05-20T20:11:32.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.166+0000 7f1bb269db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1b8000e460 con 0x55bf0afb8f30 2024-05-20T20:11:32.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.166+0000 7f1bb269db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1b8000b660 con 0x55bf0afd37f0 2024-05-20T20:11:32.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.166+0000 7f1bb269db80 10 client.0 _ll_register_callbacks cb 0x55bf0aeabcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:32.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.167+0000 7f1ba5e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afd37f0 0x55bf0afd3c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.167+0000 7f1ba5e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afd37f0 0x55bf0afd3c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43730/0 (socket says 172.21.15.28:43730) 2024-05-20T20:11:32.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.167+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3598208106 learned_addr learned my addr 172.21.15.28:0/3598208106 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:32.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.168+0000 7f1ba5400700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afd6a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.169+0000 7f1ba4a00700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afd32b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.170+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 msgr2=0x55bf0afd6a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:32.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.170+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afd6a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.170+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3598208106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 msgr2=0x55bf0afd32b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:32.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.170+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afd32b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.170+0000 7f1ba5e00700 1 -- 172.21.15.28:0/3598208106 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bf0b048680 con 0x55bf0afd37f0 2024-05-20T20:11:32.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.171+0000 7f1ba5400700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afd6a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:32.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.174+0000 7f1ba5e00700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afd37f0 0x55bf0afd3c50 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f1b98011560 tx=0x7f1b98007310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=648cbca61539295e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:32.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.174+0000 7f1b87400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:32.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.175+0000 7f1b87400700 1 -- 172.21.15.28:0/3598208106 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b9801e870 con 0x55bf0afd37f0 2024-05-20T20:11:32.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.175+0000 7f1b87400700 1 -- 172.21.15.28:0/3598208106 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f1b9801ea10 con 0x55bf0afd37f0 2024-05-20T20:11:32.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.176+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1b98023070 con 0x55bf0afd37f0 2024-05-20T20:11:32.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.177+0000 7f1b87400700 1 -- 172.21.15.28:0/3598208106 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1b98033480 con 0x55bf0afd37f0 2024-05-20T20:11:32.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.177+0000 7f1bb269db80 10 client.26596 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:32.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.177+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55bf0b0499c0 con 0x55bf0afd37f0 2024-05-20T20:11:32.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.178+0000 7f1b87400700 1 -- 172.21.15.28:0/3598208106 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b98023070 con 0x55bf0afd37f0 2024-05-20T20:11:32.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.178+0000 7f1b87400700 1 -- 172.21.15.28:0/3598208106 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1b9803d690 con 0x55bf0afd37f0 2024-05-20T20:11:32.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.179+0000 7f1bb269db80 10 client.26596 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:32.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.179+0000 7f1bb269db80 10 client.26596 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:32.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.179+0000 7f1bb269db80 20 client.26596 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:32.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.179+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0b049f50 0x55bf0b04c370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:32.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.179+0000 7f1bb269db80 4 client.26596 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:11:32.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.179+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x55bf0af0d5f0 con 0x55bf0b049f50 2024-05-20T20:11:32.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.181+0000 7f1ba5400700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0b049f50 0x55bf0b04c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:32.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.183+0000 7f1ba5400700 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0b049f50 0x55bf0b04c370 crc :-1 s=READY pgs=144 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4389fffb23e4456 server_cookie=d1b8a7c39d3cf09f in_seq=0 out_seq=0 2024-05-20T20:11:32.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.183+0000 7f1b87400700 10 client.26596 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:32.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.185+0000 7f1b87400700 1 -- 172.21.15.28:0/3598208106 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x55bf0af0d5f0 con 0x55bf0b049f50 2024-05-20T20:11:32.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.185+0000 7f1b87400700 10 client.26596 handle_command_reply: tid=0 2024-05-20T20:11:32.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.185+0000 7f1bb269db80 1 client.26596 shutdown 2024-05-20T20:11:32.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.186+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afd37f0 msgr2=0x55bf0afd3c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.186+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afd37f0 0x55bf0afd3c50 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f1b98011560 tx=0x7f1b98007310 comp rx=0 tx=0).stop 2024-05-20T20:11:32.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.189+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 shutdown_connections 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.189+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bf0b049f50 0x55bf0b04c370 crc :-1 s=READY pgs=144 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.189+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bf0afd37f0 0x55bf0afd3c50 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.189+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bf0afb8f30 0x55bf0afd6a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.189+0000 7f1bb269db80 1 --2- 172.21.15.28:0/3598208106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bf0afb9850 0x55bf0afd32b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.189+0000 7f1b87400700 0 client.26596 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.189+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 >> 172.21.15.28:0/3598208106 conn(0x55bf0aee1c60 msgr2=0x55bf0afbaf10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.190+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 shutdown_connections 2024-05-20T20:11:32.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.190+0000 7f1bb269db80 1 -- 172.21.15.28:0/3598208106 wait complete. 2024-05-20T20:11:32.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.191+0000 7f1bb269db80 20 client.26596 trim_cache size 0 max 16384 2024-05-20T20:11:32.194 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:11:32.194 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:11:32.194 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "8986377e-4061-4dd7-a5ee-b3ba4e3f85c2", 2024-05-20T20:11:32.195 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:11:32.195 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:11:32.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.195+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b880b2b40 msgr2=0x7f1b880b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.195+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b880b2b40 0x7f1b880b5000 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f1b98011920 tx=0x7f1b98007a80 comp rx=0 tx=0).stop 2024-05-20T20:11:32.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.195+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 msgr2=0x7f1ba01a0260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:32.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.195+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 0x7f1ba01a0260 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f1b940047e0 tx=0x7f1b94005e80 comp rx=0 tx=0).stop 2024-05-20T20:11:32.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.196+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 shutdown_connections 2024-05-20T20:11:32.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.196+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1ba0108460 0x7f1ba01a4b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.196+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1ba006c020 0x7f1ba01a0260 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.196+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b880b2b40 0x7f1b880b5000 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.196+0000 7f1ba7e00700 1 --2- 172.21.15.28:0/431398095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1ba0105f70 0x7f1ba01a07a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:32.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.196+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 >> 172.21.15.28:0/431398095 conn(0x7f1ba00793b0 msgr2=0x7f1ba0101420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:32.201 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.200+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 shutdown_connections 2024-05-20T20:11:32.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:32.204+0000 7f1ba7e00700 1 -- 172.21.15.28:0/431398095 wait complete. 2024-05-20T20:11:33.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:32 smithi028 ceph-mon[31178]: pgmap v180: 97 pgs: 97 active+clean; 31 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 243 MiB/s wr, 2.10k op/s 2024-05-20T20:11:33.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:32 smithi154 ceph-mon[36314]: pgmap v180: 97 pgs: 97 active+clean; 31 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 243 MiB/s wr, 2.10k op/s 2024-05-20T20:11:33.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:32 smithi033 ceph-mon[36179]: pgmap v180: 97 pgs: 97 active+clean; 31 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 243 MiB/s wr, 2.10k op/s 2024-05-20T20:11:34.003 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:11:34.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.194+0000 7f469dc00640 1 -- 172.21.15.28:0/2668106447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 msgr2=0x7f4698107d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.194+0000 7f469dc00640 1 --2- 172.21.15.28:0/2668106447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f4698107d20 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f4688009900 tx=0x7f468802a3f0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.194+0000 7f469dc00640 1 -- 172.21.15.28:0/2668106447 shutdown_connections 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.194+0000 7f469dc00640 1 --2- 172.21.15.28:0/2668106447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4698102340 0x7f469810e800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.194+0000 7f469dc00640 1 --2- 172.21.15.28:0/2668106447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4698101980 0x7f4698101e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.194+0000 7f469dc00640 1 --2- 172.21.15.28:0/2668106447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f4698107d20 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.194+0000 7f469dc00640 1 -- 172.21.15.28:0/2668106447 >> 172.21.15.28:0/2668106447 conn(0x7f4698078ad0 msgr2=0x7f4698078ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.195+0000 7f469dc00640 1 -- 172.21.15.28:0/2668106447 shutdown_connections 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.195+0000 7f469dc00640 1 -- 172.21.15.28:0/2668106447 wait complete. 2024-05-20T20:11:34.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.196+0000 7f469dc00640 1 Processor -- start 2024-05-20T20:11:34.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.196+0000 7f469dc00640 1 -- start start 2024-05-20T20:11:34.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.197+0000 7f469dc00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4698101980 0x7f46981a5400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.197+0000 7f469dc00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4698102340 0x7f46981a5940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.197+0000 7f469dc00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f469819f5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.197+0000 7f469dc00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4698110b60 con 0x7f4698101980 2024-05-20T20:11:34.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.197+0000 7f469dc00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f46981109e0 con 0x7f4698102340 2024-05-20T20:11:34.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.197+0000 7f469dc00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4698110ce0 con 0x7f4698107940 2024-05-20T20:11:34.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.198+0000 7f4697400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f469819f5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.198+0000 7f4697400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f469819f5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43750/0 (socket says 172.21.15.28:43750) 2024-05-20T20:11:34.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.198+0000 7f4697400640 1 -- 172.21.15.28:0/530091508 learned_addr learned my addr 172.21.15.28:0/530091508 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:34.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.198+0000 7f4696a00640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4698101980 0x7f46981a5400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.198+0000 7f4696000640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4698102340 0x7f46981a5940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.199+0000 7f4697400640 1 -- 172.21.15.28:0/530091508 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4698102340 msgr2=0x7f46981a5940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.199+0000 7f4697400640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4698102340 0x7f46981a5940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.199+0000 7f4697400640 1 -- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4698101980 msgr2=0x7f46981a5400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.199+0000 7f4697400640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4698101980 0x7f46981a5400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.199+0000 7f4697400640 1 -- 172.21.15.28:0/530091508 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f469819fd00 con 0x7f4698107940 2024-05-20T20:11:34.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.200+0000 7f4697400640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f469819f5b0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f469000db30 tx=0x7f469000b260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:34.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.201+0000 7f468f400640 1 -- 172.21.15.28:0/530091508 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4690015070 con 0x7f4698107940 2024-05-20T20:11:34.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.202+0000 7f468f400640 1 -- 172.21.15.28:0/530091508 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4690005020 con 0x7f4698107940 2024-05-20T20:11:34.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.202+0000 7f468f400640 1 -- 172.21.15.28:0/530091508 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f469000bea0 con 0x7f4698107940 2024-05-20T20:11:34.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.202+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f469819fff0 con 0x7f4698107940 2024-05-20T20:11:34.205 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.205+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4698196cd0 con 0x7f4698107940 2024-05-20T20:11:34.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.210+0000 7f46a0a07740 1 Processor -- start 2024-05-20T20:11:34.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.218+0000 7f468f400640 1 -- 172.21.15.28:0/530091508 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f46900054d0 con 0x7f4698107940 2024-05-20T20:11:34.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.221+0000 7f468f400640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f467c0b2a90 0x7f467c0b4f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.221+0000 7f468f400640 1 -- 172.21.15.28:0/530091508 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f46900a5e10 con 0x7f4698107940 2024-05-20T20:11:34.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.221+0000 7f4696000640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f467c0b2a90 0x7f467c0b4f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.221+0000 7f46a0a07740 1 -- start start 2024-05-20T20:11:34.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.222+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ecc90 0x5593623ed070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.222+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 0x5593623f7930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.222+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fa260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.222+0000 7f46a0a07740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55936235db90 con 0x5593623ecc90 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.222+0000 7f46a0a07740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55936234d730 con 0x5593623f7e70 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.222+0000 7f46a0a07740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559362351770 con 0x5593623ed5b0 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fa260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fa260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44930/0 (socket says 172.21.15.28:44930) 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 -- 172.21.15.28:0/964916018 learned_addr learned my addr 172.21.15.28:0/964916018 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 -- 172.21.15.28:0/964916018 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 msgr2=0x5593623f7930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 --2- 172.21.15.28:0/964916018 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 0x5593623f7930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 -- 172.21.15.28:0/964916018 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ecc90 msgr2=0x5593623ed070 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:34.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 --2- 172.21.15.28:0/964916018 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ecc90 0x5593623ed070 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 -- 172.21.15.28:0/964916018 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5593622e68b0 con 0x5593623f7e70 2024-05-20T20:11:34.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.223+0000 7f4697400640 1 --2- 172.21.15.28:0/964916018 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fa260 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f469000db00 tx=0x7f46900a9f20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e736156d029cb7cf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:34.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.224+0000 7f4696000640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f467c0b2a90 0x7f467c0b4f50 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f4684005e10 tx=0x7f4684005da0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:34.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.227+0000 7f4683e00640 1 -- 172.21.15.28:0/964916018 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f46900adbd0 con 0x5593623f7e70 2024-05-20T20:11:34.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.227+0000 7f4683e00640 1 -- 172.21.15.28:0/964916018 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f46900b2680 con 0x5593623f7e70 2024-05-20T20:11:34.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.227+0000 7f4683e00640 1 -- 172.21.15.28:0/964916018 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f46900a8ce0 con 0x5593623f7e70 2024-05-20T20:11:34.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.228+0000 7f46a0a07740 1 -- 172.21.15.28:0/964916018 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 msgr2=0x5593623fa260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.228+0000 7f46a0a07740 1 --2- 172.21.15.28:0/964916018 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fa260 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f469000db00 tx=0x7f46900a9f20 comp rx=0 tx=0).stop 2024-05-20T20:11:34.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.228+0000 7f46a0a07740 1 -- 172.21.15.28:0/964916018 shutdown_connections 2024-05-20T20:11:34.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.228+0000 7f46a0a07740 1 --2- 172.21.15.28:0/964916018 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fa260 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.228+0000 7f46a0a07740 1 --2- 172.21.15.28:0/964916018 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 0x5593623f7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.228+0000 7f46a0a07740 1 --2- 172.21.15.28:0/964916018 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ecc90 0x5593623ed070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.228+0000 7f46a0a07740 1 -- 172.21.15.28:0/964916018 >> 172.21.15.28:0/964916018 conn(0x5593623e91a0 msgr2=0x5593623e9580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:34.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.230+0000 7f46a0a07740 1 -- 172.21.15.28:0/964916018 shutdown_connections 2024-05-20T20:11:34.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.230+0000 7f46a0a07740 1 -- 172.21.15.28:0/964916018 wait complete. 2024-05-20T20:11:34.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.230+0000 7f46a0a07740 1 -- start start 2024-05-20T20:11:34.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.231+0000 7f46a0a07740 1 Processor -- start 2024-05-20T20:11:34.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.231+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623f7870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.231+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623f4b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.231+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623f7e70 0x5593623f5170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f46a0a07740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559362033790 con 0x5593623ed5b0 2024-05-20T20:11:34.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f46a0a07740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5593622f5bc0 con 0x5593623ecc90 2024-05-20T20:11:34.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f46a0a07740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5593623faaa0 con 0x5593623f7e70 2024-05-20T20:11:34.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f46a0a07740 10 client.0 _ll_register_callbacks cb 0x5593622d3400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:34.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f4697400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623f7e70 0x5593623f5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f4697400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623f7e70 0x5593623f5170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43764/0 (socket says 172.21.15.28:43764) 2024-05-20T20:11:34.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f4697400640 1 -- 172.21.15.28:0/2444709447 learned_addr learned my addr 172.21.15.28:0/2444709447 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:34.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f4696a00640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623f7870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f4696a00640 1 -- 172.21.15.28:0/2444709447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623f7e70 msgr2=0x5593623f5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:34.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f4696a00640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623f7e70 0x5593623f5170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.232+0000 7f4696a00640 1 -- 172.21.15.28:0/2444709447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 msgr2=0x5593623f4b60 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:34.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.233+0000 7f4696000640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623f4b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.234+0000 7f4696a00640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623f4b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.234+0000 7f4696a00640 1 -- 172.21.15.28:0/2444709447 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559362062210 con 0x5593623ecc90 2024-05-20T20:11:34.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.234+0000 7f4696000640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623f4b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:34.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.237+0000 7f4696a00640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623f7870 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f468802a9e0 tx=0x7f46880043b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=856387317cf5bd2d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:34.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.237+0000 7f4680c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:34.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.240+0000 7f4680c00640 1 -- 172.21.15.28:0/2444709447 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4688051070 con 0x5593623ecc90 2024-05-20T20:11:34.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.242+0000 7f4680c00640 1 -- 172.21.15.28:0/2444709447 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f4688005450 con 0x5593623ecc90 2024-05-20T20:11:34.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.242+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f46900b2680 con 0x5593623ecc90 2024-05-20T20:11:34.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.243+0000 7f4680c00640 1 -- 172.21.15.28:0/2444709447 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4688035040 con 0x5593623ecc90 2024-05-20T20:11:34.244 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.243+0000 7f46a0a07740 10 client.26684 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:34.244 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.243+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55936212e620 con 0x5593623ecc90 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f4680c00640 1 -- 172.21.15.28:0/2444709447 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f46900b2680 con 0x5593623ecc90 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f4680c00640 1 -- 172.21.15.28:0/2444709447 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4688003940 con 0x5593623ecc90 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f46a0a07740 10 client.26684 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f46a0a07740 10 client.26684 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f46a0a07740 20 client.26684 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f46a0a07740 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x559362409c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f46a0a07740 4 client.26684 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:34.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.247+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55936235fae0 con 0x559362407830 2024-05-20T20:11:34.254 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.251+0000 7f4696000640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x559362409c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.254+0000 7f4696000640 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x559362409c20 crc :-1 s=READY pgs=145 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f4c633b8adc66bb1 server_cookie=d50bddf92807ffff in_seq=0 out_seq=0 2024-05-20T20:11:34.257 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.254+0000 7f4680c00640 10 client.26684 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:34.257 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.257+0000 7f4680c00640 1 -- 172.21.15.28:0/2444709447 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55936235fae0 con 0x559362407830 2024-05-20T20:11:34.258 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.257+0000 7f4680c00640 10 client.26684 handle_command_reply: tid=0 2024-05-20T20:11:34.258 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.257+0000 7f46a0a07740 1 client.26684 shutdown 2024-05-20T20:11:34.260 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.259+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 msgr2=0x5593623f7870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.260 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f46a0a07740 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623f7870 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f468802a9e0 tx=0x7f46880043b0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.261 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 shutdown_connections 2024-05-20T20:11:34.261 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f46a0a07740 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x559362409c20 crc :-1 s=READY pgs=145 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.261 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f4680c00640 0 client.26684 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:34.261 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f46a0a07740 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623f7e70 0x5593623f5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.261 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f46a0a07740 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623f4b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.261 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f46a0a07740 1 --2- 172.21.15.28:0/2444709447 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623f7870 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.261 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.260+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 >> 172.21.15.28:0/2444709447 conn(0x5593623e91a0 msgr2=0x5593623eef20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:34.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.278+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 shutdown_connections 2024-05-20T20:11:34.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.278+0000 7f46a0a07740 1 -- 172.21.15.28:0/2444709447 wait complete. 2024-05-20T20:11:34.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.279+0000 7f46a0a07740 20 client.26684 trim_cache size 0 max 16384 2024-05-20T20:11:34.311 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:33 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:11:34.311 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:33 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:11:34.311 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:33 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:11:34.312 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:33 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:11:34.312 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:33 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:11:34.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.350+0000 7f46a0a07740 1 Processor -- start 2024-05-20T20:11:34.351 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.350+0000 7f46a0a07740 1 -- start start 2024-05-20T20:11:34.351 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623eeda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 0x5593623f73a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623f7e70 0x55936240f510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f46a0a07740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f469806c680 con 0x5593623f7e70 2024-05-20T20:11:34.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f46a0a07740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f469806e1c0 con 0x5593623ecc90 2024-05-20T20:11:34.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f46a0a07740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f469806cd50 con 0x5593623ed5b0 2024-05-20T20:11:34.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f4697400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623f7e70 0x55936240f510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f4696a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623eeda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f4696a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623eeda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:44958/0 (socket says 172.21.15.28:44958) 2024-05-20T20:11:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.351+0000 7f4696a00640 1 -- 172.21.15.28:0/1852131342 learned_addr learned my addr 172.21.15.28:0/1852131342 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.352+0000 7f4696000640 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 0x5593623f73a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.352+0000 7f4697400640 1 -- 172.21.15.28:0/1852131342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 msgr2=0x5593623f73a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.352+0000 7f4697400640 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 0x5593623f73a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.352+0000 7f4697400640 1 -- 172.21.15.28:0/1852131342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 msgr2=0x5593623eeda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.352+0000 7f4697400640 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623eeda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.352+0000 7f4697400640 1 -- 172.21.15.28:0/1852131342 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5593623fa5d0 con 0x5593623f7e70 2024-05-20T20:11:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.353+0000 7f4697400640 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623f7e70 0x55936240f510 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f46900a9520 tx=0x7f46900cc040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=60d6beea61c69937 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.353+0000 7f4683400640 1 -- 172.21.15.28:0/1852131342 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f46900b2680 con 0x5593623f7e70 2024-05-20T20:11:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.354+0000 7f4683400640 1 -- 172.21.15.28:0/1852131342 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f46900a5ab0 con 0x5593623f7e70 2024-05-20T20:11:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.354+0000 7f4683400640 1 -- 172.21.15.28:0/1852131342 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4690091070 con 0x5593623f7e70 2024-05-20T20:11:34.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.355+0000 7f46a0a07740 1 -- 172.21.15.28:0/1852131342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623f7e70 msgr2=0x55936240f510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.355+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623f7e70 0x55936240f510 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f46900a9520 tx=0x7f46900cc040 comp rx=0 tx=0).stop 2024-05-20T20:11:34.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.356+0000 7f46a0a07740 1 -- 172.21.15.28:0/1852131342 shutdown_connections 2024-05-20T20:11:34.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.356+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623f7e70 0x55936240f510 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.356+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ed5b0 0x5593623f73a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.356+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1852131342 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623ecc90 0x5593623eeda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.356+0000 7f46a0a07740 1 -- 172.21.15.28:0/1852131342 >> 172.21.15.28:0/1852131342 conn(0x5593623e91a0 msgr2=0x5593623ee690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:34.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.356+0000 7f46a0a07740 1 -- 172.21.15.28:0/1852131342 shutdown_connections 2024-05-20T20:11:34.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.357+0000 7f46a0a07740 1 -- 172.21.15.28:0/1852131342 wait complete. 2024-05-20T20:11:34.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.357+0000 7f46a0a07740 1 -- start start 2024-05-20T20:11:34.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.357+0000 7f46a0a07740 1 Processor -- start 2024-05-20T20:11:34.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.358+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ecc90 0x5593623fbb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.358+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623fc050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.358+0000 7f46a0a07740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fc590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.358+0000 7f46a0a07740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f469806daf0 con 0x5593623ed5b0 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.358+0000 7f46a0a07740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f469806d420 con 0x5593623f7e70 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.358+0000 7f46a0a07740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f469819d6d0 con 0x5593623ecc90 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.358+0000 7f4696a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ecc90 0x5593623fbb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4696a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ecc90 0x5593623fbb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43790/0 (socket says 172.21.15.28:43790) 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4696a00640 1 -- 172.21.15.28:0/1902235320 learned_addr learned my addr 172.21.15.28:0/1902235320 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4696000640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623fc050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4697400640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fc590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f46a0a07740 10 client.0 _ll_register_callbacks cb 0x7f4688009780 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4697400640 1 -- 172.21.15.28:0/1902235320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ecc90 msgr2=0x5593623fbb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4697400640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ecc90 0x5593623fbb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4697400640 1 -- 172.21.15.28:0/1902235320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 msgr2=0x5593623fc050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4697400640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623fc050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.359+0000 7f4697400640 1 -- 172.21.15.28:0/1902235320 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5593624714b0 con 0x5593623f7e70 2024-05-20T20:11:34.363 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.363+0000 7f4696a00640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ecc90 0x5593623fbb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:34.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.363+0000 7f4696000640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623fc050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:34.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.363+0000 7f4697400640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fc590 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f46900a58a0 tx=0x7f4690005b00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a013cf2a68bf1187 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:34.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.363+0000 7f4680c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:34.364 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.363+0000 7f4680c00640 1 -- 172.21.15.28:0/1902235320 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4690087070 con 0x5593623f7e70 2024-05-20T20:11:34.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.364+0000 7f4680c00640 1 -- 172.21.15.28:0/1902235320 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f46900aad50 con 0x5593623f7e70 2024-05-20T20:11:34.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.364+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f46900a5ab0 con 0x5593623f7e70 2024-05-20T20:11:34.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.364+0000 7f4680c00640 1 -- 172.21.15.28:0/1902235320 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f46900b2680 con 0x5593623f7e70 2024-05-20T20:11:34.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.364+0000 7f46a0a07740 10 client.26690 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:34.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.364+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5593624727c0 con 0x5593623f7e70 2024-05-20T20:11:34.365 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.365+0000 7f4680c00640 1 -- 172.21.15.28:0/1902235320 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f46900a5ab0 con 0x5593623f7e70 2024-05-20T20:11:34.366 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.365+0000 7f4680c00640 1 -- 172.21.15.28:0/1902235320 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f46900a5290 con 0x5593623f7e70 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.365+0000 7f46a0a07740 10 client.26690 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.366+0000 7f46a0a07740 10 client.26690 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.366+0000 7f46a0a07740 20 client.26690 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.366+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x5593624749a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.366+0000 7f46a0a07740 4 client.26690 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.366+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5593623592c0 con 0x559362407830 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.366+0000 7f4696a00640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x5593624749a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:34.367 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.367+0000 7f4696a00640 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x5593624749a0 crc :-1 s=READY pgs=146 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c21565540d0c9dda server_cookie=61c27173424450c3 in_seq=0 out_seq=0 2024-05-20T20:11:34.368 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.367+0000 7f4680c00640 10 client.26690 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:34.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:33 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:11:34.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:33 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:11:34.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:33 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:11:34.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:33 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:11:34.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:33 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:11:34.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:33 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:11:34.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:33 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:11:34.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:33 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:11:34.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:33 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:11:34.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:33 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:11:34.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.411+0000 7f4680c00640 1 -- 172.21.15.28:0/1902235320 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5593623592c0 con 0x559362407830 2024-05-20T20:11:34.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.411+0000 7f4680c00640 10 client.26690 handle_command_reply: tid=0 2024-05-20T20:11:34.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.411+0000 7f46a0a07740 1 client.26690 shutdown 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.414+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 msgr2=0x5593623fc590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fc590 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f46900a58a0 tx=0x7f4690005b00 comp rx=0 tx=0).stop 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 shutdown_connections 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x559362407830 0x5593624749a0 crc :-1 s=READY pgs=146 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5593623f7e70 0x5593623fc590 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5593623ed5b0 0x5593623fc050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 --2- 172.21.15.28:0/1902235320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5593623ecc90 0x5593623fbb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 >> 172.21.15.28:0/1902235320 conn(0x5593623e91a0 msgr2=0x559362403380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:34.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f4680c00640 0 client.26690 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:34.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 shutdown_connections 2024-05-20T20:11:34.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.415+0000 7f46a0a07740 1 -- 172.21.15.28:0/1902235320 wait complete. 2024-05-20T20:11:34.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.416+0000 7f46a0a07740 20 client.26690 trim_cache size 0 max 16384 2024-05-20T20:11:34.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:11:34.418 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:11:34.418 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:11:34.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f467c0b2a90 msgr2=0x7f467c0b4f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f467c0b2a90 0x7f467c0b4f50 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f4684005e10 tx=0x7f4684005da0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 msgr2=0x7f469819f5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:34.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f469819f5b0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f469000db30 tx=0x7f469000b260 comp rx=0 tx=0).stop 2024-05-20T20:11:34.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 shutdown_connections 2024-05-20T20:11:34.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4698102340 0x7f46981a5940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f467c0b2a90 0x7f467c0b4f50 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4698107940 0x7f469819f5b0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 --2- 172.21.15.28:0/530091508 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4698101980 0x7f46981a5400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:34.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 >> 172.21.15.28:0/530091508 conn(0x7f4698078ad0 msgr2=0x7f4698104500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:34.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.418+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 shutdown_connections 2024-05-20T20:11:34.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:34.419+0000 7f469dc00640 1 -- 172.21.15.28:0/530091508 wait complete. 2024-05-20T20:11:34.869 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:35.263 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:34 smithi028 ceph-mon[31178]: pgmap v181: 97 pgs: 97 active+clean; 32 GiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 275 MiB/s wr, 2.10k op/s 2024-05-20T20:11:35.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:34 smithi154 ceph-mon[36314]: pgmap v181: 97 pgs: 97 active+clean; 32 GiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 275 MiB/s wr, 2.10k op/s 2024-05-20T20:11:35.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:34 smithi033 ceph-mon[36179]: pgmap v181: 97 pgs: 97 active+clean; 32 GiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 275 MiB/s wr, 2.10k op/s 2024-05-20T20:11:37.199 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:36 smithi028 ceph-mon[31178]: pgmap v182: 97 pgs: 97 active+clean; 32 GiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 248 MiB/s wr, 1.73k op/s 2024-05-20T20:11:37.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:36 smithi154 ceph-mon[36314]: pgmap v182: 97 pgs: 97 active+clean; 32 GiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 248 MiB/s wr, 1.73k op/s 2024-05-20T20:11:37.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:36 smithi033 ceph-mon[36179]: pgmap v182: 97 pgs: 97 active+clean; 32 GiB data, 6.9 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 248 MiB/s wr, 1.73k op/s 2024-05-20T20:11:38.853 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 20.8 2024-05-20T20:11:38.853 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:11:39.234 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:38 smithi028 ceph-mon[31178]: pgmap v183: 97 pgs: 97 active+clean; 32 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 258 MiB/s wr, 1.73k op/s 2024-05-20T20:11:39.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:38 smithi154 ceph-mon[36314]: pgmap v183: 97 pgs: 97 active+clean; 32 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 258 MiB/s wr, 1.73k op/s 2024-05-20T20:11:39.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:38 smithi033 ceph-mon[36179]: pgmap v183: 97 pgs: 97 active+clean; 32 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 258 MiB/s wr, 1.73k op/s 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.825+0000 7f8fc5800700 1 -- 172.21.15.28:0/405234176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 msgr2=0x7f8fc0112650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.825+0000 7f8fc5800700 1 --2- 172.21.15.28:0/405234176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 0x7f8fc0112650 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f8fb800b0a0 tx=0x7f8fb800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.826+0000 7f8fc5800700 1 -- 172.21.15.28:0/405234176 shutdown_connections 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.826+0000 7f8fc5800700 1 --2- 172.21.15.28:0/405234176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 0x7f8fc0112650 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.826+0000 7f8fc5800700 1 --2- 172.21.15.28:0/405234176 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fc010b780 0x7f8fc010bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.826+0000 7f8fc5800700 1 --2- 172.21.15.28:0/405234176 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 0x7f8fc0105c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.826+0000 7f8fc5800700 1 -- 172.21.15.28:0/405234176 >> 172.21.15.28:0/405234176 conn(0x7f8fc0101270 msgr2=0x7f8fc0103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.826+0000 7f8fc5800700 1 -- 172.21.15.28:0/405234176 shutdown_connections 2024-05-20T20:11:39.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.826+0000 7f8fc5800700 1 -- 172.21.15.28:0/405234176 wait complete. 2024-05-20T20:11:39.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.827+0000 7f8fc5800700 1 Processor -- start 2024-05-20T20:11:39.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.828+0000 7f8fc5800700 1 -- start start 2024-05-20T20:11:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fc5800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 0x7f8fc019bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fc5800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 0x7f8fc019c320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fc5800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fc010b780 0x7f8fc01a06b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fc5800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc01159d0 con 0x7f8fc010b780 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fc5800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc0115850 con 0x7f8fc0106140 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fc5800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc0115b50 con 0x7f8fc0105780 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 0x7f8fc019bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 0x7f8fc019bde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56616/0 (socket says 172.21.15.28:56616) 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fbea00700 1 -- 172.21.15.28:0/1975147079 learned_addr learned my addr 172.21.15.28:0/1975147079 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fbea00700 1 -- 172.21.15.28:0/1975147079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 msgr2=0x7f8fc019c320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.829+0000 7f8fbe000700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 0x7f8fc019c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.830+0000 7f8fbf400700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fc010b780 0x7f8fc01a06b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.830+0000 7f8fbea00700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 0x7f8fc019c320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.830+0000 7f8fbea00700 1 -- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fc010b780 msgr2=0x7f8fc01a06b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.830+0000 7f8fbea00700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fc010b780 0x7f8fc01a06b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.830+0000 7f8fbea00700 1 -- 172.21.15.28:0/1975147079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8fc01a0df0 con 0x7f8fc0105780 2024-05-20T20:11:39.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.830+0000 7f8fbe000700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 0x7f8fc019c320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:39.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.832+0000 7f8fbea00700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 0x7f8fc019bde0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f8fb000e9a0 tx=0x7f8fb000ed60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:39.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.832+0000 7f8fb7400700 1 -- 172.21.15.28:0/1975147079 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fb000cbd0 con 0x7f8fc0105780 2024-05-20T20:11:39.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.832+0000 7f8fb7400700 1 -- 172.21.15.28:0/1975147079 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fb000cd70 con 0x7f8fc0105780 2024-05-20T20:11:39.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.832+0000 7f8fb7400700 1 -- 172.21.15.28:0/1975147079 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fb0018680 con 0x7f8fc0105780 2024-05-20T20:11:39.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.833+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8fc01a10e0 con 0x7f8fc0105780 2024-05-20T20:11:39.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.833+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8fc01a8960 con 0x7f8fc0105780 2024-05-20T20:11:39.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.840+0000 7f8fd01ebb80 1 Processor -- start 2024-05-20T20:11:39.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fd01ebb80 1 -- start start 2024-05-20T20:11:39.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fb7400700 1 -- 172.21.15.28:0/1975147079 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f8fb0010970 con 0x7f8fc0105780 2024-05-20T20:11:39.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe21290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe217d0 0x55cbdfe1b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1bce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cbdfd79e40 con 0x55cbdfe1b880 2024-05-20T20:11:39.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cbdfd699e0 con 0x55cbdfe20eb0 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.841+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cbdfd6da20 con 0x55cbdfe217d0 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbf400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe21290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe217d0 0x55cbdfe1b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe217d0 0x55cbdfe1b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56622/0 (socket says 172.21.15.28:56622) 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 -- 172.21.15.28:0/864514613 learned_addr learned my addr 172.21.15.28:0/864514613 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbe000700 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1bce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 -- 172.21.15.28:0/864514613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe20eb0 msgr2=0x55cbdfe21290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe21290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 -- 172.21.15.28:0/864514613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 msgr2=0x55cbdfe1bce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1bce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.842+0000 7f8fbea00700 1 -- 172.21.15.28:0/864514613 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cbdfd32420 con 0x55cbdfe217d0 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.843+0000 7f8fbea00700 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe217d0 0x55cbdfe1b260 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8fb00dfef0 tx=0x7f8fb00e8ef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=49e1b287d99a85b2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:39.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.843+0000 7f8fabe00700 1 -- 172.21.15.28:0/864514613 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fb00f1070 con 0x55cbdfe217d0 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.843+0000 7f8fabe00700 1 -- 172.21.15.28:0/864514613 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fb00eacd0 con 0x55cbdfe217d0 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.843+0000 7f8fabe00700 1 -- 172.21.15.28:0/864514613 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fb00ec690 con 0x55cbdfe217d0 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.844+0000 7f8fb7400700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fa40b2b40 0x7f8fa40b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.844+0000 7f8fb7400700 1 -- 172.21.15.28:0/1975147079 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8fb00e2820 con 0x7f8fc0105780 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.844+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/864514613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe217d0 msgr2=0x55cbdfe1b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.844+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe217d0 0x55cbdfe1b260 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8fb00dfef0 tx=0x7f8fb00e8ef0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.845+0000 7f8fbf400700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fa40b2b40 0x7f8fa40b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.846+0000 7f8fbf400700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fa40b2b40 0x7f8fa40b5000 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f8fb8009850 tx=0x7f8fb8009780 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:39.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.850+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/864514613 shutdown_connections 2024-05-20T20:11:39.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.850+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1bce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.850+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe21290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.850+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/864514613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe217d0 0x55cbdfe1b260 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.850+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/864514613 >> 172.21.15.28:0/864514613 conn(0x55cbdfd49be0 msgr2=0x55cbdfe16010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:39.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.852+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/864514613 shutdown_connections 2024-05-20T20:11:39.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.852+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/864514613 wait complete. 2024-05-20T20:11:39.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.853+0000 7f8fd01ebb80 1 -- start start 2024-05-20T20:11:39.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.853+0000 7f8fd01ebb80 1 Processor -- start 2024-05-20T20:11:39.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.858+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe27440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.858+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe247e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.858+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.858+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cbdfc6ffa0 con 0x55cbdfe1b880 2024-05-20T20:11:39.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.858+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cbdfd0a760 con 0x55cbdfe217d0 2024-05-20T20:11:39.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.859+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cbdfe28240 con 0x55cbdfe20eb0 2024-05-20T20:11:39.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.859+0000 7f8fd01ebb80 10 client.0 _ll_register_callbacks cb 0x55cbdfd13c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:39.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.860+0000 7f8fbe000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe247e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.860+0000 7f8fbe000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe247e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56632/0 (socket says 172.21.15.28:56632) 2024-05-20T20:11:39.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.860+0000 7f8fbe000700 1 -- 172.21.15.28:0/257064437 learned_addr learned my addr 172.21.15.28:0/257064437 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:39.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.860+0000 7f8fbf400700 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.860+0000 7f8fbea00700 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe27440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.866+0000 7f8fbe000700 1 -- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 msgr2=0x55cbdfe24d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.866+0000 7f8fbe000700 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.866+0000 7f8fbe000700 1 -- 172.21.15.28:0/257064437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 msgr2=0x55cbdfe27440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.866+0000 7f8fbe000700 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe27440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.866+0000 7f8fbe000700 1 -- 172.21.15.28:0/257064437 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cbdfcb98d0 con 0x55cbdfe20eb0 2024-05-20T20:11:39.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.872+0000 7f8fbe000700 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe247e0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f8fac013780 tx=0x7f8fac013a90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bc361f8919cb487f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:39.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.872+0000 7f8fa8c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:39.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.873+0000 7f8fa8c00700 1 -- 172.21.15.28:0/257064437 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fac004020 con 0x55cbdfe20eb0 2024-05-20T20:11:39.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.873+0000 7f8fa8c00700 1 -- 172.21.15.28:0/257064437 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8fac0041c0 con 0x55cbdfe20eb0 2024-05-20T20:11:39.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.873+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8fb00eacd0 con 0x55cbdfe20eb0 2024-05-20T20:11:39.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.874+0000 7f8fa8c00700 1 -- 172.21.15.28:0/257064437 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8fac0043c0 con 0x55cbdfe20eb0 2024-05-20T20:11:39.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.875+0000 7f8fd01ebb80 10 client.26623 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:39.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.875+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cbdfd3a060 con 0x55cbdfe20eb0 2024-05-20T20:11:39.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.875+0000 7f8fa8c00700 1 -- 172.21.15.28:0/257064437 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fb00eacd0 con 0x55cbdfe20eb0 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.876+0000 7f8fa8c00700 1 -- 172.21.15.28:0/257064437 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8fac00f040 con 0x55cbdfe20eb0 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.876+0000 7f8fd01ebb80 10 client.26623 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.876+0000 7f8fd01ebb80 10 client.26623 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.876+0000 7f8fd01ebb80 20 client.26623 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.876+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfe45d90 0x55cbdfe481b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.876+0000 7f8fd01ebb80 4 client.26623 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.876+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55cbdfd7bd90 con 0x55cbdfe45d90 2024-05-20T20:11:39.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.877+0000 7f8fbea00700 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfe45d90 0x55cbdfe481b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.880+0000 7f8fbea00700 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfe45d90 0x55cbdfe481b0 crc :-1 s=READY pgs=147 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=41c653b8002ca541 server_cookie=76e9bbaffa9d1318 in_seq=0 out_seq=0 2024-05-20T20:11:39.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.880+0000 7f8fa8c00700 10 client.26623 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:39.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.881+0000 7f8fa8c00700 1 -- 172.21.15.28:0/257064437 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55cbdfd7bd90 con 0x55cbdfe45d90 2024-05-20T20:11:39.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.881+0000 7f8fa8c00700 10 client.26623 handle_command_reply: tid=0 2024-05-20T20:11:39.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.881+0000 7f8fd01ebb80 1 client.26623 shutdown 2024-05-20T20:11:39.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 msgr2=0x55cbdfe247e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe247e0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f8fac013780 tx=0x7f8fac013a90 comp rx=0 tx=0).stop 2024-05-20T20:11:39.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 shutdown_connections 2024-05-20T20:11:39.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfe45d90 0x55cbdfe481b0 crc :-1 s=READY pgs=147 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe27440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fa8c00700 0 client.26623 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:39.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/257064437 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe247e0 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 >> 172.21.15.28:0/257064437 conn(0x55cbdfd49be0 msgr2=0x55cbdfe1a250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:39.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.882+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 shutdown_connections 2024-05-20T20:11:39.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.883+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/257064437 wait complete. 2024-05-20T20:11:39.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.883+0000 7f8fd01ebb80 20 client.26623 trim_cache size 0 max 16384 2024-05-20T20:11:39.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.926+0000 7f8fd01ebb80 1 Processor -- start 2024-05-20T20:11:39.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.935+0000 7f8fd01ebb80 1 -- start start 2024-05-20T20:11:39.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.935+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.935+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe245c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.936+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.936+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc00745b0 con 0x55cbdfe217d0 2024-05-20T20:11:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.936+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc01a9510 con 0x55cbdfe1b880 2024-05-20T20:11:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.936+0000 7f8fd01ebb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc0074c80 con 0x55cbdfe20eb0 2024-05-20T20:11:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.936+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.936+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51020/0 (socket says 172.21.15.28:51020) 2024-05-20T20:11:39.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.936+0000 7f8fbea00700 1 -- 172.21.15.28:0/3175405259 learned_addr learned my addr 172.21.15.28:0/3175405259 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:39.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.937+0000 7f8fbe000700 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe245c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.937+0000 7f8fbf400700 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.938+0000 7f8fbea00700 1 -- 172.21.15.28:0/3175405259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 msgr2=0x55cbdfe245c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.938+0000 7f8fbea00700 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe245c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.938+0000 7f8fbea00700 1 -- 172.21.15.28:0/3175405259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe217d0 msgr2=0x55cbdfe24b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.938+0000 7f8fbea00700 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.938+0000 7f8fbea00700 1 -- 172.21.15.28:0/3175405259 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cbdfe25740 con 0x55cbdfe1b880 2024-05-20T20:11:39.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.938+0000 7f8fbf400700 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:11:39.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.939+0000 7f8fbea00700 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1b260 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f8fb00edfd0 tx=0x7f8fb00e8550 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9c18ec0de005273 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:39.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.939+0000 7f8fab400700 1 -- 172.21.15.28:0/3175405259 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fb00f1040 con 0x55cbdfe1b880 2024-05-20T20:11:39.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.939+0000 7f8fab400700 1 -- 172.21.15.28:0/3175405259 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fb00b22b0 con 0x55cbdfe1b880 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.939+0000 7f8fab400700 1 -- 172.21.15.28:0/3175405259 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fb00a9620 con 0x55cbdfe1b880 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.940+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/3175405259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe1b880 msgr2=0x55cbdfe1b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.940+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1b260 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f8fb00edfd0 tx=0x7f8fb00e8550 comp rx=0 tx=0).stop 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.941+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/3175405259 shutdown_connections 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.941+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe217d0 0x55cbdfe24b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.941+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe1b880 0x55cbdfe1b260 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.941+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/3175405259 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe245c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.941+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/3175405259 >> 172.21.15.28:0/3175405259 conn(0x55cbdfd49be0 msgr2=0x55cbdfe1bc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.941+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/3175405259 shutdown_connections 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.941+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/3175405259 wait complete. 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.942+0000 7f8fd01ebb80 1 -- start start 2024-05-20T20:11:39.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.942+0000 7f8fd01ebb80 1 Processor -- start 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.943+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe2d860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.943+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe2a4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.943+0000 7f8fd01ebb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfeb0980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.944+0000 7f8fbe000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe2a4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.944+0000 7f8fbe000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe2a4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56644/0 (socket says 172.21.15.28:56644) 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.944+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe2d860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.944+0000 7f8fbf400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfeb0980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.944+0000 7f8fbf400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfeb0980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51024/0 (socket says 172.21.15.28:51024) 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.944+0000 7f8fbea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe2d860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:39590/0 (socket says 172.21.15.28:39590) 2024-05-20T20:11:39.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.944+0000 7f8fbe000700 1 -- 172.21.15.28:0/1582293406 learned_addr learned my addr 172.21.15.28:0/1582293406 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.945+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc01a8e40 con 0x55cbdfe1b880 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.945+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc0075350 con 0x55cbdfe217d0 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.945+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8fc0072510 con 0x55cbdfe20eb0 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.945+0000 7f8fd01ebb80 10 client.0 _ll_register_callbacks cb 0x55cbdfd13c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.946+0000 7f8fbf400700 1 -- 172.21.15.28:0/1582293406 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 msgr2=0x55cbdfe2a4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.946+0000 7f8fbf400700 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe2a4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.946+0000 7f8fbf400700 1 -- 172.21.15.28:0/1582293406 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 msgr2=0x55cbdfe2d860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.946+0000 7f8fbf400700 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe2d860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.946+0000 7f8fbf400700 1 -- 172.21.15.28:0/1582293406 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cbdfe2df80 con 0x55cbdfe217d0 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.947+0000 7f8fa8c00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.947+0000 7f8fbf400700 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfeb0980 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7f8fb800b780 tx=0x7f8fb80099e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c45ef4d95ac12ba4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:39.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.947+0000 7f8fa8c00700 1 -- 172.21.15.28:0/1582293406 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fb8021070 con 0x55cbdfe217d0 2024-05-20T20:11:39.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.947+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55cbdfd75570 con 0x55cbdfe217d0 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.947+0000 7f8fa8c00700 1 -- 172.21.15.28:0/1582293406 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f8fb802d070 con 0x55cbdfe217d0 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.948+0000 7f8fa8c00700 1 -- 172.21.15.28:0/1582293406 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8fb8021070 con 0x55cbdfe217d0 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.948+0000 7f8fd01ebb80 10 client.26705 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.948+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cbdfec25f0 con 0x55cbdfe217d0 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.949+0000 7f8fa8c00700 1 -- 172.21.15.28:0/1582293406 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55cbdfd75570 con 0x55cbdfe217d0 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.950+0000 7f8fa8c00700 1 -- 172.21.15.28:0/1582293406 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8fb8011070 con 0x55cbdfe217d0 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.950+0000 7f8fd01ebb80 10 client.26705 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.950+0000 7f8fd01ebb80 10 client.26705 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.950+0000 7f8fd01ebb80 20 client.26705 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.950+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfec2c30 0x55cbdfec5050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.951+0000 7f8fd01ebb80 4 client.26705 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.951+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55cbdfbec250 con 0x55cbdfec2c30 2024-05-20T20:11:39.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.951+0000 7f8fbea00700 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfec2c30 0x55cbdfec5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:39.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.952+0000 7f8fbea00700 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfec2c30 0x55cbdfec5050 crc :-1 s=READY pgs=148 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4f89f18a947cd55 server_cookie=caf77d27509a903 in_seq=0 out_seq=0 2024-05-20T20:11:39.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.952+0000 7f8fa8c00700 10 client.26705 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:39.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.952+0000 7f8fa8c00700 1 -- 172.21.15.28:0/1582293406 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x55cbdfbec250 con 0x55cbdfec2c30 2024-05-20T20:11:39.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.952+0000 7f8fa8c00700 10 client.26705 handle_command_reply: tid=0 2024-05-20T20:11:39.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.952+0000 7f8fd01ebb80 1 client.26705 shutdown 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.952+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 msgr2=0x55cbdfeb0980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfeb0980 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7f8fb800b780 tx=0x7f8fb80099e0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 shutdown_connections 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cbdfec2c30 0x55cbdfec5050 crc :-1 s=READY pgs=148 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cbdfe217d0 0x55cbdfeb0980 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cbdfe1b880 0x55cbdfe2d860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fd01ebb80 1 --2- 172.21.15.28:0/1582293406 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cbdfe20eb0 0x55cbdfe2a4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 >> 172.21.15.28:0/1582293406 conn(0x55cbdfd49be0 msgr2=0x55cbdfe245c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.953+0000 7f8fa8c00700 0 client.26705 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:39.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.954+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 shutdown_connections 2024-05-20T20:11:39.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.954+0000 7f8fd01ebb80 1 -- 172.21.15.28:0/1582293406 wait complete. 2024-05-20T20:11:39.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.954+0000 7f8fd01ebb80 20 client.26705 trim_cache size 0 max 16384 2024-05-20T20:11:39.966 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:11:39.966 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:11:39.966 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:11:39.966 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fa40b2b40 msgr2=0x7f8fa40b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fa40b2b40 0x7f8fa40b5000 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f8fb8009850 tx=0x7f8fb8009780 comp rx=0 tx=0).stop 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 msgr2=0x7f8fc019bde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 0x7f8fc019bde0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f8fb000e9a0 tx=0x7f8fb000ed60 comp rx=0 tx=0).stop 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 shutdown_connections 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fc010b780 0x7f8fc01a06b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fc0105780 0x7f8fc019bde0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.960+0000 7f8fc5800700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fa40b2b40 0x7f8fa40b5000 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.961+0000 7f8fc5800700 1 --2- 172.21.15.28:0/1975147079 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fc0106140 0x7f8fc019c320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.961+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 >> 172.21.15.28:0/1975147079 conn(0x7f8fc0101270 msgr2=0x7f8fc01017f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.961+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 shutdown_connections 2024-05-20T20:11:39.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:39.962+0000 7f8fc5800700 1 -- 172.21.15.28:0/1975147079 wait complete. 2024-05-20T20:11:41.303 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:40 smithi028 ceph-mon[31178]: pgmap v184: 97 pgs: 97 active+clean; 33 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 318 MiB/s wr, 1.92k op/s 2024-05-20T20:11:41.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:40 smithi154 ceph-mon[36314]: pgmap v184: 97 pgs: 97 active+clean; 33 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 318 MiB/s wr, 1.92k op/s 2024-05-20T20:11:41.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:40 smithi033 ceph-mon[36179]: pgmap v184: 97 pgs: 97 active+clean; 33 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 318 MiB/s wr, 1.92k op/s 2024-05-20T20:11:41.693 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:11:41.694 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:11:41.739 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:43.278 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:42 smithi028 ceph-mon[31178]: pgmap v185: 97 pgs: 97 active+clean; 33 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 281 MiB/s wr, 1.46k op/s 2024-05-20T20:11:43.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:42 smithi154 ceph-mon[36314]: pgmap v185: 97 pgs: 97 active+clean; 33 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 281 MiB/s wr, 1.46k op/s 2024-05-20T20:11:43.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:42 smithi033 ceph-mon[36179]: pgmap v185: 97 pgs: 97 active+clean; 33 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 281 MiB/s wr, 1.46k op/s 2024-05-20T20:11:44.110 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:45.251 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:44 smithi028 ceph-mon[31178]: pgmap v186: 97 pgs: 97 active+clean; 34 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 312 MiB/s wr, 1.27k op/s 2024-05-20T20:11:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:44 smithi154 ceph-mon[36314]: pgmap v186: 97 pgs: 97 active+clean; 34 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 312 MiB/s wr, 1.27k op/s 2024-05-20T20:11:45.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:44 smithi033 ceph-mon[36179]: pgmap v186: 97 pgs: 97 active+clean; 34 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 312 MiB/s wr, 1.27k op/s 2024-05-20T20:11:47.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:47 smithi028 ceph-mon[31178]: pgmap v187: 97 pgs: 97 active+clean; 34 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 263 MiB/s wr, 895 op/s 2024-05-20T20:11:47.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:47 smithi154 ceph-mon[36314]: pgmap v187: 97 pgs: 97 active+clean; 34 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 263 MiB/s wr, 895 op/s 2024-05-20T20:11:47.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:47 smithi033 ceph-mon[36179]: pgmap v187: 97 pgs: 97 active+clean; 34 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 263 MiB/s wr, 895 op/s 2024-05-20T20:11:47.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.666+0000 7fc892a00640 1 -- 172.21.15.28:0/1371085726 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 msgr2=0x7fc88c117f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.666+0000 7fc892a00640 1 --2- 172.21.15.28:0/1371085726 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c117f90 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fc884009ca0 tx=0x7fc88402a970 comp rx=0 tx=0).stop 2024-05-20T20:11:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.666+0000 7fc892a00640 1 -- 172.21.15.28:0/1371085726 shutdown_connections 2024-05-20T20:11:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.666+0000 7fc892a00640 1 --2- 172.21.15.28:0/1371085726 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc88c10a370 0x7fc88c115660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.666+0000 7fc892a00640 1 --2- 172.21.15.28:0/1371085726 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c117f90 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.667+0000 7fc892a00640 1 --2- 172.21.15.28:0/1371085726 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc88c109a50 0x7fc88c109e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.667+0000 7fc892a00640 1 -- 172.21.15.28:0/1371085726 >> 172.21.15.28:0/1371085726 conn(0x7fc88c06db20 msgr2=0x7fc88c06df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.667+0000 7fc892a00640 1 -- 172.21.15.28:0/1371085726 shutdown_connections 2024-05-20T20:11:47.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.668+0000 7fc892a00640 1 -- 172.21.15.28:0/1371085726 wait complete. 2024-05-20T20:11:47.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.672+0000 7fc892a00640 1 Processor -- start 2024-05-20T20:11:47.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.673+0000 7fc892a00640 1 -- start start 2024-05-20T20:11:47.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.673+0000 7fc892a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc88c109a50 0x7fc88c11e800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.673+0000 7fc892a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc88c10a370 0x7fc88c11ed40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.673+0000 7fc892a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c11b090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.673+0000 7fc892a00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c10ba00 con 0x7fc88c115ba0 2024-05-20T20:11:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.673+0000 7fc892a00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c10b880 con 0x7fc88c10a370 2024-05-20T20:11:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.673+0000 7fc892a00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c10bb80 con 0x7fc88c109a50 2024-05-20T20:11:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.674+0000 7fc890a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c11b090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.674+0000 7fc890a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c11b090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52566/0 (socket says 172.21.15.28:52566) 2024-05-20T20:11:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.674+0000 7fc890a00640 1 -- 172.21.15.28:0/515461376 learned_addr learned my addr 172.21.15.28:0/515461376 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:47.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.678+0000 7fc890a00640 1 -- 172.21.15.28:0/515461376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc88c109a50 msgr2=0x7fc88c11e800 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:11:47.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.678+0000 7fc890a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc88c109a50 0x7fc88c11e800 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.678+0000 7fc890a00640 1 -- 172.21.15.28:0/515461376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc88c10a370 msgr2=0x7fc88c11ed40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.678+0000 7fc890a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc88c10a370 0x7fc88c11ed40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.678+0000 7fc890a00640 1 -- 172.21.15.28:0/515461376 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc88c11b950 con 0x7fc88c115ba0 2024-05-20T20:11:47.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.679+0000 7fc890a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c11b090 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fc884005c00 tx=0x7fc884003ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:47.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.680+0000 7fc889600640 1 -- 172.21.15.28:0/515461376 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc884048070 con 0x7fc88c115ba0 2024-05-20T20:11:47.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.680+0000 7fc889600640 1 -- 172.21.15.28:0/515461376 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc88402f870 con 0x7fc88c115ba0 2024-05-20T20:11:47.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.680+0000 7fc889600640 1 -- 172.21.15.28:0/515461376 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc884048070 con 0x7fc88c115ba0 2024-05-20T20:11:47.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.682+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc88c11bbe0 con 0x7fc88c115ba0 2024-05-20T20:11:47.683 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.682+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc88c1a4220 con 0x7fc88c115ba0 2024-05-20T20:11:47.691 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.691+0000 7fc895834740 1 Processor -- start 2024-05-20T20:11:47.692 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.692+0000 7fc889600640 1 -- 172.21.15.28:0/515461376 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc88404b070 con 0x7fc88c115ba0 2024-05-20T20:11:47.698 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.697+0000 7fc895834740 1 -- start start 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.697+0000 7fc895834740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7402070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.697+0000 7fc895834740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa740c930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.697+0000 7fc895834740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740f260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.697+0000 7fc895834740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556fa7372b90 con 0x556fa740ce70 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.697+0000 7fc895834740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556fa7362730 con 0x556fa7401c90 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.697+0000 7fc895834740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556fa7366770 con 0x556fa74025b0 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7402070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7402070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38102/0 (socket says 172.21.15.28:38102) 2024-05-20T20:11:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 -- 172.21.15.28:0/1356041575 learned_addr learned my addr 172.21.15.28:0/1356041575 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:47.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 -- 172.21.15.28:0/1356041575 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 msgr2=0x556fa740c930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:47.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 --2- 172.21.15.28:0/1356041575 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa740c930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 -- 172.21.15.28:0/1356041575 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 msgr2=0x556fa740f260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:47.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 --2- 172.21.15.28:0/1356041575 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740f260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.698+0000 7fc883400640 1 -- 172.21.15.28:0/1356041575 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556fa72fb8b0 con 0x556fa7401c90 2024-05-20T20:11:47.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.700+0000 7fc883400640 1 --2- 172.21.15.28:0/1356041575 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7402070 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fc87c005bc0 tx=0x7fc87c00f360 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c2f2894af6606923 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:47.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.700+0000 7fc889600640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8700b2c80 0x7fc8700b5140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.701+0000 7fc881200640 1 -- 172.21.15.28:0/1356041575 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc87c00fd90 con 0x556fa7401c90 2024-05-20T20:11:47.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.701+0000 7fc881200640 1 -- 172.21.15.28:0/1356041575 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc87c0054d0 con 0x556fa7401c90 2024-05-20T20:11:47.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.701+0000 7fc881200640 1 -- 172.21.15.28:0/1356041575 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc87c0116a0 con 0x556fa7401c90 2024-05-20T20:11:47.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.702+0000 7fc889600640 1 -- 172.21.15.28:0/515461376 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fc884103210 con 0x7fc88c115ba0 2024-05-20T20:11:47.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.703+0000 7fc88be00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8700b2c80 0x7fc8700b5140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.705 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 -- 172.21.15.28:0/1356041575 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 msgr2=0x556fa7402070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc88be00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8700b2c80 0x7fc8700b5140 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x556fa7403230 tx=0x7fc874006bd0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:47.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 --2- 172.21.15.28:0/1356041575 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7402070 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fc87c005bc0 tx=0x7fc87c00f360 comp rx=0 tx=0).stop 2024-05-20T20:11:47.706 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 -- 172.21.15.28:0/1356041575 shutdown_connections 2024-05-20T20:11:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 --2- 172.21.15.28:0/1356041575 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740f260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 --2- 172.21.15.28:0/1356041575 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa740c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 --2- 172.21.15.28:0/1356041575 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7402070 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 -- 172.21.15.28:0/1356041575 >> 172.21.15.28:0/1356041575 conn(0x556fa73fe1a0 msgr2=0x556fa73fe580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 -- 172.21.15.28:0/1356041575 shutdown_connections 2024-05-20T20:11:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.705+0000 7fc895834740 1 -- 172.21.15.28:0/1356041575 wait complete. 2024-05-20T20:11:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.706+0000 7fc895834740 1 -- start start 2024-05-20T20:11:47.708 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.707+0000 7fc895834740 1 Processor -- start 2024-05-20T20:11:47.708 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.708+0000 7fc895834740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa740c860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.708 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.708+0000 7fc895834740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa7409b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.708 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.708+0000 7fc895834740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740a160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.708+0000 7fc88be00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa740c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.708+0000 7fc88be00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa740c860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38112/0 (socket says 172.21.15.28:38112) 2024-05-20T20:11:47.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.708+0000 7fc883400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa7409b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.708+0000 7fc883400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa7409b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:42816/0 (socket says 172.21.15.28:42816) 2024-05-20T20:11:47.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.709+0000 7fc890a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740a160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.709+0000 7fc890a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740a160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52584/0 (socket says 172.21.15.28:52584) 2024-05-20T20:11:47.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.709+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556fa7048790 con 0x556fa740ce70 2024-05-20T20:11:47.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.709+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556fa730abc0 con 0x556fa7401c90 2024-05-20T20:11:47.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.709+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556fa740faa0 con 0x556fa74025b0 2024-05-20T20:11:47.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.709+0000 7fc895834740 10 client.0 _ll_register_callbacks cb 0x556fa72e8400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:47.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.709+0000 7fc88be00640 1 -- 172.21.15.28:0/3871532458 learned_addr learned my addr 172.21.15.28:0/3871532458 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:47.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.710+0000 7fc883400640 1 -- 172.21.15.28:0/3871532458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 msgr2=0x556fa740c860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:47.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.710+0000 7fc883400640 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa740c860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.710+0000 7fc883400640 1 -- 172.21.15.28:0/3871532458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 msgr2=0x556fa740a160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:47.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.710+0000 7fc883400640 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740a160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.710+0000 7fc883400640 1 -- 172.21.15.28:0/3871532458 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556fa7077210 con 0x556fa74025b0 2024-05-20T20:11:47.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.711+0000 7fc883400640 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa7409b50 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fc87c005b90 tx=0x7fc87c0049d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=78f8e51a55308d7e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:47.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.712+0000 7fc879600640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.712+0000 7fc879600640 1 -- 172.21.15.28:0/3871532458 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc87c00fda0 con 0x556fa74025b0 2024-05-20T20:11:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.712+0000 7fc879600640 1 -- 172.21.15.28:0/3871532458 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fc87c005020 con 0x556fa74025b0 2024-05-20T20:11:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.712+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc87c0054d0 con 0x556fa74025b0 2024-05-20T20:11:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.713+0000 7fc879600640 1 -- 172.21.15.28:0/3871532458 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc87c029050 con 0x556fa74025b0 2024-05-20T20:11:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.713+0000 7fc895834740 10 client.26632 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.713+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556fa7143620 con 0x556fa74025b0 2024-05-20T20:11:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.714+0000 7fc879600640 1 -- 172.21.15.28:0/3871532458 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc87c0054d0 con 0x556fa74025b0 2024-05-20T20:11:47.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.714+0000 7fc879600640 1 -- 172.21.15.28:0/3871532458 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc87c003850 con 0x556fa74025b0 2024-05-20T20:11:47.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.714+0000 7fc895834740 10 client.26632 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:47.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.714+0000 7fc895834740 10 client.26632 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:47.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.715+0000 7fc895834740 20 client.26632 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:47.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.715+0000 7fc895834740 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa741c810 0x556fa741ec00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.715+0000 7fc895834740 4 client.26632 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:47.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.715+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556fa7374ae0 con 0x556fa741c810 2024-05-20T20:11:47.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.715+0000 7fc88be00640 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa741c810 0x556fa741ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.716+0000 7fc879600640 10 client.26632 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:47.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.716+0000 7fc88be00640 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa741c810 0x556fa741ec00 crc :-1 s=READY pgs=149 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f743da653262e19c server_cookie=76a923b45a2b71a3 in_seq=0 out_seq=0 2024-05-20T20:11:47.718 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.717+0000 7fc879600640 1 -- 172.21.15.28:0/3871532458 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x556fa7374ae0 con 0x556fa741c810 2024-05-20T20:11:47.718 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.717+0000 7fc879600640 10 client.26632 handle_command_reply: tid=0 2024-05-20T20:11:47.718 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.717+0000 7fc895834740 1 client.26632 shutdown 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.717+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 msgr2=0x556fa7409b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.717+0000 7fc895834740 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa7409b50 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fc87c005b90 tx=0x7fc87c0049d0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 shutdown_connections 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa741c810 0x556fa741ec00 crc :-1 s=READY pgs=149 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa740ce70 0x556fa740a160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa74025b0 0x556fa7409b50 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 --2- 172.21.15.28:0/3871532458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa740c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 >> 172.21.15.28:0/3871532458 conn(0x556fa73fe1a0 msgr2=0x556fa7403ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc879600640 0 client.26632 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:47.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 shutdown_connections 2024-05-20T20:11:47.720 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 1 -- 172.21.15.28:0/3871532458 wait complete. 2024-05-20T20:11:47.720 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.718+0000 7fc895834740 20 client.26632 trim_cache size 0 max 16384 2024-05-20T20:11:47.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.772+0000 7fc895834740 1 Processor -- start 2024-05-20T20:11:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.773+0000 7fc895834740 1 -- start start 2024-05-20T20:11:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.773+0000 7fc895834740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7418d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.773+0000 7fc895834740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa74025b0 0x556fa740ba90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.773+0000 7fc895834740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7421c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.773+0000 7fc895834740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c1c5c50 con 0x556fa74025b0 2024-05-20T20:11:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.773+0000 7fc895834740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c1c7790 con 0x556fa7401c90 2024-05-20T20:11:47.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.773+0000 7fc895834740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c1c6320 con 0x556fa740ce70 2024-05-20T20:11:47.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.774+0000 7fc883400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa74025b0 0x556fa740ba90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.774+0000 7fc88be00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7418d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.774+0000 7fc890a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7421c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.774+0000 7fc883400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa74025b0 0x556fa740ba90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52598/0 (socket says 172.21.15.28:52598) 2024-05-20T20:11:47.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.774+0000 7fc890a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7421c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:42832/0 (socket says 172.21.15.28:42832) 2024-05-20T20:11:47.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.774+0000 7fc883400640 1 -- 172.21.15.28:0/104803610 learned_addr learned my addr 172.21.15.28:0/104803610 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:47.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.775+0000 7fc883400640 1 -- 172.21.15.28:0/104803610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 msgr2=0x556fa7421c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:47.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.775+0000 7fc883400640 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7421c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.775+0000 7fc883400640 1 -- 172.21.15.28:0/104803610 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 msgr2=0x556fa7418d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:47.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.775+0000 7fc883400640 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7418d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.775+0000 7fc883400640 1 -- 172.21.15.28:0/104803610 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556fa740a4d0 con 0x556fa74025b0 2024-05-20T20:11:47.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.775+0000 7fc88be00640 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7418d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:47.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.775+0000 7fc890a00640 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7421c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:47.777 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.776+0000 7fc883400640 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa74025b0 0x556fa740ba90 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fc87c00d390 tx=0x7fc87c024450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dab89d2f31d02f13 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:47.777 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.776+0000 7fc87be00640 1 -- 172.21.15.28:0/104803610 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc87c033070 con 0x556fa74025b0 2024-05-20T20:11:47.777 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.776+0000 7fc87be00640 1 -- 172.21.15.28:0/104803610 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc87c01d070 con 0x556fa74025b0 2024-05-20T20:11:47.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.777+0000 7fc87be00640 1 -- 172.21.15.28:0/104803610 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc87c0044b0 con 0x556fa74025b0 2024-05-20T20:11:47.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.778+0000 7fc895834740 1 -- 172.21.15.28:0/104803610 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa74025b0 msgr2=0x556fa740ba90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.779 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.778+0000 7fc895834740 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa74025b0 0x556fa740ba90 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fc87c00d390 tx=0x7fc87c024450 comp rx=0 tx=0).stop 2024-05-20T20:11:47.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.779+0000 7fc895834740 1 -- 172.21.15.28:0/104803610 shutdown_connections 2024-05-20T20:11:47.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.779+0000 7fc895834740 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7421c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.780+0000 7fc895834740 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa74025b0 0x556fa740ba90 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.780+0000 7fc895834740 1 --2- 172.21.15.28:0/104803610 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa7401c90 0x556fa7418d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.780+0000 7fc895834740 1 -- 172.21.15.28:0/104803610 >> 172.21.15.28:0/104803610 conn(0x556fa73fe1a0 msgr2=0x556fa740b380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:47.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.780+0000 7fc895834740 1 -- 172.21.15.28:0/104803610 shutdown_connections 2024-05-20T20:11:47.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.780+0000 7fc895834740 1 -- 172.21.15.28:0/104803610 wait complete. 2024-05-20T20:11:47.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.781+0000 7fc895834740 1 -- start start 2024-05-20T20:11:47.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.782+0000 7fc895834740 1 Processor -- start 2024-05-20T20:11:47.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.782+0000 7fc895834740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa7401c90 0x556fa741f180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.783 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.782+0000 7fc895834740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa74025b0 0x556fa741f6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc883400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa74025b0 0x556fa741f6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc883400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa74025b0 0x556fa741f6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38144/0 (socket says 172.21.15.28:38144) 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc88be00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa7401c90 0x556fa741f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc883400640 1 -- 172.21.15.28:0/407150153 learned_addr learned my addr 172.21.15.28:0/407150153 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc895834740 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7485040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c1c70c0 con 0x556fa7401c90 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c1c69f0 con 0x556fa74025b0 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc88c1aabd0 con 0x556fa740ce70 2024-05-20T20:11:47.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc895834740 10 client.0 _ll_register_callbacks cb 0x7fc87c00d9d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.783+0000 7fc890a00640 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7485040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc883400640 1 -- 172.21.15.28:0/407150153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 msgr2=0x556fa7485040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc883400640 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7485040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc883400640 1 -- 172.21.15.28:0/407150153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa7401c90 msgr2=0x556fa741f180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc883400640 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa7401c90 0x556fa741f180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc883400640 1 -- 172.21.15.28:0/407150153 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556fa7485630 con 0x556fa74025b0 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc87c01d070 con 0x556fa74025b0 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc883400640 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa74025b0 0x556fa741f6c0 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x556fa7366840 tx=0x7fc87c004680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=33389ee5c7ae5853 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc879600640 10 client.26720 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc879600640 1 -- 172.21.15.28:0/407150153 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc87c033040 con 0x556fa74025b0 2024-05-20T20:11:47.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc879600640 1 -- 172.21.15.28:0/407150153 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fc87c0035f0 con 0x556fa74025b0 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc879600640 1 -- 172.21.15.28:0/407150153 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc87c003830 con 0x556fa74025b0 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc895834740 10 client.26720 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.784+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x556fa7486860 con 0x556fa74025b0 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc879600640 1 -- 172.21.15.28:0/407150153 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc87c036070 con 0x556fa74025b0 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc895834740 10 client.26720 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc895834740 10 client.26720 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc895834740 20 client.26720 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc895834740 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa7486e20 0x556fa7489240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc895834740 4 client.26720 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:11:47.787 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x556fa736e2c0 con 0x556fa7486e20 2024-05-20T20:11:47.788 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.785+0000 7fc88be00640 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa7486e20 0x556fa7489240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.788 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.786+0000 7fc88be00640 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa7486e20 0x556fa7489240 crc :-1 s=READY pgs=150 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b2cdf48193a9b688 server_cookie=bad85af0b9166983 in_seq=0 out_seq=0 2024-05-20T20:11:47.788 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.786+0000 7fc879600640 10 client.26720 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:47.810 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.810+0000 7fc879600640 1 -- 172.21.15.28:0/407150153 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x556fa736e2c0 con 0x556fa7486e20 2024-05-20T20:11:47.811 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.810+0000 7fc879600640 10 client.26720 handle_command_reply: tid=0 2024-05-20T20:11:47.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.810+0000 7fc895834740 1 client.26720 shutdown 2024-05-20T20:11:47.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.811+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa74025b0 msgr2=0x556fa741f6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.811+0000 7fc895834740 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa74025b0 0x556fa741f6c0 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x556fa7366840 tx=0x7fc87c004680 comp rx=0 tx=0).stop 2024-05-20T20:11:47.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.811+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 shutdown_connections 2024-05-20T20:11:47.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.811+0000 7fc895834740 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556fa7486e20 0x556fa7489240 crc :-1 s=READY pgs=150 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.812+0000 7fc879600640 0 client.26720 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.812+0000 7fc895834740 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556fa74025b0 0x556fa741f6c0 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.812+0000 7fc895834740 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556fa740ce70 0x556fa7485040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.812+0000 7fc895834740 1 --2- 172.21.15.28:0/407150153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556fa7401c90 0x556fa741f180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.812+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 >> 172.21.15.28:0/407150153 conn(0x556fa73fe1a0 msgr2=0x556fa73fd310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.812+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 shutdown_connections 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.812+0000 7fc895834740 1 -- 172.21.15.28:0/407150153 wait complete. 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.813+0000 7fc895834740 20 client.26720 trim_cache size 0 max 16384 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:11:47.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:11:47.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:11:47.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.813+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8700b2c80 msgr2=0x7fc8700b5140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.813+0000 7fc892a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8700b2c80 0x7fc8700b5140 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x556fa7403230 tx=0x7fc874006bd0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.813+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 msgr2=0x7fc88c11b090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.813+0000 7fc892a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c11b090 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fc884005c00 tx=0x7fc884003ba0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 shutdown_connections 2024-05-20T20:11:47.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc88c10a370 0x7fc88c11ed40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc8700b2c80 0x7fc8700b5140 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc88c115ba0 0x7fc88c11b090 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 --2- 172.21.15.28:0/515461376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc88c109a50 0x7fc88c11e800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 >> 172.21.15.28:0/515461376 conn(0x7fc88c06db20 msgr2=0x7fc88c071690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:47.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 shutdown_connections 2024-05-20T20:11:47.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:11:47.814+0000 7fc892a00640 1 -- 172.21.15.28:0/515461376 wait complete. 2024-05-20T20:11:47.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.921+0000 7fdd4be00700 1 -- 172.21.15.28:0/3193871402 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441041a0 msgr2=0x7fdd441081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.921+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3193871402 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441041a0 0x7fdd441081c0 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7fdd300099c0 tx=0x7fdd30009cd0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.923+0000 7fdd4be00700 1 -- 172.21.15.28:0/3193871402 shutdown_connections 2024-05-20T20:11:47.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.923+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3193871402 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd441088f0 0x7fdd44110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.924+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3193871402 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd441037f0 0x7fdd44103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.924+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3193871402 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441041a0 0x7fdd441081c0 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.926+0000 7fdd4be00700 1 -- 172.21.15.28:0/3193871402 >> 172.21.15.28:0/3193871402 conn(0x7fdd4407a6a0 msgr2=0x7fdd4407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.926+0000 7fdd4be00700 1 -- 172.21.15.28:0/3193871402 shutdown_connections 2024-05-20T20:11:47.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.926+0000 7fdd4be00700 1 -- 172.21.15.28:0/3193871402 wait complete. 2024-05-20T20:11:47.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.929+0000 7fdd4be00700 1 Processor -- start 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.930+0000 7fdd4be00700 1 -- start start 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.931+0000 7fdd4be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441037f0 0x7fdd44110450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.931+0000 7fdd4be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd441041a0 0x7fdd4410d620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.931+0000 7fdd4be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd441088f0 0x7fdd4410dd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.931+0000 7fdd4be00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdd44077330 con 0x7fdd441088f0 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.931+0000 7fdd4be00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdd440771b0 con 0x7fdd441037f0 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.931+0000 7fdd4be00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdd440774b0 con 0x7fdd441041a0 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441037f0 0x7fdd44110450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441037f0 0x7fdd44110450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38166/0 (socket says 172.21.15.28:38166) 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 -- 172.21.15.28:0/3153526556 learned_addr learned my addr 172.21.15.28:0/3153526556 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 -- 172.21.15.28:0/3153526556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd441041a0 msgr2=0x7fdd4410d620 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd441041a0 0x7fdd4410d620 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 -- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd441088f0 msgr2=0x7fdd4410dd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:47.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49e00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd441088f0 0x7fdd4410dd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd441088f0 0x7fdd4410dd90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:47.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49400700 1 -- 172.21.15.28:0/3153526556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdd44073fb0 con 0x7fdd441037f0 2024-05-20T20:11:47.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.932+0000 7fdd49e00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd441088f0 0x7fdd4410dd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:47.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.933+0000 7fdd49400700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441037f0 0x7fdd44110450 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7fdd3800b530 tx=0x7fdd3800b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:47.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.933+0000 7fdd42000700 1 -- 172.21.15.28:0/3153526556 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd38015020 con 0x7fdd441037f0 2024-05-20T20:11:47.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.933+0000 7fdd42000700 1 -- 172.21.15.28:0/3153526556 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd38011ce0 con 0x7fdd441037f0 2024-05-20T20:11:47.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.933+0000 7fdd42000700 1 -- 172.21.15.28:0/3153526556 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd38010dd0 con 0x7fdd441037f0 2024-05-20T20:11:47.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.933+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdd440742a0 con 0x7fdd441037f0 2024-05-20T20:11:47.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.934+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdd44074790 con 0x7fdd441037f0 2024-05-20T20:11:47.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.935+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdd4406a230 con 0x7fdd441037f0 2024-05-20T20:11:47.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.940+0000 7fdd42000700 1 -- 172.21.15.28:0/3153526556 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fdd38018020 con 0x7fdd441037f0 2024-05-20T20:11:47.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.943+0000 7fdd42000700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd2c0b2af0 0x7fdd2c0b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:47.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.944+0000 7fdd42000700 1 -- 172.21.15.28:0/3153526556 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7fdd380e1d70 con 0x7fdd441037f0 2024-05-20T20:11:47.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.946+0000 7fdd42000700 1 -- 172.21.15.28:0/3153526556 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fdd380e6070 con 0x7fdd441037f0 2024-05-20T20:11:47.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.948+0000 7fdd48a00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd2c0b2af0 0x7fdd2c0b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:47.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:47.949+0000 7fdd48a00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd2c0b2af0 0x7fdd2c0b4fb0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fdd3000b580 tx=0x7fdd30005bc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:48.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.531+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fdd4410ea40 con 0x7fdd441037f0 2024-05-20T20:11:48.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.532+0000 7fdd42000700 1 -- 172.21.15.28:0/3153526556 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fdd3801f020 con 0x7fdd441037f0 2024-05-20T20:11:48.534 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:11:48.535 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:11:48.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd2c0b2af0 msgr2=0x7fdd2c0b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:48.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd2c0b2af0 0x7fdd2c0b4fb0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fdd3000b580 tx=0x7fdd30005bc0 comp rx=0 tx=0).stop 2024-05-20T20:11:48.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441037f0 msgr2=0x7fdd44110450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:48.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441037f0 0x7fdd44110450 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7fdd3800b530 tx=0x7fdd3800b8f0 comp rx=0 tx=0).stop 2024-05-20T20:11:48.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 shutdown_connections 2024-05-20T20:11:48.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd441088f0 0x7fdd4410dd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:48.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd2c0b2af0 0x7fdd2c0b4fb0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:48.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd441037f0 0x7fdd44110450 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:48.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 --2- 172.21.15.28:0/3153526556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd441041a0 0x7fdd4410d620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:48.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.537+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 >> 172.21.15.28:0/3153526556 conn(0x7fdd4407a6a0 msgr2=0x7fdd44101860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:48.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.538+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 shutdown_connections 2024-05-20T20:11:48.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:48.538+0000 7fdd4be00700 1 -- 172.21.15.28:0/3153526556 wait complete. 2024-05-20T20:11:48.540 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:11:49.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:49 smithi028 ceph-mon[31178]: pgmap v188: 97 pgs: 97 active+clean; 35 GiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 275 MiB/s wr, 834 op/s 2024-05-20T20:11:49.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:49 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3153526556' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:11:49.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:49 smithi154 ceph-mon[36314]: pgmap v188: 97 pgs: 97 active+clean; 35 GiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 275 MiB/s wr, 834 op/s 2024-05-20T20:11:49.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:49 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3153526556' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:11:49.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:49 smithi033 ceph-mon[36179]: pgmap v188: 97 pgs: 97 active+clean; 35 GiB data, 7.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 275 MiB/s wr, 834 op/s 2024-05-20T20:11:49.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:49 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3153526556' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:11:49.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.797+0000 7f17e9800700 1 -- 172.21.15.28:0/631452455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410f6a0 msgr2=0x7f17e410a540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:49.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.797+0000 7f17e9800700 1 --2- 172.21.15.28:0/631452455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410f6a0 0x7f17e410a540 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f17d4009990 tx=0x7f17d4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:11:49.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.807+0000 7f17e9800700 1 -- 172.21.15.28:0/631452455 shutdown_connections 2024-05-20T20:11:49.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.807+0000 7f17e9800700 1 --2- 172.21.15.28:0/631452455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e410f290 0x7f17e410afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:49.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.807+0000 7f17e9800700 1 --2- 172.21.15.28:0/631452455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410f6a0 0x7f17e410a540 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:49.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.808+0000 7f17e9800700 1 --2- 172.21.15.28:0/631452455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e410ee10 0x7f17e410aa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:49.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.808+0000 7f17e9800700 1 -- 172.21.15.28:0/631452455 >> 172.21.15.28:0/631452455 conn(0x7f17e406e490 msgr2=0x7f17e406e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:49.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.809+0000 7f17e9800700 1 -- 172.21.15.28:0/631452455 shutdown_connections 2024-05-20T20:11:49.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.810+0000 7f17e9800700 1 -- 172.21.15.28:0/631452455 wait complete. 2024-05-20T20:11:49.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.813+0000 7f17e9800700 1 Processor -- start 2024-05-20T20:11:49.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.814+0000 7f17e9800700 1 -- start start 2024-05-20T20:11:49.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.814+0000 7f17e9800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410ee10 0x7f17e411dc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:49.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.814+0000 7f17e9800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e410f290 0x7f17e411ad80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:49.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.814+0000 7f17e9800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e410f6a0 0x7f17e411b2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:49.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.814+0000 7f17e9800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f17e4119c60 con 0x7f17e410f290 2024-05-20T20:11:49.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.814+0000 7f17e9800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f17e4119ae0 con 0x7f17e410ee10 2024-05-20T20:11:49.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.814+0000 7f17e9800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f17e4119de0 con 0x7f17e410f6a0 2024-05-20T20:11:49.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.815+0000 7f17e3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410ee10 0x7f17e411dc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:49.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.815+0000 7f17e3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410ee10 0x7f17e411dc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38184/0 (socket says 172.21.15.28:38184) 2024-05-20T20:11:49.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.815+0000 7f17e3e00700 1 -- 172.21.15.28:0/2407119532 learned_addr learned my addr 172.21.15.28:0/2407119532 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:49.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.818+0000 7f17e3e00700 1 -- 172.21.15.28:0/2407119532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e410f6a0 msgr2=0x7f17e411b2c0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:11:49.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.818+0000 7f17e3e00700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e410f6a0 0x7f17e411b2c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:49.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.818+0000 7f17e3e00700 1 -- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e410f290 msgr2=0x7f17e411ad80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:11:49.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.818+0000 7f17e3e00700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e410f290 0x7f17e411ad80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:49.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.818+0000 7f17e3e00700 1 -- 172.21.15.28:0/2407119532 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f17e411b9f0 con 0x7f17e410ee10 2024-05-20T20:11:49.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.820+0000 7f17e3e00700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410ee10 0x7f17e411dc30 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f17d40098e0 tx=0x7f17d4009300 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:49.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.822+0000 7f17e0c00700 1 -- 172.21.15.28:0/2407119532 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4026070 con 0x7f17e410ee10 2024-05-20T20:11:49.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.822+0000 7f17e0c00700 1 -- 172.21.15.28:0/2407119532 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f17d400fbb0 con 0x7f17e410ee10 2024-05-20T20:11:49.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.823+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f17e41b4d50 con 0x7f17e410ee10 2024-05-20T20:11:49.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.823+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17e41b50f0 con 0x7f17e410ee10 2024-05-20T20:11:49.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.823+0000 7f17e0c00700 1 -- 172.21.15.28:0/2407119532 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4017650 con 0x7f17e410ee10 2024-05-20T20:11:49.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.828+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f17e410b550 con 0x7f17e410ee10 2024-05-20T20:11:49.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.833+0000 7f17e0c00700 1 -- 172.21.15.28:0/2407119532 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f17d4029070 con 0x7f17e410ee10 2024-05-20T20:11:49.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.837+0000 7f17e0c00700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17c80b2b70 0x7f17c80b5030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:49.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.838+0000 7f17e3400700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17c80b2b70 0x7f17c80b5030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:49.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.839+0000 7f17e0c00700 1 -- 172.21.15.28:0/2407119532 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f17d40e30c0 con 0x7f17e410ee10 2024-05-20T20:11:49.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.841+0000 7f17e3400700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17c80b2b70 0x7f17c80b5030 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f17e411c4a0 tx=0x7f17d0009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:49.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:49.842+0000 7f17e0c00700 1 -- 172.21.15.28:0/2407119532 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f17d4013070 con 0x7f17e410ee10 2024-05-20T20:11:50.249 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 140.8 --await-for 45.0' 2024-05-20T20:11:50.249 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 140.8 --await-for 45.0 2024-05-20T20:11:50.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.280+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f17e406a230 con 0x7f17e410ee10 2024-05-20T20:11:50.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.281+0000 7f17e0c00700 1 -- 172.21.15.28:0/2407119532 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f17d40aabe0 con 0x7f17e410ee10 2024-05-20T20:11:50.283 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:11:50.283 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:11:50.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.286+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17c80b2b70 msgr2=0x7f17c80b5030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:50.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.286+0000 7f17e9800700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17c80b2b70 0x7f17c80b5030 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f17e411c4a0 tx=0x7f17d0009260 comp rx=0 tx=0).stop 2024-05-20T20:11:50.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.286+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410ee10 msgr2=0x7f17e411dc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:50.289 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.286+0000 7f17e9800700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410ee10 0x7f17e411dc30 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f17d40098e0 tx=0x7f17d4009300 comp rx=0 tx=0).stop 2024-05-20T20:11:50.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.289+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 shutdown_connections 2024-05-20T20:11:50.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.289+0000 7f17e9800700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e410f6a0 0x7f17e411b2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:50.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.289+0000 7f17e9800700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17c80b2b70 0x7f17c80b5030 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:50.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.289+0000 7f17e9800700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e410ee10 0x7f17e411dc30 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:50.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.289+0000 7f17e9800700 1 --2- 172.21.15.28:0/2407119532 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e410f290 0x7f17e411ad80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:50.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.289+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 >> 172.21.15.28:0/2407119532 conn(0x7f17e406e490 msgr2=0x7f17e4111060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:50.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.290+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 shutdown_connections 2024-05-20T20:11:50.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:50.290+0000 7f17e9800700 1 -- 172.21.15.28:0/2407119532 wait complete. 2024-05-20T20:11:50.294 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:11:51.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:51 smithi028 ceph-mon[31178]: pgmap v189: 97 pgs: 97 active+clean; 35 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 333 MiB/s wr, 991 op/s 2024-05-20T20:11:51.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:51 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2407119532' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:11:51.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:51 smithi154 ceph-mon[36314]: pgmap v189: 97 pgs: 97 active+clean; 35 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 333 MiB/s wr, 991 op/s 2024-05-20T20:11:51.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:51 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2407119532' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:11:51.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:51 smithi033 ceph-mon[36179]: pgmap v189: 97 pgs: 97 active+clean; 35 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 333 MiB/s wr, 991 op/s 2024-05-20T20:11:51.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:51 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2407119532' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:11:51.437 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:11:51.606 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:53.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:53 smithi154 ceph-mon[36314]: pgmap v190: 97 pgs: 97 active+clean; 35 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 289 MiB/s wr, 737 op/s 2024-05-20T20:11:53.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:53 smithi033 ceph-mon[36179]: pgmap v190: 97 pgs: 97 active+clean; 35 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 289 MiB/s wr, 737 op/s 2024-05-20T20:11:53.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:53 smithi028 ceph-mon[31178]: pgmap v190: 97 pgs: 97 active+clean; 35 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 289 MiB/s wr, 737 op/s 2024-05-20T20:11:53.744 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:11:54.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:54 smithi154 ceph-mon[36314]: pgmap v191: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 319 MiB/s wr, 910 op/s 2024-05-20T20:11:54.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:54 smithi033 ceph-mon[36179]: pgmap v191: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 319 MiB/s wr, 910 op/s 2024-05-20T20:11:54.508 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:54 smithi028 ceph-mon[31178]: pgmap v191: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 319 MiB/s wr, 910 op/s 2024-05-20T20:11:56.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.659+0000 7f2f8ca00700 1 -- 172.21.15.28:0/388735674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f8810a540 msgr2=0x7f2f8810a920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.661 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:56 smithi028 ceph-mon[31178]: pgmap v192: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 262 MiB/s wr, 891 op/s 2024-05-20T20:11:56.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.659+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/388735674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f8810a540 0x7f2f8810a920 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7f2f70009990 tx=0x7f2f70009ca0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.660+0000 7f2f8ca00700 1 -- 172.21.15.28:0/388735674 shutdown_connections 2024-05-20T20:11:56.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.660+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/388735674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f88116690 0x7f2f88118a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.660+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/388735674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f8810a540 0x7f2f8810a920 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.660+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/388735674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f8810ae60 0x7f2f88116150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.660+0000 7f2f8ca00700 1 -- 172.21.15.28:0/388735674 >> 172.21.15.28:0/388735674 conn(0x7f2f8806eb30 msgr2=0x7f2f8806ef40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:56.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.660+0000 7f2f8ca00700 1 -- 172.21.15.28:0/388735674 shutdown_connections 2024-05-20T20:11:56.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.661+0000 7f2f8ca00700 1 -- 172.21.15.28:0/388735674 wait complete. 2024-05-20T20:11:56.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.662+0000 7f2f8ca00700 1 Processor -- start 2024-05-20T20:11:56.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.664+0000 7f2f8ca00700 1 -- start start 2024-05-20T20:11:56.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.665+0000 7f2f8ca00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f8810a540 0x7f2f881a24e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.666+0000 7f2f8ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f8810ae60 0x7f2f881a2a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.666+0000 7f2f8ca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f88116690 0x7f2f881a6d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.666+0000 7f2f8ca00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2f8811bfd0 con 0x7f2f8810ae60 2024-05-20T20:11:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.666+0000 7f2f8ca00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2f8811be50 con 0x7f2f88116690 2024-05-20T20:11:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.666+0000 7f2f8ca00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2f8811c150 con 0x7f2f8810a540 2024-05-20T20:11:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.667+0000 7f2f86800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f88116690 0x7f2f881a6d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.667+0000 7f2f86800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f88116690 0x7f2f881a6d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38190/0 (socket says 172.21.15.28:38190) 2024-05-20T20:11:56.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.667+0000 7f2f86800700 1 -- 172.21.15.28:0/1910862869 learned_addr learned my addr 172.21.15.28:0/1910862869 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:56.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.667+0000 7f2f85e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f8810a540 0x7f2f881a24e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.669+0000 7f2f86800700 1 -- 172.21.15.28:0/1910862869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f8810a540 msgr2=0x7f2f881a24e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.669+0000 7f2f86800700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f8810a540 0x7f2f881a24e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.669+0000 7f2f86800700 1 -- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f8810ae60 msgr2=0x7f2f881a2a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.671+0000 7f2f85400700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f8810ae60 0x7f2f881a2a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.671+0000 7f2f86800700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f8810ae60 0x7f2f881a2a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.671+0000 7f2f86800700 1 -- 172.21.15.28:0/1910862869 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2f881a74a0 con 0x7f2f88116690 2024-05-20T20:11:56.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.672+0000 7f2f85400700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f8810ae60 0x7f2f881a2a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:56.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.672+0000 7f2f86800700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f88116690 0x7f2f881a6d60 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f2f8000b7b0 tx=0x7f2f8000bb70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:56.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.673+0000 7f2f7ea00700 1 -- 172.21.15.28:0/1910862869 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f8000f840 con 0x7f2f88116690 2024-05-20T20:11:56.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.674+0000 7f2f7ea00700 1 -- 172.21.15.28:0/1910862869 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f80017400 con 0x7f2f88116690 2024-05-20T20:11:56.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.674+0000 7f2f7ea00700 1 -- 172.21.15.28:0/1910862869 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f8000d630 con 0x7f2f88116690 2024-05-20T20:11:56.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.674+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2f881a7790 con 0x7f2f88116690 2024-05-20T20:11:56.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.675+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2f8810bee0 con 0x7f2f88116690 2024-05-20T20:11:56.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.681+0000 7f2f7ea00700 1 -- 172.21.15.28:0/1910862869 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f2f8000d7d0 con 0x7f2f88116690 2024-05-20T20:11:56.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.682+0000 7f2f973b6b80 1 Processor -- start 2024-05-20T20:11:56.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.683+0000 7f2f7ea00700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f6c0b2b20 0x7f2f6c0b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.683+0000 7f2f7ea00700 1 -- 172.21.15.28:0/1910862869 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2f800e23e0 con 0x7f2f88116690 2024-05-20T20:11:56.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.684+0000 7f2f85400700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f6c0b2b20 0x7f2f6c0b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.684+0000 7f2f973b6b80 1 -- start start 2024-05-20T20:11:56.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb65297850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652919f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb65292450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f973b6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cb651effd0 con 0x55cb65292010 2024-05-20T20:11:56.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f973b6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cb651dfb70 con 0x55cb65297d90 2024-05-20T20:11:56.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f973b6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cb651e3bb0 con 0x55cb65297470 2024-05-20T20:11:56.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f85e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb65297850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f86800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb65292450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f85e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb65297850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:42900/0 (socket says 172.21.15.28:42900) 2024-05-20T20:11:56.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.685+0000 7f2f85e00700 1 -- 172.21.15.28:0/3933596217 learned_addr learned my addr 172.21.15.28:0/3933596217 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:56.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.686+0000 7f2f85400700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f6c0b2b20 0x7f2f6c0b4fe0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f2f88074c30 tx=0x7f2f78009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:56.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.687+0000 7f2f85400700 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652919f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.687+0000 7f2f85e00700 1 -- 172.21.15.28:0/3933596217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 msgr2=0x55cb652919f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:56.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.687+0000 7f2f85e00700 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652919f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.687+0000 7f2f85e00700 1 -- 172.21.15.28:0/3933596217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 msgr2=0x55cb65292450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:56.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.687+0000 7f2f85e00700 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb65292450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.687+0000 7f2f85e00700 1 -- 172.21.15.28:0/3933596217 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cb651a85b0 con 0x55cb65297470 2024-05-20T20:11:56.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.688+0000 7f2f85e00700 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb65297850 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f2f8806ca90 tx=0x7f2f70005da0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3c4e5b6b076eef16 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:56.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.689+0000 7f2f77400700 1 -- 172.21.15.28:0/3933596217 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f70005070 con 0x55cb65297470 2024-05-20T20:11:56.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.689+0000 7f2f77400700 1 -- 172.21.15.28:0/3933596217 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f700056c0 con 0x55cb65297470 2024-05-20T20:11:56.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.689+0000 7f2f77400700 1 -- 172.21.15.28:0/3933596217 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f700038e0 con 0x55cb65297470 2024-05-20T20:11:56.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.692+0000 7f2f973b6b80 1 -- 172.21.15.28:0/3933596217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 msgr2=0x55cb65297850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.693+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb65297850 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f2f8806ca90 tx=0x7f2f70005da0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.694+0000 7f2f973b6b80 1 -- 172.21.15.28:0/3933596217 shutdown_connections 2024-05-20T20:11:56.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.694+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb65292450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.694+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb65297850 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.694+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/3933596217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652919f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.694+0000 7f2f973b6b80 1 -- 172.21.15.28:0/3933596217 >> 172.21.15.28:0/3933596217 conn(0x55cb651bfd70 msgr2=0x55cb6528c4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:56.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.695+0000 7f2f973b6b80 1 -- 172.21.15.28:0/3933596217 shutdown_connections 2024-05-20T20:11:56.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.696+0000 7f2f973b6b80 1 -- 172.21.15.28:0/3933596217 wait complete. 2024-05-20T20:11:56.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.697+0000 7f2f973b6b80 1 -- start start 2024-05-20T20:11:56.697 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.697+0000 7f2f973b6b80 1 Processor -- start 2024-05-20T20:11:56.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.699+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb6529da80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.700+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb6529ae20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.700+0000 7f2f85e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb6529da80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.700+0000 7f2f85e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb6529da80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:42912/0 (socket says 172.21.15.28:42912) 2024-05-20T20:11:56.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.701+0000 7f2f85e00700 1 -- 172.21.15.28:0/192669649 learned_addr learned my addr 172.21.15.28:0/192669649 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:56.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.701+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb6529b360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.701+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cb650e6130 con 0x55cb65297d90 2024-05-20T20:11:56.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.701+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cb651808f0 con 0x55cb65297470 2024-05-20T20:11:56.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.701+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cb6529e810 con 0x55cb65292010 2024-05-20T20:11:56.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.701+0000 7f2f973b6b80 10 client.0 _ll_register_callbacks cb 0x55cb65189e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:56.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.702+0000 7f2f86800700 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb6529b360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.702+0000 7f2f85e00700 1 -- 172.21.15.28:0/192669649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 msgr2=0x55cb6529ae20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:56.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.702+0000 7f2f85e00700 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb6529ae20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.703+0000 7f2f85e00700 1 -- 172.21.15.28:0/192669649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 msgr2=0x55cb6529b360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:56.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.703+0000 7f2f85e00700 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb6529b360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.703+0000 7f2f85e00700 1 -- 172.21.15.28:0/192669649 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cb6512fa60 con 0x55cb65292010 2024-05-20T20:11:56.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.703+0000 7f2f86800700 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb6529b360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:56.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.704+0000 7f2f85e00700 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb6529da80 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f2f70009990 tx=0x7f2f7000f740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cf820f730983b595 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:56.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.704+0000 7f2f6be00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:56.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.704+0000 7f2f6be00700 1 -- 172.21.15.28:0/192669649 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f7000be60 con 0x55cb65292010 2024-05-20T20:11:56.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.704+0000 7f2f6be00700 1 -- 172.21.15.28:0/192669649 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2f70005320 con 0x55cb65292010 2024-05-20T20:11:56.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.704+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55cb651f1f20 con 0x55cb65292010 2024-05-20T20:11:56.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.705+0000 7f2f6be00700 1 -- 172.21.15.28:0/192669649 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2f70005520 con 0x55cb65292010 2024-05-20T20:11:56.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.705+0000 7f2f973b6b80 10 client.26653 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:56.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.705+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cb651b01f0 con 0x55cb65292010 2024-05-20T20:11:56.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.705+0000 7f2f6be00700 1 -- 172.21.15.28:0/192669649 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55cb651f1f20 con 0x55cb65292010 2024-05-20T20:11:56.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.705+0000 7f2f6be00700 1 -- 172.21.15.28:0/192669649 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f2f70003ac0 con 0x55cb65292010 2024-05-20T20:11:56.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.706+0000 7f2f973b6b80 10 client.26653 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.706+0000 7f2f973b6b80 10 client.26653 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.706+0000 7f2f973b6b80 20 client.26653 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.706+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb652ab320 0x55cb652ad790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.706+0000 7f2f973b6b80 4 client.26653 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.706+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55cb651eb700 con 0x55cb652ab320 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.706+0000 7f2f85400700 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb652ab320 0x55cb652ad790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.707+0000 7f2f85400700 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb652ab320 0x55cb652ad790 crc :-1 s=READY pgs=151 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=89f1d642a08e4fc3 server_cookie=b0d31782220f1425 in_seq=0 out_seq=0 2024-05-20T20:11:56.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.707+0000 7f2f6be00700 10 client.26653 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:56.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.708+0000 7f2f6be00700 1 -- 172.21.15.28:0/192669649 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55cb651eb700 con 0x55cb652ab320 2024-05-20T20:11:56.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.708+0000 7f2f6be00700 10 client.26653 handle_command_reply: tid=0 2024-05-20T20:11:56.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.709+0000 7f2f973b6b80 1 client.26653 shutdown 2024-05-20T20:11:56.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.709+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 msgr2=0x55cb6529da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.709+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb6529da80 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f2f70009990 tx=0x7f2f7000f740 comp rx=0 tx=0).stop 2024-05-20T20:11:56.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.710+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 shutdown_connections 2024-05-20T20:11:56.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.710+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb652ab320 0x55cb652ad790 crc :-1 s=READY pgs=151 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.710+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb6529b360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.710+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb6529da80 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.710+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/192669649 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb6529ae20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.710+0000 7f2f6be00700 0 client.26653 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:56.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.710+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 >> 172.21.15.28:0/192669649 conn(0x55cb651bfd70 msgr2=0x55cb652909e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:56.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.711+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 shutdown_connections 2024-05-20T20:11:56.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.711+0000 7f2f973b6b80 1 -- 172.21.15.28:0/192669649 wait complete. 2024-05-20T20:11:56.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.711+0000 7f2f973b6b80 20 client.26653 trim_cache size 0 max 16384 2024-05-20T20:11:56.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.738+0000 7f2f973b6b80 1 Processor -- start 2024-05-20T20:11:56.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.740+0000 7f2f973b6b80 1 -- start start 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.740+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb65298a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.740+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb65291940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.740+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb65286520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.740+0000 7f2f973b6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2f881b44a0 con 0x55cb65297d90 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.740+0000 7f2f973b6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2f881b5fe0 con 0x55cb65297470 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.740+0000 7f2f973b6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2f881b4b70 con 0x55cb65292010 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.741+0000 7f2f86800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb65286520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.742+0000 7f2f86800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb65286520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52690/0 (socket says 172.21.15.28:52690) 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.742+0000 7f2f86800700 1 -- 172.21.15.28:0/1355866323 learned_addr learned my addr 172.21.15.28:0/1355866323 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.743+0000 7f2f86800700 1 -- 172.21.15.28:0/1355866323 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 msgr2=0x55cb65298a20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.743+0000 7f2f86800700 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb65298a20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.743+0000 7f2f86800700 1 -- 172.21.15.28:0/1355866323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 msgr2=0x55cb65291940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.743+0000 7f2f85400700 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb65291940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.743+0000 7f2f86800700 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb65291940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.743+0000 7f2f86800700 1 -- 172.21.15.28:0/1355866323 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cb652ade10 con 0x55cb65297d90 2024-05-20T20:11:56.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.743+0000 7f2f85400700 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb65291940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:56.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.745+0000 7f2f86800700 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb65286520 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f2f800dfba0 tx=0x7f2f800cc040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7a0ec9cfd98d8b92 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:56.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.747+0000 7f2f76a00700 1 -- 172.21.15.28:0/1355866323 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f800b2530 con 0x55cb65297d90 2024-05-20T20:11:56.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.748+0000 7f2f76a00700 1 -- 172.21.15.28:0/1355866323 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f800a5cc0 con 0x55cb65297d90 2024-05-20T20:11:56.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.748+0000 7f2f76a00700 1 -- 172.21.15.28:0/1355866323 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f800b2530 con 0x55cb65297d90 2024-05-20T20:11:56.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.751+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1355866323 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 msgr2=0x55cb65286520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.751+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb65286520 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f2f800dfba0 tx=0x7f2f800cc040 comp rx=0 tx=0).stop 2024-05-20T20:11:56.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.753+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1355866323 shutdown_connections 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.753+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65297d90 0x55cb65286520 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.753+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65292010 0x55cb65298a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.753+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1355866323 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297470 0x55cb65291940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.753+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1355866323 >> 172.21.15.28:0/1355866323 conn(0x55cb651bfd70 msgr2=0x55cb65285e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.754+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1355866323 shutdown_connections 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.755+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1355866323 wait complete. 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.757+0000 7f2f973b6b80 1 -- start start 2024-05-20T20:11:56.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.758+0000 7f2f973b6b80 1 Processor -- start 2024-05-20T20:11:56.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.765+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb652a40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.765+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb652a0ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.766+0000 7f2f973b6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652a1010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.767+0000 7f2f85400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb652a0ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.767+0000 7f2f85400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb652a0ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:42930/0 (socket says 172.21.15.28:42930) 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.767+0000 7f2f85400700 1 -- 172.21.15.28:0/1094637192 learned_addr learned my addr 172.21.15.28:0/1094637192 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.767+0000 7f2f86800700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652a1010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.768+0000 7f2f85e00700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb652a40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.767+0000 7f2f973b6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2f881b5910 con 0x55cb65292010 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.769+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2f881b5240 con 0x55cb65297d90 2024-05-20T20:11:56.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.769+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2f881b2460 con 0x55cb65297470 2024-05-20T20:11:56.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.770+0000 7f2f973b6b80 10 client.0 _ll_register_callbacks cb 0x55cb65189e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:56.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.770+0000 7f2f85400700 1 -- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 msgr2=0x55cb652a1010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:56.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.770+0000 7f2f85400700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652a1010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.771+0000 7f2f85400700 1 -- 172.21.15.28:0/1094637192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 msgr2=0x55cb652a40f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:56.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.773+0000 7f2f85400700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb652a40f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.774+0000 7f2f85400700 1 -- 172.21.15.28:0/1094637192 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cb652a16d0 con 0x55cb65297470 2024-05-20T20:11:56.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.774+0000 7f2f85e00700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb652a40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:56.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.776+0000 7f2f85400700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb652a0ad0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f2f78009fd0 tx=0x7f2f780047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=acbe60d45a97794f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:56.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.776+0000 7f2f6be00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:56.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.777+0000 7f2f6be00700 1 -- 172.21.15.28:0/1094637192 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2f78004230 con 0x55cb65297470 2024-05-20T20:11:56.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.777+0000 7f2f6be00700 1 -- 172.21.15.28:0/1094637192 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f2f780043d0 con 0x55cb65297470 2024-05-20T20:11:56.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.777+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2f800a5cc0 con 0x55cb65297470 2024-05-20T20:11:56.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.778+0000 7f2f6be00700 1 -- 172.21.15.28:0/1094637192 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2f78036050 con 0x55cb65297470 2024-05-20T20:11:56.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.778+0000 7f2f973b6b80 10 client.26656 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:56.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.778+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cb65312da0 con 0x55cb65297470 2024-05-20T20:11:56.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f6be00700 1 -- 172.21.15.28:0/1094637192 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2f800a5cc0 con 0x55cb65297470 2024-05-20T20:11:56.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f6be00700 1 -- 172.21.15.28:0/1094637192 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f2f7800f040 con 0x55cb65297470 2024-05-20T20:11:56.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f973b6b80 10 client.26656 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:56.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f973b6b80 10 client.26656 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:11:56.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f973b6b80 20 client.26656 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:56.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb65313550 0x55cb65313960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:56.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f973b6b80 4 client.26656 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 140.8, "await_for": 45.0}] 2024-05-20T20:11:56.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.779+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 140.8, "await_for": 45.0}) -- 0x55cb650623e0 con 0x55cb65313550 2024-05-20T20:11:56.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.780+0000 7f2f85e00700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb65313550 0x55cb65313960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:56.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.780+0000 7f2f85e00700 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb65313550 0x55cb65313960 crc :-1 s=READY pgs=152 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8907d5e49eba19a7 server_cookie=cd2306c8389bb0b8 in_seq=0 out_seq=0 2024-05-20T20:11:56.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.780+0000 7f2f6be00700 10 client.26656 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:56.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.822+0000 7f2f6be00700 1 -- 172.21.15.28:0/1094637192 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x55cb650623e0 con 0x55cb65313550 2024-05-20T20:11:56.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.822+0000 7f2f6be00700 10 client.26656 handle_command_reply: tid=0 2024-05-20T20:11:56.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.822+0000 7f2f973b6b80 1 client.26656 shutdown 2024-05-20T20:11:56.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 msgr2=0x55cb652a0ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb652a0ad0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f2f78009fd0 tx=0x7f2f780047c0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 shutdown_connections 2024-05-20T20:11:56.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb65313550 0x55cb65313960 crc :-1 s=READY pgs=152 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb65297d90 0x55cb652a1010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb65292010 0x55cb652a40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 --2- 172.21.15.28:0/1094637192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb65297470 0x55cb652a0ad0 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 >> 172.21.15.28:0/1094637192 conn(0x55cb651bfd70 msgr2=0x55cb65286520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f6be00700 0 client.26656 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 shutdown_connections 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.823+0000 7f2f973b6b80 1 -- 172.21.15.28:0/1094637192 wait complete. 2024-05-20T20:11:56.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.824+0000 7f2f973b6b80 20 client.26656 trim_cache size 0 max 16384 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 14, 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "b24c10ea": { 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 14, 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:11:56.825 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 140.8, 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:11:56.826 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:11:56.827 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:11:56.827 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:11:56.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f6c0b2b20 msgr2=0x7f2f6c0b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f6c0b2b20 0x7f2f6c0b4fe0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f2f88074c30 tx=0x7f2f78009260 comp rx=0 tx=0).stop 2024-05-20T20:11:56.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f88116690 msgr2=0x7f2f881a6d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:56.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f88116690 0x7f2f881a6d60 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f2f8000b7b0 tx=0x7f2f8000bb70 comp rx=0 tx=0).stop 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 shutdown_connections 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f2f6c0b2b20 0x7f2f6c0b4fe0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2f88116690 0x7f2f881a6d60 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2f8810a540 0x7f2f881a24e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 --2- 172.21.15.28:0/1910862869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2f8810ae60 0x7f2f881a2a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 >> 172.21.15.28:0/1910862869 conn(0x7f2f8806eb30 msgr2=0x7f2f88073540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 shutdown_connections 2024-05-20T20:11:56.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:56.825+0000 7f2f8ca00700 1 -- 172.21.15.28:0/1910862869 wait complete. 2024-05-20T20:11:56.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:56 smithi154 ceph-mon[36314]: pgmap v192: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 262 MiB/s wr, 891 op/s 2024-05-20T20:11:56.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:56 smithi033 ceph-mon[36179]: pgmap v192: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 262 MiB/s wr, 891 op/s 2024-05-20T20:11:58.741 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set 'b24c10ea', quiesce took 19.9 seconds. Will release after: 0.9000000000000021 2024-05-20T20:11:59.015 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:11:58 smithi028 ceph-mon[31178]: pgmap v193: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 264 MiB/s wr, 961 op/s 2024-05-20T20:11:59.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:11:58 smithi033 ceph-mon[36179]: pgmap v193: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 264 MiB/s wr, 961 op/s 2024-05-20T20:11:59.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:11:58 smithi154 ceph-mon[36314]: pgmap v193: 97 pgs: 97 active+clean; 36 GiB data, 7.7 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 264 MiB/s wr, 961 op/s 2024-05-20T20:11:59.643 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:11:59.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.700+0000 7f83f7600700 1 -- 172.21.15.28:0/2436010708 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 msgr2=0x7f83f0116e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.700+0000 7f83f7600700 1 --2- 172.21.15.28:0/2436010708 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 0x7f83f0116e30 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f83e800c310 tx=0x7f83e800c620 comp rx=0 tx=0).stop 2024-05-20T20:11:59.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.702+0000 7f83f7600700 1 -- 172.21.15.28:0/2436010708 shutdown_connections 2024-05-20T20:11:59.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.702+0000 7f83f7600700 1 --2- 172.21.15.28:0/2436010708 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 0x7f83f0116e30 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.702+0000 7f83f7600700 1 --2- 172.21.15.28:0/2436010708 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83f010ef90 0x7f83f010f370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.702+0000 7f83f7600700 1 --2- 172.21.15.28:0/2436010708 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83f010a540 0x7f83f010a9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.702+0000 7f83f7600700 1 -- 172.21.15.28:0/2436010708 >> 172.21.15.28:0/2436010708 conn(0x7f83f006ea90 msgr2=0x7f83f006eea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:59.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.703+0000 7f83f7600700 1 -- 172.21.15.28:0/2436010708 shutdown_connections 2024-05-20T20:11:59.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.703+0000 7f83f7600700 1 -- 172.21.15.28:0/2436010708 wait complete. 2024-05-20T20:11:59.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.705+0000 7f83f7600700 1 Processor -- start 2024-05-20T20:11:59.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.706+0000 7f83f7600700 1 -- start start 2024-05-20T20:11:59.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.707+0000 7f83f7600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83f010a540 0x7f83f01137b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.707 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.707+0000 7f83f7600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 0x7f83f0113cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.707+0000 7f83f7600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83f010ef90 0x7f83f01b4540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83f010a540 0x7f83f01137b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f7600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f83f01198c0 con 0x7f83f010ef90 2024-05-20T20:11:59.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f7600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f83f0119740 con 0x7f83f010af00 2024-05-20T20:11:59.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f7600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f83f0119a40 con 0x7f83f010a540 2024-05-20T20:11:59.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 0x7f83f0113cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 0x7f83f0113cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37064/0 (socket says 172.21.15.28:37064) 2024-05-20T20:11:59.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83e7e00700 1 -- 172.21.15.28:0/1239597278 learned_addr learned my addr 172.21.15.28:0/1239597278 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:59.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f5600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83f010ef90 0x7f83f01b4540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f4c00700 1 -- 172.21.15.28:0/1239597278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 msgr2=0x7f83f0113cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f4c00700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 0x7f83f0113cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.708+0000 7f83f4c00700 1 -- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83f010ef90 msgr2=0x7f83f01b4540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.709+0000 7f83f4c00700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83f010ef90 0x7f83f01b4540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.709+0000 7f83f4c00700 1 -- 172.21.15.28:0/1239597278 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f83f01b4b50 con 0x7f83f010a540 2024-05-20T20:11:59.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.709+0000 7f83f5600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83f010ef90 0x7f83f01b4540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:59.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.709+0000 7f83f4c00700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83f010a540 0x7f83f01137b0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f83d800c970 tx=0x7f83d800cc80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:59.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.711+0000 7f83ee000700 1 -- 172.21.15.28:0/1239597278 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83d8014070 con 0x7f83f010a540 2024-05-20T20:11:59.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.712+0000 7f83ee000700 1 -- 172.21.15.28:0/1239597278 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f83d8004440 con 0x7f83f010a540 2024-05-20T20:11:59.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.712+0000 7f83ee000700 1 -- 172.21.15.28:0/1239597278 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83d8003b40 con 0x7f83f010a540 2024-05-20T20:11:59.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.712+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f83f01b4e40 con 0x7f83f010a540 2024-05-20T20:11:59.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.712+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f83f01b52d0 con 0x7f83f010a540 2024-05-20T20:11:59.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.719+0000 7f83ee000700 1 -- 172.21.15.28:0/1239597278 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f83d8017070 con 0x7f83f010a540 2024-05-20T20:11:59.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.721+0000 7f8401fcbb80 1 Processor -- start 2024-05-20T20:11:59.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f8401fcbb80 1 -- start start 2024-05-20T20:11:59.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f83ee000700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83d40b2af0 0x7f83d40b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d423139f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4230db20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d4230e5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562d4226be40 con 0x562d42313610 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562d4225b9e0 con 0x562d4230e140 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562d4225fa20 con 0x562d42313f30 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.722+0000 7f83ee000700 1 -- 172.21.15.28:0/1239597278 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f83d80e22d0 con 0x7f83f010a540 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.723+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4230db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.723+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4230db20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56702/0 (socket says 172.21.15.28:56702) 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.723+0000 7f83e7e00700 1 -- 172.21.15.28:0/2409207972 learned_addr learned my addr 172.21.15.28:0/2409207972 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.723+0000 7f83f5600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83d40b2af0 0x7f83d40b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.724+0000 7f83f5600700 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d4230e5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.724+0000 7f83f4c00700 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d423139f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.725+0000 7f83e7e00700 1 -- 172.21.15.28:0/2409207972 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 msgr2=0x562d4230e5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.725+0000 7f83e7e00700 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d4230e5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.725+0000 7f83e7e00700 1 -- 172.21.15.28:0/2409207972 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 msgr2=0x562d423139f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.725+0000 7f83e7e00700 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d423139f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.725+0000 7f83e7e00700 1 -- 172.21.15.28:0/2409207972 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562d42224420 con 0x562d42313f30 2024-05-20T20:11:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.725+0000 7f83f5600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83d40b2af0 0x7f83d40b4fb0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f83e800c1b0 tx=0x7f83e800c120 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.726+0000 7f83e7e00700 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4230db20 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f83e0007d90 tx=0x7f83e00106b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f37fa4f9b4556a2c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.726+0000 7f83e6000700 1 -- 172.21.15.28:0/2409207972 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83e0010e10 con 0x562d42313f30 2024-05-20T20:11:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.726+0000 7f83e6000700 1 -- 172.21.15.28:0/2409207972 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f83e00118f0 con 0x562d42313f30 2024-05-20T20:11:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.726+0000 7f83e6000700 1 -- 172.21.15.28:0/2409207972 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83e0019790 con 0x562d42313f30 2024-05-20T20:11:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.727+0000 7f8401fcbb80 1 -- 172.21.15.28:0/2409207972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 msgr2=0x562d4230db20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.727+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4230db20 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f83e0007d90 tx=0x7f83e00106b0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.727+0000 7f8401fcbb80 1 -- 172.21.15.28:0/2409207972 shutdown_connections 2024-05-20T20:11:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.727+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d4230e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.727+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d423139f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.727+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/2409207972 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4230db20 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.727+0000 7f8401fcbb80 1 -- 172.21.15.28:0/2409207972 >> 172.21.15.28:0/2409207972 conn(0x562d4223bbe0 msgr2=0x562d42308690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.729+0000 7f8401fcbb80 1 -- 172.21.15.28:0/2409207972 shutdown_connections 2024-05-20T20:11:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.729+0000 7f8401fcbb80 1 -- 172.21.15.28:0/2409207972 wait complete. 2024-05-20T20:11:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.729+0000 7f8401fcbb80 1 -- start start 2024-05-20T20:11:59.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.730+0000 7f8401fcbb80 1 Processor -- start 2024-05-20T20:11:59.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.730+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d42319c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.730+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d42316fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d42317500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562d42161fa0 con 0x562d42313610 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562d421fc760 con 0x562d4230e140 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562d4231aa30 con 0x562d42313f30 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f8401fcbb80 10 client.0 _ll_register_callbacks cb 0x562d42205c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f83f4c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d42319c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f83f4c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d42319c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37096/0 (socket says 172.21.15.28:37096) 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f83f4c00700 1 -- 172.21.15.28:0/3926067056 learned_addr learned my addr 172.21.15.28:0/3926067056 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f83f5600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d42317500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.731+0000 7f83e7e00700 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d42316fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83f4c00700 1 -- 172.21.15.28:0/3926067056 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 msgr2=0x562d42317500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83f4c00700 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d42317500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83f4c00700 1 -- 172.21.15.28:0/3926067056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 msgr2=0x562d42316fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83f4c00700 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d42316fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83f4c00700 1 -- 172.21.15.28:0/3926067056 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562d421ab8d0 con 0x562d4230e140 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83e7e00700 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d42316fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83f4c00700 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d42319c20 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f83d80a94f0 tx=0x7f83d8018690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b8b0f6b7716a022e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83dea00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83dea00700 1 -- 172.21.15.28:0/3926067056 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83d80c3060 con 0x562d4230e140 2024-05-20T20:11:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f83dea00700 1 -- 172.21.15.28:0/3926067056 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f83d80a4b80 con 0x562d4230e140 2024-05-20T20:11:59.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.732+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f83e00118f0 con 0x562d4230e140 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.733+0000 7f83dea00700 1 -- 172.21.15.28:0/3926067056 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f83d80ad910 con 0x562d4230e140 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.733+0000 7f8401fcbb80 10 client.26753 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.733+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562d4222c060 con 0x562d4230e140 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.733+0000 7f83dea00700 1 -- 172.21.15.28:0/3926067056 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f83e00118f0 con 0x562d4230e140 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.733+0000 7f83dea00700 1 -- 172.21.15.28:0/3926067056 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f83d80aeaa0 con 0x562d4230e140 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.734+0000 7f8401fcbb80 10 client.26753 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.734+0000 7f8401fcbb80 10 client.26753 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.734+0000 7f8401fcbb80 20 client.26753 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.734+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423272c0 0x562d423296f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.734+0000 7f8401fcbb80 4 client.26753 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.734+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562d4226dd90 con 0x562d423272c0 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.734+0000 7f83e7e00700 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423272c0 0x562d423296f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.735+0000 7f83e7e00700 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423272c0 0x562d423296f0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=e4b02b9bc140ea29 server_cookie=8a18ccd19bfcc241 in_seq=0 out_seq=0 2024-05-20T20:11:59.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.735+0000 7f83dea00700 10 client.26753 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:11:59.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.736+0000 7f83dea00700 1 -- 172.21.15.28:0/3926067056 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562d4226dd90 con 0x562d423272c0 2024-05-20T20:11:59.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.736+0000 7f83dea00700 10 client.26753 handle_command_reply: tid=0 2024-05-20T20:11:59.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.736+0000 7f8401fcbb80 1 client.26753 shutdown 2024-05-20T20:11:59.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.736+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 msgr2=0x562d42319c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.736+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d42319c20 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f83d80a94f0 tx=0x7f83d8018690 comp rx=0 tx=0).stop 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.737+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 shutdown_connections 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.737+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423272c0 0x562d423296f0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.737+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d42317500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.737+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d4230e140 0x562d42319c20 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.737+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/3926067056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42313610 0x562d42316fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.737+0000 7f83dea00700 0 client.26753 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:11:59.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.737+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 >> 172.21.15.28:0/3926067056 conn(0x562d4223bbe0 msgr2=0x562d42307d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:59.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.738+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 shutdown_connections 2024-05-20T20:11:59.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.738+0000 7f8401fcbb80 1 -- 172.21.15.28:0/3926067056 wait complete. 2024-05-20T20:11:59.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.738+0000 7f8401fcbb80 20 client.26753 trim_cache size 0 max 16384 2024-05-20T20:11:59.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.759+0000 7f8401fcbb80 1 Processor -- start 2024-05-20T20:11:59.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.759+0000 7f8401fcbb80 1 -- start start 2024-05-20T20:11:59.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.759+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313610 0x562d4230db20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.759+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313f30 0x562d423174b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.759+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d423084f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.759+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f83f01bcb50 con 0x562d42323ad0 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.759+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f83f01be690 con 0x562d42313f30 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.760+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f83f01bd220 con 0x562d42313610 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.760+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313f30 0x562d423174b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.760+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313f30 0x562d423174b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37110/0 (socket says 172.21.15.28:37110) 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.760+0000 7f83e7e00700 1 -- 172.21.15.28:0/4137860455 learned_addr learned my addr 172.21.15.28:0/4137860455 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.760+0000 7f83f5600700 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d423084f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.761+0000 7f83f5600700 1 -- 172.21.15.28:0/4137860455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313610 msgr2=0x562d4230db20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.761+0000 7f83f5600700 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313610 0x562d4230db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.761+0000 7f83f5600700 1 -- 172.21.15.28:0/4137860455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313f30 msgr2=0x562d423174b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.761+0000 7f83f5600700 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313f30 0x562d423174b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.761+0000 7f83f5600700 1 -- 172.21.15.28:0/4137860455 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562d42329d70 con 0x562d42323ad0 2024-05-20T20:11:59.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.761+0000 7f83e7e00700 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313f30 0x562d423174b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:11:59.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.762+0000 7f83f5600700 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d423084f0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f83e800caa0 tx=0x7f83e8022000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8481bf92b28bab4f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:59.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.763+0000 7f83e5600700 1 -- 172.21.15.28:0/4137860455 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83e802f070 con 0x562d42323ad0 2024-05-20T20:11:59.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.764+0000 7f83e5600700 1 -- 172.21.15.28:0/4137860455 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f83e8012e20 con 0x562d42323ad0 2024-05-20T20:11:59.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.764+0000 7f83e5600700 1 -- 172.21.15.28:0/4137860455 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83e8027050 con 0x562d42323ad0 2024-05-20T20:11:59.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.766+0000 7f8401fcbb80 1 -- 172.21.15.28:0/4137860455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 msgr2=0x562d423084f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.766+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d423084f0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f83e800caa0 tx=0x7f83e8022000 comp rx=0 tx=0).stop 2024-05-20T20:11:59.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.766+0000 7f8401fcbb80 1 -- 172.21.15.28:0/4137860455 shutdown_connections 2024-05-20T20:11:59.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.766+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d423084f0 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.766+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313610 0x562d4230db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.766+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/4137860455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313f30 0x562d423174b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.767+0000 7f8401fcbb80 1 -- 172.21.15.28:0/4137860455 >> 172.21.15.28:0/4137860455 conn(0x562d4223bbe0 msgr2=0x562d42316da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:59.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.767+0000 7f8401fcbb80 1 -- 172.21.15.28:0/4137860455 shutdown_connections 2024-05-20T20:11:59.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.767+0000 7f8401fcbb80 1 -- 172.21.15.28:0/4137860455 wait complete. 2024-05-20T20:11:59.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.768+0000 7f8401fcbb80 1 -- start start 2024-05-20T20:11:59.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.768+0000 7f8401fcbb80 1 Processor -- start 2024-05-20T20:11:59.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.769+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313610 0x562d4238d6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.769+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4231fcc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.769+0000 7f8401fcbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d42320200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.769+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f83f01bdfc0 con 0x562d42323ad0 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.769+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f83f01bd8f0 con 0x562d42313610 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.769+0000 7f8401fcbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f83f01baaf0 con 0x562d42313f30 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.770+0000 7f8401fcbb80 10 client.0 _ll_register_callbacks cb 0x562d42205c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.770+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4231fcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.770+0000 7f83e7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4231fcc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56726/0 (socket says 172.21.15.28:56726) 2024-05-20T20:11:59.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.770+0000 7f83e7e00700 1 -- 172.21.15.28:0/496279022 learned_addr learned my addr 172.21.15.28:0/496279022 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:11:59.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.770+0000 7f83f4c00700 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313610 0x562d4238d6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83e7e00700 1 -- 172.21.15.28:0/496279022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313610 msgr2=0x562d4238d6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83f5600700 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d42320200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83e7e00700 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313610 0x562d4238d6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83e7e00700 1 -- 172.21.15.28:0/496279022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 msgr2=0x562d42320200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:11:59.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83e7e00700 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d42320200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83e7e00700 1 -- 172.21.15.28:0/496279022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562d423208d0 con 0x562d42313f30 2024-05-20T20:11:59.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83e7e00700 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4231fcc0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f83e0007d90 tx=0x7f83e0005ce0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4d49d2551df9a172 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:11:59.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.771+0000 7f83dea00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:11:59.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.772+0000 7f83dea00700 1 -- 172.21.15.28:0/496279022 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83e002c040 con 0x562d42313f30 2024-05-20T20:11:59.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.772+0000 7f83dea00700 1 -- 172.21.15.28:0/496279022 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f83e0005040 con 0x562d42313f30 2024-05-20T20:11:59.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.772+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f83e8012e20 con 0x562d42313f30 2024-05-20T20:11:59.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.772+0000 7f83dea00700 1 -- 172.21.15.28:0/496279022 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f83e0037050 con 0x562d42313f30 2024-05-20T20:11:59.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.772+0000 7f8401fcbb80 10 client.26671 fetch_fsmap learned FSMap version 29 2024-05-20T20:11:59.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.772+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562d423a3020 con 0x562d42313f30 2024-05-20T20:11:59.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f83dea00700 1 -- 172.21.15.28:0/496279022 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f83e8012e20 con 0x562d42313f30 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f83dea00700 1 -- 172.21.15.28:0/496279022 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f83d8003ce0 con 0x562d42313f30 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f8401fcbb80 10 client.26671 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f8401fcbb80 10 client.26671 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f8401fcbb80 20 client.26671 populate_metadata read hostname 'smithi028' 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423a3500 0x562d423a5920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f8401fcbb80 4 client.26671 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.773+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x562d42267570 con 0x562d423a3500 2024-05-20T20:11:59.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.774+0000 7f83f4c00700 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423a3500 0x562d423a5920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:11:59.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.775+0000 7f83f4c00700 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423a3500 0x562d423a5920 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=365452b85ab50ddc server_cookie=ad5c1770bca01708 in_seq=0 out_seq=0 2024-05-20T20:11:59.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.775+0000 7f83dea00700 10 client.26671 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:11:59.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.776+0000 7f83dea00700 1 -- 172.21.15.28:0/496279022 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x562d42267570 con 0x562d423a3500 2024-05-20T20:11:59.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.776+0000 7f83dea00700 10 client.26671 handle_command_reply: tid=0 2024-05-20T20:11:59.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.776+0000 7f8401fcbb80 1 client.26671 shutdown 2024-05-20T20:11:59.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.776+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 msgr2=0x562d4231fcc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.776+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4231fcc0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f83e0007d90 tx=0x7f83e0005ce0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 shutdown_connections 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562d42323ad0 0x562d42320200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562d42313610 0x562d4238d6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562d423a3500 0x562d423a5920 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 --2- 172.21.15.28:0/496279022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562d42313f30 0x562d4231fcc0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 >> 172.21.15.28:0/496279022 conn(0x562d4223bbe0 msgr2=0x562d42314310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f83dea00700 0 client.26671 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:11:59.777 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 shutdown_connections 2024-05-20T20:11:59.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 1 -- 172.21.15.28:0/496279022 wait complete. 2024-05-20T20:11:59.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.777+0000 7f8401fcbb80 20 client.26671 trim_cache size 0 max 16384 2024-05-20T20:11:59.778 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:11:59.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83d40b2af0 msgr2=0x7f83d40b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83d40b2af0 0x7f83d40b4fb0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f83e800c1b0 tx=0x7f83e800c120 comp rx=0 tx=0).stop 2024-05-20T20:11:59.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83f010a540 msgr2=0x7f83f01137b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:11:59.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83f010a540 0x7f83f01137b0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f83d800c970 tx=0x7f83d800cc80 comp rx=0 tx=0).stop 2024-05-20T20:11:59.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 shutdown_connections 2024-05-20T20:11:59.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f83f010ef90 0x7f83f01b4540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f83f010a540 0x7f83f01137b0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f83d40b2af0 0x7f83d40b4fb0 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 --2- 172.21.15.28:0/1239597278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f83f010af00 0x7f83f0113cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:11:59.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.779+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 >> 172.21.15.28:0/1239597278 conn(0x7f83f006ea90 msgr2=0x7f83f010dbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:11:59.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.780+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 shutdown_connections 2024-05-20T20:11:59.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:11:59.780+0000 7f83f7600700 1 -- 172.21.15.28:0/1239597278 wait complete. 2024-05-20T20:12:01.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.056+0000 7f0252800640 1 -- 172.21.15.28:0/669573571 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c10a410 msgr2=0x7f024c116300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.056+0000 7f0252800640 1 --2- 172.21.15.28:0/669573571 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c10a410 0x7f024c116300 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f02380074e0 tx=0x7f023802b0a0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.063+0000 7f0252800640 1 -- 172.21.15.28:0/669573571 shutdown_connections 2024-05-20T20:12:01.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.063+0000 7f0252800640 1 --2- 172.21.15.28:0/669573571 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c10a410 0x7f024c116300 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.063+0000 7f0252800640 1 --2- 172.21.15.28:0/669573571 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c109a50 0x7f024c109ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.063+0000 7f0252800640 1 --2- 172.21.15.28:0/669573571 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f024c10f540 0x7f024c10f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.063+0000 7f0252800640 1 -- 172.21.15.28:0/669573571 >> 172.21.15.28:0/669573571 conn(0x7f024c06db20 msgr2=0x7f024c06df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:01.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.063+0000 7f0252800640 1 -- 172.21.15.28:0/669573571 shutdown_connections 2024-05-20T20:12:01.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.063+0000 7f0252800640 1 -- 172.21.15.28:0/669573571 wait complete. 2024-05-20T20:12:01.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.069+0000 7f0252800640 1 Processor -- start 2024-05-20T20:12:01.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.071+0000 7f0252800640 1 -- start start 2024-05-20T20:12:01.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.072+0000 7f0252800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c109a50 0x7f024c1a80c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.072+0000 7f0252800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f024c10a410 0x7f024c1a8600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.072+0000 7f0252800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c10f540 0x7f024c1ac930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.072+0000 7f0252800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f024c1193f0 con 0x7f024c109a50 2024-05-20T20:12:01.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.072+0000 7f0252800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f024c119270 con 0x7f024c10a410 2024-05-20T20:12:01.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.072+0000 7f0252800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f024c119570 con 0x7f024c10f540 2024-05-20T20:12:01.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.075+0000 7f024be00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c10f540 0x7f024c1ac930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.075+0000 7f024be00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c10f540 0x7f024c1ac930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56748/0 (socket says 172.21.15.28:56748) 2024-05-20T20:12:01.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.075+0000 7f024be00640 1 -- 172.21.15.28:0/2299326521 learned_addr learned my addr 172.21.15.28:0/2299326521 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:01.077 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.077+0000 7f024aa00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f024c10a410 0x7f024c1a8600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.078 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.077+0000 7f024b400640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c109a50 0x7f024c1a80c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.078+0000 7f024be00640 1 -- 172.21.15.28:0/2299326521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f024c10a410 msgr2=0x7f024c1a8600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.078+0000 7f024be00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f024c10a410 0x7f024c1a8600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.078+0000 7f024be00640 1 -- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c109a50 msgr2=0x7f024c1a80c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.079+0000 7f024be00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c109a50 0x7f024c1a80c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.079+0000 7f024be00640 1 -- 172.21.15.28:0/2299326521 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f024c1ad030 con 0x7f024c10f540 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.079+0000 7f024b400640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c109a50 0x7f024c1a80c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.079+0000 7f024aa00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f024c10a410 0x7f024c1a8600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.080+0000 7f024be00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c10f540 0x7f024c1ac930 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f023800add0 tx=0x7f023802bc10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.081+0000 7f0243e00640 1 -- 172.21.15.28:0/2299326521 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f023802be30 con 0x7f024c10f540 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.081+0000 7f0243e00640 1 -- 172.21.15.28:0/2299326521 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0238004260 con 0x7f024c10f540 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.081+0000 7f0243e00640 1 -- 172.21.15.28:0/2299326521 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f023800a0d0 con 0x7f024c10f540 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.082+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f024c1ad2c0 con 0x7f024c10f540 2024-05-20T20:12:01.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.082+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f024c19e8c0 con 0x7f024c10f540 2024-05-20T20:12:01.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.086+0000 7f0255639740 1 Processor -- start 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.088+0000 7f0243e00640 1 -- 172.21.15.28:0/2299326521 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0238042070 con 0x7f024c10f540 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.089+0000 7f0255639740 1 -- start start 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.089+0000 7f0255639740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db7070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.089+0000 7f0255639740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db75b0 0x558dd0dc1930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.089+0000 7f0255639740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0dc1e70 0x558dd0dc4260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.089+0000 7f0255639740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558dd0d27b90 con 0x558dd0db75b0 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.089+0000 7f0255639740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558dd0d17730 con 0x558dd0dc1e70 2024-05-20T20:12:01.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.089+0000 7f0255639740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558dd0d1b770 con 0x558dd0db6c90 2024-05-20T20:12:01.090 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:00 smithi028 ceph-mon[31178]: pgmap v194: 97 pgs: 97 active+clean; 36 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 319 MiB/s wr, 1.29k op/s 2024-05-20T20:12:01.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.091+0000 7f0243e00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02300b2be0 0x7f02300b50a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.091+0000 7f0243e00640 1 -- 172.21.15.28:0/2299326521 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0238101140 con 0x7f024c10f540 2024-05-20T20:12:01.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.091+0000 7f024be00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02300b2be0 0x7f02300b50a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.092+0000 7f024b400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db75b0 0x558dd0dc1930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.092+0000 7f024aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db7070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.092+0000 7f024aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db7070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56754/0 (socket says 172.21.15.28:56754) 2024-05-20T20:12:01.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.092+0000 7f024aa00640 1 -- 172.21.15.28:0/3298810600 learned_addr learned my addr 172.21.15.28:0/3298810600 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:01.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.092+0000 7f024aa00640 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0dc1e70 0x558dd0dc4260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.093+0000 7f024be00640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02300b2be0 0x7f02300b50a0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f023802bfd0 tx=0x7f023802b0a0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.094+0000 7f024aa00640 1 -- 172.21.15.28:0/3298810600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0dc1e70 msgr2=0x558dd0dc4260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.094+0000 7f024aa00640 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0dc1e70 0x558dd0dc4260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.094+0000 7f024aa00640 1 -- 172.21.15.28:0/3298810600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db75b0 msgr2=0x558dd0dc1930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.094+0000 7f024aa00640 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db75b0 0x558dd0dc1930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.094+0000 7f024aa00640 1 -- 172.21.15.28:0/3298810600 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558dd0cb08b0 con 0x558dd0db6c90 2024-05-20T20:12:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.095+0000 7f024aa00640 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db7070 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f023c00cf90 tx=0x7f023c00c1d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=870415b49e4bf9b6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:01.095 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.096+0000 7f0240c00640 1 -- 172.21.15.28:0/3298810600 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f023c00f040 con 0x558dd0db6c90 2024-05-20T20:12:01.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.096+0000 7f0240c00640 1 -- 172.21.15.28:0/3298810600 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f023c003690 con 0x558dd0db6c90 2024-05-20T20:12:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.097+0000 7f0240c00640 1 -- 172.21.15.28:0/3298810600 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f023c00f040 con 0x558dd0db6c90 2024-05-20T20:12:01.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.098+0000 7f0255639740 1 -- 172.21.15.28:0/3298810600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 msgr2=0x558dd0db7070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.098+0000 7f0255639740 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db7070 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f023c00cf90 tx=0x7f023c00c1d0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.101+0000 7f0255639740 1 -- 172.21.15.28:0/3298810600 shutdown_connections 2024-05-20T20:12:01.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.101+0000 7f0255639740 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0dc1e70 0x558dd0dc4260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.101+0000 7f0255639740 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db75b0 0x558dd0dc1930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.101+0000 7f0255639740 1 --2- 172.21.15.28:0/3298810600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db7070 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.101+0000 7f0255639740 1 -- 172.21.15.28:0/3298810600 >> 172.21.15.28:0/3298810600 conn(0x558dd0db31a0 msgr2=0x558dd0db3580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:01.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.110+0000 7f0255639740 1 -- 172.21.15.28:0/3298810600 shutdown_connections 2024-05-20T20:12:01.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.110+0000 7f0255639740 1 -- 172.21.15.28:0/3298810600 wait complete. 2024-05-20T20:12:01.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.110+0000 7f0255639740 1 -- start start 2024-05-20T20:12:01.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.111+0000 7f0255639740 1 Processor -- start 2024-05-20T20:12:01.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.113+0000 7f0255639740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0dc1760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.113+0000 7f0255639740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dbea50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.113+0000 7f0255639740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 0x558dd0dbef90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.113+0000 7f0255639740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558dd09fd790 con 0x558dd0dc1e70 2024-05-20T20:12:01.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.114+0000 7f0255639740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558dd0cbfbc0 con 0x558dd0db75b0 2024-05-20T20:12:01.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.114+0000 7f0255639740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558dd0dc4a90 con 0x558dd0db6c90 2024-05-20T20:12:01.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.114+0000 7f0255639740 10 client.0 _ll_register_callbacks cb 0x558dd0c9d400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:01.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.119+0000 7f024b400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dbea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.119+0000 7f024b400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dbea50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37146/0 (socket says 172.21.15.28:37146) 2024-05-20T20:12:01.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.119+0000 7f024b400640 1 -- 172.21.15.28:0/2065304461 learned_addr learned my addr 172.21.15.28:0/2065304461 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:01.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.121+0000 7f024b400640 1 -- 172.21.15.28:0/2065304461 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 msgr2=0x558dd0dc1760 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:12:01.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.121+0000 7f024b400640 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0dc1760 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.121+0000 7f024b400640 1 -- 172.21.15.28:0/2065304461 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 msgr2=0x558dd0dbef90 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:12:01.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.121+0000 7f024b400640 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 0x558dd0dbef90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.121+0000 7f024b400640 1 -- 172.21.15.28:0/2065304461 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558dd0a2c210 con 0x558dd0db75b0 2024-05-20T20:12:01.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.122+0000 7f024b400640 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dbea50 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f0244007300 tx=0x7f02440077c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=603c06028870392 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:01.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.122+0000 7f0235600640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:01.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.122+0000 7f0235600640 1 -- 172.21.15.28:0/2065304461 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0244014760 con 0x558dd0db75b0 2024-05-20T20:12:01.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.123+0000 7f0235600640 1 -- 172.21.15.28:0/2065304461 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0244014900 con 0x558dd0db75b0 2024-05-20T20:12:01.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.123+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f023c003690 con 0x558dd0db75b0 2024-05-20T20:12:01.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.123+0000 7f0235600640 1 -- 172.21.15.28:0/2065304461 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0244014b00 con 0x558dd0db75b0 2024-05-20T20:12:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.128+0000 7f0255639740 10 client.26765 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:01.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.128+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558dd0af8620 con 0x558dd0db75b0 2024-05-20T20:12:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.129+0000 7f0235600640 1 -- 172.21.15.28:0/2065304461 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f023c003690 con 0x558dd0db75b0 2024-05-20T20:12:01.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.129+0000 7f0235600640 1 -- 172.21.15.28:0/2065304461 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f024401c300 con 0x558dd0db75b0 2024-05-20T20:12:01.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.130+0000 7f0255639740 10 client.26765 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:01.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.130+0000 7f0255639740 10 client.26765 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:01.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.130+0000 7f0255639740 20 client.26765 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:01.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.130+0000 7f0255639740 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0dd1cf0 0x558dd0dd4170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.130+0000 7f0255639740 4 client.26765 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:01.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.130+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558dd0d29ae0 con 0x558dd0dd1cf0 2024-05-20T20:12:01.138 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.139+0000 7f024aa00640 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0dd1cf0 0x558dd0dd4170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.141+0000 7f024aa00640 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0dd1cf0 0x558dd0dd4170 crc :-1 s=READY pgs=153 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1d9ad76b4afea798 server_cookie=c2ce21f04af6f7af in_seq=0 out_seq=0 2024-05-20T20:12:01.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.143+0000 7f0235600640 10 client.26765 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:01.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.144+0000 7f0235600640 1 -- 172.21.15.28:0/2065304461 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558dd0d29ae0 con 0x558dd0dd1cf0 2024-05-20T20:12:01.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.144+0000 7f0235600640 10 client.26765 handle_command_reply: tid=0 2024-05-20T20:12:01.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.144+0000 7f0255639740 1 client.26765 shutdown 2024-05-20T20:12:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.145+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 msgr2=0x558dd0dbea50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.145+0000 7f0255639740 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dbea50 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f0244007300 tx=0x7f02440077c0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.145+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 shutdown_connections 2024-05-20T20:12:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.145+0000 7f0255639740 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0dd1cf0 0x558dd0dd4170 crc :-1 s=READY pgs=153 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.145+0000 7f0235600640 0 client.26765 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.145+0000 7f0255639740 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dbea50 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.146+0000 7f0255639740 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 0x558dd0dbef90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.146+0000 7f0255639740 1 --2- 172.21.15.28:0/2065304461 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0dc1760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.146+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 >> 172.21.15.28:0/2065304461 conn(0x558dd0db31a0 msgr2=0x558dd0db8f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.147+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 shutdown_connections 2024-05-20T20:12:01.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.150+0000 7f0255639740 1 -- 172.21.15.28:0/2065304461 wait complete. 2024-05-20T20:12:01.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.150+0000 7f0255639740 20 client.26765 trim_cache size 0 max 16384 2024-05-20T20:12:01.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.180+0000 7f0255639740 1 Processor -- start 2024-05-20T20:12:01.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.180+0000 7f0255639740 1 -- start start 2024-05-20T20:12:01.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f0255639740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db2a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f0255639740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dd9a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f0255639740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 0x558dd0dd9fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f0255639740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f024c1c5d40 con 0x558dd0dc1e70 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f0255639740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f024c1c7880 con 0x558dd0db75b0 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f0255639740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f024c1c6410 con 0x558dd0db6c90 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dd9a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dd9a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37152/0 (socket says 172.21.15.28:37152) 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 -- 172.21.15.28:0/1081682096 learned_addr learned my addr 172.21.15.28:0/1081682096 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024aa00640 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db2a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 -- 172.21.15.28:0/1081682096 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 msgr2=0x558dd0db2a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db2a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 -- 172.21.15.28:0/1081682096 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 msgr2=0x558dd0dd9fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 0x558dd0dd9fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.181+0000 7f024b400640 1 -- 172.21.15.28:0/1081682096 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558dd0dd0420 con 0x558dd0db75b0 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.182+0000 7f024aa00640 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db2a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:01.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.182+0000 7f024b400640 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dd9a80 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f024401c8c0 tx=0x7f0244004ad0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dee40ca72068d40a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:01.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.182+0000 7f0237e00640 1 -- 172.21.15.28:0/1081682096 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0244014b00 con 0x558dd0db75b0 2024-05-20T20:12:01.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.182+0000 7f0237e00640 1 -- 172.21.15.28:0/1081682096 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f024401a070 con 0x558dd0db75b0 2024-05-20T20:12:01.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.182+0000 7f0237e00640 1 -- 172.21.15.28:0/1081682096 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0244027430 con 0x558dd0db75b0 2024-05-20T20:12:01.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.183+0000 7f0255639740 1 -- 172.21.15.28:0/1081682096 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 msgr2=0x558dd0dd9a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.183+0000 7f0255639740 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dd9a80 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f024401c8c0 tx=0x7f0244004ad0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.183+0000 7f0255639740 1 -- 172.21.15.28:0/1081682096 shutdown_connections 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.183+0000 7f0255639740 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dd9a80 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.183+0000 7f0255639740 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0dc1e70 0x558dd0dd9fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.183+0000 7f0255639740 1 --2- 172.21.15.28:0/1081682096 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0db6c90 0x558dd0db2a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.183+0000 7f0255639740 1 -- 172.21.15.28:0/1081682096 >> 172.21.15.28:0/1081682096 conn(0x558dd0db31a0 msgr2=0x558dd0dccb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.184+0000 7f0255639740 1 -- 172.21.15.28:0/1081682096 shutdown_connections 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.184+0000 7f0255639740 1 -- 172.21.15.28:0/1081682096 wait complete. 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.184+0000 7f0255639740 1 -- start start 2024-05-20T20:12:01.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.184+0000 7f0255639740 1 Processor -- start 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f0255639740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db6c90 0x558dd0dc6d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f0255639740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dc7240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f0255639740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0dc1e70 0x558dd0dc7780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f0255639740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f024c1c71b0 con 0x558dd0db6c90 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f0255639740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f024c1c6ae0 con 0x558dd0db75b0 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f0255639740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f024c1a51c0 con 0x558dd0dc1e70 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f0255639740 10 client.0 _ll_register_callbacks cb 0x7f0244009f20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f024b400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dc7240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.185+0000 7f024aa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db6c90 0x558dd0dc6d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024b400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dc7240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37156/0 (socket says 172.21.15.28:37156) 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024b400640 1 -- 172.21.15.28:0/2420314855 learned_addr learned my addr 172.21.15.28:0/2420314855 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024aa00640 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0dc1e70 0x558dd0dc7780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.187 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024b400640 1 -- 172.21.15.28:0/2420314855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0dc1e70 msgr2=0x558dd0dc7780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:01.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024b400640 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0dc1e70 0x558dd0dc7780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024b400640 1 -- 172.21.15.28:0/2420314855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db6c90 msgr2=0x558dd0dc6d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:01.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024b400640 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db6c90 0x558dd0dc6d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024b400640 1 -- 172.21.15.28:0/2420314855 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558dd0db2fd0 con 0x558dd0db75b0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.186+0000 7f024aa00640 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db6c90 0x558dd0dc6d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.187+0000 7f024b400640 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dc7240 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f0244029040 tx=0x7f024401cb50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=392221587aeeaafc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.187+0000 7f0235600640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.187+0000 7f0235600640 1 -- 172.21.15.28:0/2420314855 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0244034070 con 0x558dd0db75b0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.187+0000 7f0235600640 1 -- 172.21.15.28:0/2420314855 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(81..81 src has 1..81) ==== 8498+0+0 (secure 0 0 0) 0x7f0244014560 con 0x558dd0db75b0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.187+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f024401a070 con 0x558dd0db75b0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.188+0000 7f0235600640 1 -- 172.21.15.28:0/2420314855 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0244007000 con 0x558dd0db75b0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.188+0000 7f0255639740 10 client.26771 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.188+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558dd0e3b9e0 con 0x558dd0db75b0 2024-05-20T20:12:01.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.188+0000 7f0235600640 1 -- 172.21.15.28:0/2420314855 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f024401a070 con 0x558dd0db75b0 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.188+0000 7f0235600640 1 -- 172.21.15.28:0/2420314855 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0244026070 con 0x558dd0db75b0 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.188+0000 7f0255639740 10 client.26771 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.188+0000 7f0255639740 10 client.26771 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.189+0000 7f0255639740 20 client.26771 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.189+0000 7f0255639740 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0e3c070 0x558dd0e3e490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.189+0000 7f0255639740 4 client.26771 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.189+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x558dd0d232c0 con 0x558dd0e3c070 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.189+0000 7f024aa00640 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0e3c070 0x558dd0e3e490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.190+0000 7f024aa00640 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0e3c070 0x558dd0e3e490 crc :-1 s=READY pgs=154 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=be7a8879c344b1d7 server_cookie=98e087e51673df8d in_seq=0 out_seq=0 2024-05-20T20:12:01.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.190+0000 7f0235600640 10 client.26771 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:01.201 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:12:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.209+0000 7f0235600640 1 -- 172.21.15.28:0/2420314855 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x558dd0d232c0 con 0x558dd0e3c070 2024-05-20T20:12:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.209+0000 7f0235600640 10 client.26771 handle_command_reply: tid=0 2024-05-20T20:12:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.209+0000 7f0255639740 1 client.26771 shutdown 2024-05-20T20:12:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.209+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 msgr2=0x558dd0dc7240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.209+0000 7f0255639740 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dc7240 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f0244029040 tx=0x7f024401cb50 comp rx=0 tx=0).stop 2024-05-20T20:12:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 shutdown_connections 2024-05-20T20:12:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558dd0e3c070 0x558dd0e3e490 crc :-1 s=READY pgs=154 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558dd0db75b0 0x558dd0dc7240 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558dd0dc1e70 0x558dd0dc7780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 --2- 172.21.15.28:0/2420314855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558dd0db6c90 0x558dd0dc6d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 >> 172.21.15.28:0/2420314855 conn(0x558dd0db31a0 msgr2=0x558dd0db2310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0235600640 0 client.26771 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 shutdown_connections 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 1 -- 172.21.15.28:0/2420314855 wait complete. 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.210+0000 7f0255639740 20 client.26771 trim_cache size 0 max 16384 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:12:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02300b2be0 msgr2=0x7f02300b50a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02300b2be0 0x7f02300b50a0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f023802bfd0 tx=0x7f023802b0a0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c10f540 msgr2=0x7f024c1ac930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c10f540 0x7f024c1ac930 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f023800add0 tx=0x7f023802bc10 comp rx=0 tx=0).stop 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 shutdown_connections 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f024c10a410 0x7f024c1a8600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f02300b2be0 0x7f02300b50a0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f024c10f540 0x7f024c1ac930 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 --2- 172.21.15.28:0/2299326521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f024c109a50 0x7f024c1a80c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:01.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.212+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 >> 172.21.15.28:0/2299326521 conn(0x7f024c06db20 msgr2=0x7f024c073100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:01.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.213+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 shutdown_connections 2024-05-20T20:12:01.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:01.213+0000 7f0252800640 1 -- 172.21.15.28:0/2299326521 wait complete. 2024-05-20T20:12:01.237 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:00 smithi154 ceph-mon[36314]: pgmap v194: 97 pgs: 97 active+clean; 36 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 319 MiB/s wr, 1.29k op/s 2024-05-20T20:12:01.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:00 smithi033 ceph-mon[36179]: pgmap v194: 97 pgs: 97 active+clean; 36 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 319 MiB/s wr, 1.29k op/s 2024-05-20T20:12:03.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:02 smithi028 ceph-mon[31178]: pgmap v195: 97 pgs: 97 active+clean; 36 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 283 MiB/s wr, 1.22k op/s 2024-05-20T20:12:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:02 smithi154 ceph-mon[36314]: pgmap v195: 97 pgs: 97 active+clean; 36 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 283 MiB/s wr, 1.22k op/s 2024-05-20T20:12:03.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:02 smithi033 ceph-mon[36179]: pgmap v195: 97 pgs: 97 active+clean; 36 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 283 MiB/s wr, 1.22k op/s 2024-05-20T20:12:03.707 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:05.279 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:04 smithi028 ceph-mon[31178]: pgmap v196: 97 pgs: 97 active+clean; 37 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 326 MiB/s wr, 1.46k op/s 2024-05-20T20:12:05.279 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:04 smithi028 ceph-mon[31178]: osdmap e82: 12 total, 12 up, 12 in 2024-05-20T20:12:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:04 smithi154 ceph-mon[36314]: pgmap v196: 97 pgs: 97 active+clean; 37 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 326 MiB/s wr, 1.46k op/s 2024-05-20T20:12:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:04 smithi154 ceph-mon[36314]: osdmap e82: 12 total, 12 up, 12 in 2024-05-20T20:12:05.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:04 smithi033 ceph-mon[36179]: pgmap v196: 97 pgs: 97 active+clean; 37 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 326 MiB/s wr, 1.46k op/s 2024-05-20T20:12:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:04 smithi033 ceph-mon[36179]: osdmap e82: 12 total, 12 up, 12 in 2024-05-20T20:12:05.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 -- 172.21.15.28:0/2321178525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4075970 msgr2=0x7fcbc4075d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:05.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/2321178525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4075970 0x7fcbc4075d50 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7fcbb800b0a0 tx=0x7fcbb800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 -- 172.21.15.28:0/2321178525 shutdown_connections 2024-05-20T20:12:05.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/2321178525 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcbc410fcb0 0x7fcbc4112120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/2321178525 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4075970 0x7fcbc4075d50 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/2321178525 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbc4076290 0x7fcbc410f770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 -- 172.21.15.28:0/2321178525 >> 172.21.15.28:0/2321178525 conn(0x7fcbc40fefc0 msgr2=0x7fcbc41013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 -- 172.21.15.28:0/2321178525 shutdown_connections 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.279+0000 7fcbcaa00700 1 -- 172.21.15.28:0/2321178525 wait complete. 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 Processor -- start 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 -- start start 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbc4075970 0x7fcbc419bd70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4076290 0x7fcbc419c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcbc410fcb0 0x7fcbc41a0640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fcbc4115880 con 0x7fcbc4076290 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fcbc4115700 con 0x7fcbc410fcb0 2024-05-20T20:12:05.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.280+0000 7fcbcaa00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fcbc4115a00 con 0x7fcbc4075970 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4076290 0x7fcbc419c2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4076290 0x7fcbc419c2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34268/0 (socket says 172.21.15.28:34268) 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 -- 172.21.15.28:0/3744366123 learned_addr learned my addr 172.21.15.28:0/3744366123 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc9600700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbc4075970 0x7fcbc419bd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbca000700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcbc410fcb0 0x7fcbc41a0640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 -- 172.21.15.28:0/3744366123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbc4075970 msgr2=0x7fcbc419bd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbc4075970 0x7fcbc419bd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 -- 172.21.15.28:0/3744366123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcbc410fcb0 msgr2=0x7fcbc41a0640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcbc410fcb0 0x7fcbc41a0640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc8c00700 1 -- 172.21.15.28:0/3744366123 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcbc41a0d80 con 0x7fcbc4076290 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.281+0000 7fcbc9600700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbc4075970 0x7fcbc419bd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.282+0000 7fcbc8c00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4076290 0x7fcbc419c2b0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fcbb000e9a0 tx=0x7fcbb000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:05.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.282+0000 7fcbc2000700 1 -- 172.21.15.28:0/3744366123 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcbb000ca10 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.282+0000 7fcbc2000700 1 -- 172.21.15.28:0/3744366123 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcbb000cbb0 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.282+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcbc41a1070 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.282+0000 7fcbc2000700 1 -- 172.21.15.28:0/3744366123 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcbb0018680 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.282+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcbc41a88f0 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.283+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcbc406a230 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.286+0000 7fcbc2000700 1 -- 172.21.15.28:0/3744366123 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fcbb0010430 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.287+0000 7fcbc2000700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcbac0b2a70 0x7fcbac0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.287+0000 7fcbc2000700 1 -- 172.21.15.28:0/3744366123 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(82..82 src has 1..82) ==== 8586+0+0 (secure 0 0 0) 0x7fcbb00e2af0 con 0x7fcbc4076290 2024-05-20T20:12:05.745 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.287+0000 7fcbc9600700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcbac0b2a70 0x7fcbac0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.288+0000 7fcbc9600700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcbac0b2a70 0x7fcbac0b4f30 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7fcbb80062a0 tx=0x7fcbb8006210 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.290+0000 7fcbc2000700 1 -- 172.21.15.28:0/3744366123 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fcbb00aa430 con 0x7fcbc4076290 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.571+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fcbc419d0d0 con 0x7fcbc4076290 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.572+0000 7fcbc2000700 1 -- 172.21.15.28:0/3744366123 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fcbb00a9d30 con 0x7fcbc4076290 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcbac0b2a70 msgr2=0x7fcbac0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcbac0b2a70 0x7fcbac0b4f30 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7fcbb80062a0 tx=0x7fcbb8006210 comp rx=0 tx=0).stop 2024-05-20T20:12:05.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4076290 msgr2=0x7fcbc419c2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4076290 0x7fcbc419c2b0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7fcbb000e9a0 tx=0x7fcbb000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 shutdown_connections 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcbac0b2a70 0x7fcbac0b4f30 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcbc410fcb0 0x7fcbc41a0640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbc4075970 0x7fcbc419bd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 --2- 172.21.15.28:0/3744366123 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcbc4076290 0x7fcbc419c2b0 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 >> 172.21.15.28:0/3744366123 conn(0x7fcbc40fefc0 msgr2=0x7fcbc40ffcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 shutdown_connections 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:05.576+0000 7fcbcaa00700 1 -- 172.21.15.28:0/3744366123 wait complete. 2024-05-20T20:12:05.747 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:12:06.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:05 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3744366123' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:06.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:05 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3744366123' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:06.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:05 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3744366123' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:06.612 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id b24c10ea --release --await' 2024-05-20T20:12:06.612 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id b24c10ea --release --await 2024-05-20T20:12:07.203 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:06 smithi028 ceph-mon[31178]: pgmap v198: 97 pgs: 97 active+clean; 37 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 307 MiB/s wr, 1.43k op/s 2024-05-20T20:12:07.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.234+0000 7f3bd8000700 1 -- 172.21.15.28:0/3232502373 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd01095b0 msgr2=0x7f3bd0109990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.234+0000 7f3bd8000700 1 --2- 172.21.15.28:0/3232502373 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd01095b0 0x7f3bd0109990 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7f3bc4009990 tx=0x7f3bc4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.234+0000 7f3bd8000700 1 -- 172.21.15.28:0/3232502373 shutdown_connections 2024-05-20T20:12:07.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.234+0000 7f3bd8000700 1 --2- 172.21.15.28:0/3232502373 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3bd006bbe0 0x7f3bd01103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.234+0000 7f3bd8000700 1 --2- 172.21.15.28:0/3232502373 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd01095b0 0x7f3bd0109990 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.234+0000 7f3bd8000700 1 --2- 172.21.15.28:0/3232502373 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3bd006b220 0x7f3bd006b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.235+0000 7f3bd8000700 1 -- 172.21.15.28:0/3232502373 >> 172.21.15.28:0/3232502373 conn(0x7f3bd00793f0 msgr2=0x7f3bd0079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:07.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.235+0000 7f3bd8000700 1 -- 172.21.15.28:0/3232502373 shutdown_connections 2024-05-20T20:12:07.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.235+0000 7f3bd8000700 1 -- 172.21.15.28:0/3232502373 wait complete. 2024-05-20T20:12:07.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.236+0000 7f3bd8000700 1 Processor -- start 2024-05-20T20:12:07.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.236+0000 7f3bd8000700 1 -- start start 2024-05-20T20:12:07.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd8000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3bd006b220 0x7f3bd0102510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd8000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd006bbe0 0x7f3bd0102a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd8000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3bd01095b0 0x7f3bd0102f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd8000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd0113720 con 0x7f3bd006bbe0 2024-05-20T20:12:07.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd8000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01135a0 con 0x7f3bd006b220 2024-05-20T20:12:07.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd8000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01138a0 con 0x7f3bd01095b0 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd4c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd006bbe0 0x7f3bd0102a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd4c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd006bbe0 0x7f3bd0102a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42156/0 (socket says 172.21.15.28:42156) 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3bd01095b0 0x7f3bd0102f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd4c00700 1 -- 172.21.15.28:0/1675162898 learned_addr learned my addr 172.21.15.28:0/1675162898 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.237+0000 7f3bd5600700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3bd006b220 0x7f3bd0102510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bd4c00700 1 -- 172.21.15.28:0/1675162898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3bd01095b0 msgr2=0x7f3bd0102f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3bd01095b0 0x7f3bd0102f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bd4c00700 1 -- 172.21.15.28:0/1675162898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3bd006b220 msgr2=0x7f3bd0102510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3bd006b220 0x7f3bd0102510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bd4c00700 1 -- 172.21.15.28:0/1675162898 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3bd01a81a0 con 0x7f3bd006bbe0 2024-05-20T20:12:07.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bd6000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3bd01095b0 0x7f3bd0102f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:07.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd006bbe0 0x7f3bd0102a50 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f3bbc00e9a0 tx=0x7f3bbc00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:07.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.238+0000 7f3bce000700 1 -- 172.21.15.28:0/1675162898 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bbc00ca10 con 0x7f3bd006bbe0 2024-05-20T20:12:07.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.239+0000 7f3bce000700 1 -- 172.21.15.28:0/1675162898 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3bbc00cbb0 con 0x7f3bd006bbe0 2024-05-20T20:12:07.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.239+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3bd01a8490 con 0x7f3bd006bbe0 2024-05-20T20:12:07.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.239+0000 7f3bce000700 1 -- 172.21.15.28:0/1675162898 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bbc018680 con 0x7f3bd006bbe0 2024-05-20T20:12:07.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.239+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3bd01a8a20 con 0x7f3bd006bbe0 2024-05-20T20:12:07.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.242+0000 7f3bce000700 1 -- 172.21.15.28:0/1675162898 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3bbc010430 con 0x7f3bd006bbe0 2024-05-20T20:12:07.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.243+0000 7f3be28d5b80 1 Processor -- start 2024-05-20T20:12:07.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.244+0000 7f3be28d5b80 1 -- start start 2024-05-20T20:12:07.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.244+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd539f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.244+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd4db20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.244+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd4e5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.244+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55655bcabe40 con 0x55655bd53f30 2024-05-20T20:12:07.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.244+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55655bc9b9e0 con 0x55655bd4e140 2024-05-20T20:12:07.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.244+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55655bc9fa20 con 0x55655bd53610 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd539f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd5600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd4db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd539f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37936/0 (socket says 172.21.15.28:37936) 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 -- 172.21.15.28:0/787657530 learned_addr learned my addr 172.21.15.28:0/787657530 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd4e5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 -- 172.21.15.28:0/787657530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 msgr2=0x55655bd4e5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd4e5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 -- 172.21.15.28:0/787657530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 msgr2=0x55655bd4db20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd4db20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd6000700 1 -- 172.21.15.28:0/787657530 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55655bc64420 con 0x55655bd53610 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd4e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:07.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.245+0000 7f3bce000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3bb80b2a70 0x7f3bb80b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.246+0000 7f3bd6000700 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd539f0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8003eb0 tx=0x7f3bc8009920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e5a9c4434284c214 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:07.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.246+0000 7f3bce000700 1 -- 172.21.15.28:0/1675162898 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(82..82 src has 1..82) ==== 8586+0+0 (secure 0 0 0) 0x7f3bbc0e27b0 con 0x7f3bd006bbe0 2024-05-20T20:12:07.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.246+0000 7f3bd5600700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3bb80b2a70 0x7f3bb80b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.246+0000 7f3bc2a00700 1 -- 172.21.15.28:0/787657530 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bc8019020 con 0x55655bd53610 2024-05-20T20:12:07.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.246+0000 7f3bc2a00700 1 -- 172.21.15.28:0/787657530 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3bc80092d0 con 0x55655bd53610 2024-05-20T20:12:07.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.246+0000 7f3bc2a00700 1 -- 172.21.15.28:0/787657530 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bc800e870 con 0x55655bd53610 2024-05-20T20:12:07.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 -- 172.21.15.28:0/787657530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 msgr2=0x55655bd539f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd539f0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8003eb0 tx=0x7f3bc8009920 comp rx=0 tx=0).stop 2024-05-20T20:12:07.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3bd5600700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3bb80b2a70 0x7f3bb80b4f30 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f3bc4006010 tx=0x7f3bc400b260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:07.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 -- 172.21.15.28:0/787657530 shutdown_connections 2024-05-20T20:12:07.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd4e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd539f0 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/787657530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd4db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 -- 172.21.15.28:0/787657530 >> 172.21.15.28:0/787657530 conn(0x55655bc7bbe0 msgr2=0x55655bd48690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:07.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 -- 172.21.15.28:0/787657530 shutdown_connections 2024-05-20T20:12:07.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.247+0000 7f3be28d5b80 1 -- 172.21.15.28:0/787657530 wait complete. 2024-05-20T20:12:07.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.248+0000 7f3be28d5b80 1 -- start start 2024-05-20T20:12:07.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.248+0000 7f3be28d5b80 1 Processor -- start 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd4e140 0x55655bd59c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53610 0x55655bd56fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd57520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55655bba1fa0 con 0x55655bd53f30 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55655bc3c760 con 0x55655bd53610 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55655bd5a9e0 con 0x55655bd4e140 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3be28d5b80 10 client.0 _ll_register_callbacks cb 0x55655bc45c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3bd5600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd4e140 0x55655bd59c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3bd5600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd4e140 0x55655bd59c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37946/0 (socket says 172.21.15.28:37946) 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3bd5600700 1 -- 172.21.15.28:0/2628054865 learned_addr learned my addr 172.21.15.28:0/2628054865 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:07.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53610 0x55655bd56fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.249+0000 7f3bd6000700 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd57520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.250+0000 7f3bd5600700 1 -- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53610 msgr2=0x55655bd56fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.250+0000 7f3bd5600700 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53610 0x55655bd56fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.250+0000 7f3bd5600700 1 -- 172.21.15.28:0/2628054865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 msgr2=0x55655bd57520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.250+0000 7f3bd5600700 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd57520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.250+0000 7f3bd5600700 1 -- 172.21.15.28:0/2628054865 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55655bbeb8d0 con 0x55655bd4e140 2024-05-20T20:12:07.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.250+0000 7f3bd5600700 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd4e140 0x55655bd59c40 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f3bc400baa0 tx=0x7f3bc4009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f890d16b750a5cad server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.250+0000 7f3bb7400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.251+0000 7f3bb7400700 1 -- 172.21.15.28:0/2628054865 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bc4034060 con 0x55655bd4e140 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.251+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3bc80092d0 con 0x55655bd4e140 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.251+0000 7f3bb7400700 1 -- 172.21.15.28:0/2628054865 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(82..82 src has 1..82) ==== 8586+0+0 (secure 0 0 0) 0x7f3bc4003c90 con 0x55655bd4e140 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.251+0000 7f3bb7400700 1 -- 172.21.15.28:0/2628054865 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3bc402b040 con 0x55655bd4e140 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.251+0000 7f3be28d5b80 10 client.26698 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.251+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55655bc6c060 con 0x55655bd4e140 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3bb7400700 1 -- 172.21.15.28:0/2628054865 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3bc80092d0 con 0x55655bd4e140 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3bb7400700 1 -- 172.21.15.28:0/2628054865 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3bc4024440 con 0x55655bd4e140 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3be28d5b80 10 client.26698 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3be28d5b80 10 client.26698 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3be28d5b80 20 client.26698 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:07.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bd67490 0x55655bd69880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3be28d5b80 4 client.26698 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:07.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55655bcadd90 con 0x55655bd67490 2024-05-20T20:12:07.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.252+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bd67490 0x55655bd69880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.253+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bd67490 0x55655bd69880 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=87ed1f0fa1723ff3 server_cookie=b34fb5bd9f0d4665 in_seq=0 out_seq=0 2024-05-20T20:12:07.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.253+0000 7f3bb7400700 10 client.26698 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:12:07.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.254+0000 7f3bb7400700 1 -- 172.21.15.28:0/2628054865 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55655bcadd90 con 0x55655bd67490 2024-05-20T20:12:07.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.254+0000 7f3bb7400700 10 client.26698 handle_command_reply: tid=0 2024-05-20T20:12:07.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.254+0000 7f3be28d5b80 1 client.26698 shutdown 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.254+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd4e140 msgr2=0x55655bd59c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.254+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd4e140 0x55655bd59c40 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f3bc400baa0 tx=0x7f3bc4009310 comp rx=0 tx=0).stop 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 shutdown_connections 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bd67490 0x55655bd69880 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd57520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd4e140 0x55655bd59c40 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/2628054865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53610 0x55655bd56fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 >> 172.21.15.28:0/2628054865 conn(0x55655bc7bbe0 msgr2=0x55655bd47d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3bb7400700 0 client.26698 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 shutdown_connections 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 1 -- 172.21.15.28:0/2628054865 wait complete. 2024-05-20T20:12:07.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.255+0000 7f3be28d5b80 20 client.26698 trim_cache size 0 max 16384 2024-05-20T20:12:07.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 Processor -- start 2024-05-20T20:12:07.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 -- start start 2024-05-20T20:12:07.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd4e140 0x55655bd539f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd54ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53f30 0x55655bd57430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01b00b0 con 0x55655bd4e140 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01b1bf0 con 0x55655bd53f30 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.273+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01b0780 con 0x55655bd53610 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.274+0000 7f3bd6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53f30 0x55655bd57430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.274+0000 7f3bd4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd54ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.274+0000 7f3bd6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53f30 0x55655bd57430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49414/0 (socket says 172.21.15.28:49414) 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.274+0000 7f3bd4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd54ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37962/0 (socket says 172.21.15.28:37962) 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.274+0000 7f3bd6000700 1 -- 172.21.15.28:0/1578228698 learned_addr learned my addr 172.21.15.28:0/1578228698 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.274+0000 7f3bd5600700 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd4e140 0x55655bd539f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bd4c00700 1 -- 172.21.15.28:0/1578228698 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53f30 msgr2=0x55655bd57430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53f30 0x55655bd57430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bd4c00700 1 -- 172.21.15.28:0/1578228698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd4e140 msgr2=0x55655bd539f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd4e140 0x55655bd539f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bd4c00700 1 -- 172.21.15.28:0/1578228698 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55655bd55590 con 0x55655bd53610 2024-05-20T20:12:07.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bd5600700 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd4e140 0x55655bd539f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:07.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd54ba0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f3bbc0dfe60 tx=0x7f3bbc0cc040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=49eb9b136e77208f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:07.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bc2000700 1 -- 172.21.15.28:0/1578228698 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bbc0a9500 con 0x55655bd53610 2024-05-20T20:12:07.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bc2000700 1 -- 172.21.15.28:0/1578228698 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3bbc0aa0f0 con 0x55655bd53610 2024-05-20T20:12:07.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.275+0000 7f3bc2000700 1 -- 172.21.15.28:0/1578228698 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bbc0b23c0 con 0x55655bd53610 2024-05-20T20:12:07.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 -- 172.21.15.28:0/1578228698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 msgr2=0x55655bd54ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd54ba0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f3bbc0dfe60 tx=0x7f3bbc0cc040 comp rx=0 tx=0).stop 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 -- 172.21.15.28:0/1578228698 shutdown_connections 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd53f30 0x55655bd57430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd4e140 0x55655bd539f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/1578228698 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd54ba0 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 -- 172.21.15.28:0/1578228698 >> 172.21.15.28:0/1578228698 conn(0x55655bc7bbe0 msgr2=0x55655bd58690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.276+0000 7f3be28d5b80 1 -- 172.21.15.28:0/1578228698 shutdown_connections 2024-05-20T20:12:07.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.277+0000 7f3be28d5b80 1 -- 172.21.15.28:0/1578228698 wait complete. 2024-05-20T20:12:07.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.277+0000 7f3be28d5b80 1 -- start start 2024-05-20T20:12:07.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.277+0000 7f3be28d5b80 1 Processor -- start 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd5fe50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd60390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3be28d5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd6e560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01b1520 con 0x55655bd53f30 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01b0e50 con 0x55655bd4e140 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3be28d5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3bd01ae070 con 0x55655bd53610 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3be28d5b80 10 client.0 _ll_register_callbacks cb 0x55655bc45c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:07.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3bd5600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd5fe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3bd6000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd6e560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3bd5600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd5fe50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49426/0 (socket says 172.21.15.28:49426) 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3bd5600700 1 -- 172.21.15.28:0/726981527 learned_addr learned my addr 172.21.15.28:0/726981527 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.278+0000 7f3bd4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd60390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bd4c00700 1 -- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 msgr2=0x55655bd5fe50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd5fe50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bd4c00700 1 -- 172.21.15.28:0/726981527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 msgr2=0x55655bd6e560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd6e560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bd4c00700 1 -- 172.21.15.28:0/726981527 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55655bd6edd0 con 0x55655bd53610 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bd5600700 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd5fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bd4c00700 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd60390 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f3bbc0aa2a0 tx=0x7f3bbc0e4c70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=85a1b24c5efcc295 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bb7400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bb7400700 1 -- 172.21.15.28:0/726981527 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3bbc0a99f0 con 0x55655bd53610 2024-05-20T20:12:07.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3bbc0aa0f0 con 0x55655bd53610 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.279+0000 7f3bb7400700 1 -- 172.21.15.28:0/726981527 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(82..82 src has 1..82) ==== 8586+0+0 (secure 0 0 0) 0x7f3bbc0a9e60 con 0x55655bd53610 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3bb7400700 1 -- 172.21.15.28:0/726981527 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3bbc0b2850 con 0x55655bd53610 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 10 client.26704 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55655bdd1b60 con 0x55655bd53610 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3bb7400700 1 -- 172.21.15.28:0/726981527 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3bbc0aa0f0 con 0x55655bd53610 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3bb7400700 1 -- 172.21.15.28:0/726981527 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3bbc0e4ce0 con 0x55655bd53610 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 10 client.26704 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 10 client.26704 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 20 client.26704 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bdd2160 0x55655bdd4550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 4 client.26704 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55655bca7570 con 0x55655bdd2160 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.280+0000 7f3bd5600700 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bdd2160 0x55655bdd4550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:07.284 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.281+0000 7f3bd5600700 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bdd2160 0x55655bdd4550 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=d1771afa7f3f6342 server_cookie=58f91ce531cd2cb4 in_seq=0 out_seq=0 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.281+0000 7f3bb7400700 10 client.26704 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.281+0000 7f3bb7400700 1 -- 172.21.15.28:0/726981527 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55655bca7570 con 0x55655bdd2160 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.281+0000 7f3bb7400700 10 client.26704 handle_command_reply: tid=0 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.281+0000 7f3be28d5b80 1 client.26704 shutdown 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.281+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 msgr2=0x55655bd60390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.281+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd60390 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f3bbc0aa2a0 tx=0x7f3bbc0e4c70 comp rx=0 tx=0).stop 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 shutdown_connections 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55655bdd2160 0x55655bdd4550 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55655bd53f30 0x55655bd6e560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55655bd4e140 0x55655bd5fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 --2- 172.21.15.28:0/726981527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55655bd53610 0x55655bd60390 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3bb7400700 0 client.26704 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:12:07.285 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 >> 172.21.15.28:0/726981527 conn(0x55655bc7bbe0 msgr2=0x55655bd561a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:07.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 shutdown_connections 2024-05-20T20:12:07.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 1 -- 172.21.15.28:0/726981527 wait complete. 2024-05-20T20:12:07.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.282+0000 7f3be28d5b80 20 client.26704 trim_cache size 0 max 16384 2024-05-20T20:12:07.286 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3bb80b2a70 msgr2=0x7f3bb80b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3bb80b2a70 0x7f3bb80b4f30 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f3bc4006010 tx=0x7f3bc400b260 comp rx=0 tx=0).stop 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd006bbe0 msgr2=0x7f3bd0102a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd006bbe0 0x7f3bd0102a50 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f3bbc00e9a0 tx=0x7f3bbc00ecb0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 shutdown_connections 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3bd01095b0 0x7f3bd0102f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3bb80b2a70 0x7f3bb80b4f30 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3bd006b220 0x7f3bd0102510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 --2- 172.21.15.28:0/1675162898 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3bd006bbe0 0x7f3bd0102a50 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 >> 172.21.15.28:0/1675162898 conn(0x7f3bd00793f0 msgr2=0x7f3bd0101120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:07.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 shutdown_connections 2024-05-20T20:12:07.288 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:07.283+0000 7f3bd8000700 1 -- 172.21.15.28:0/1675162898 wait complete. 2024-05-20T20:12:07.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:06 smithi154 ceph-mon[36314]: pgmap v198: 97 pgs: 97 active+clean; 37 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 307 MiB/s wr, 1.43k op/s 2024-05-20T20:12:07.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:06 smithi033 ceph-mon[36179]: pgmap v198: 97 pgs: 97 active+clean; 37 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 307 MiB/s wr, 1.43k op/s 2024-05-20T20:12:07.554 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:08.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:07 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:12:08.153 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:12:08.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:07 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:12:08.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:07 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:12:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:08 smithi028 ceph-mon[31178]: pgmap v199: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 304 MiB/s wr, 1.43k op/s 2024-05-20T20:12:09.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:08 smithi154 ceph-mon[36314]: pgmap v199: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 304 MiB/s wr, 1.43k op/s 2024-05-20T20:12:09.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:08 smithi033 ceph-mon[36179]: pgmap v199: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 304 MiB/s wr, 1.43k op/s 2024-05-20T20:12:09.742 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:10.155 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:09 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:12:10.155 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:09 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:12:10.155 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:09 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:12:10.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:09 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:12:10.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:09 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:12:10.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:09 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:12:10.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:09 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:12:10.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:09 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:12:10.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:09 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:12:11.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:10 smithi028 ceph-mon[31178]: pgmap v200: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 222 MiB/s wr, 1.11k op/s 2024-05-20T20:12:11.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:10 smithi028 ceph-mon[31178]: osdmap e83: 12 total, 12 up, 12 in 2024-05-20T20:12:11.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 -- 172.21.15.28:0/3071724070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 msgr2=0x7f510010a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 --2- 172.21.15.28:0/3071724070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f510010a1b0 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f50f00099c0 tx=0x7f50f0009cd0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.109 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 -- 172.21.15.28:0/3071724070 shutdown_connections 2024-05-20T20:12:11.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 --2- 172.21.15.28:0/3071724070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f510010eeb0 0x7f5100116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 --2- 172.21.15.28:0/3071724070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f510010a1b0 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 --2- 172.21.15.28:0/3071724070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f510010a780 0x7f510010e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 -- 172.21.15.28:0/3071724070 >> 172.21.15.28:0/3071724070 conn(0x7f5100076d10 msgr2=0x7f5100108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:11.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 -- 172.21.15.28:0/3071724070 shutdown_connections 2024-05-20T20:12:11.110 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.853+0000 7f5107000700 1 -- 172.21.15.28:0/3071724070 wait complete. 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 Processor -- start 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 -- start start 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f51001a0290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f510010a780 0x7f51001a07d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f510010eeb0 0x7f51001a4b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5100119da0 con 0x7f5100109dd0 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5100119c20 con 0x7f510010a780 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.854+0000 7f5107000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5100119f20 con 0x7f510010eeb0 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f51001a0290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f51001a0290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42216/0 (socket says 172.21.15.28:42216) 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 -- 172.21.15.28:0/156673676 learned_addr learned my addr 172.21.15.28:0/156673676 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f5105000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f510010eeb0 0x7f51001a4b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ff400700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f510010a780 0x7f51001a07d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 -- 172.21.15.28:0/156673676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f510010eeb0 msgr2=0x7f51001a4b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f510010eeb0 0x7f51001a4b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 -- 172.21.15.28:0/156673676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f510010a780 msgr2=0x7f51001a07d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f510010a780 0x7f51001a07d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 -- 172.21.15.28:0/156673676 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51001a52a0 con 0x7f5100109dd0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ff400700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f510010a780 0x7f51001a07d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.855+0000 7f50ffe00700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f51001a0290 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f50f0006010 tx=0x7f50f00055f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.856+0000 7f50fcc00700 1 -- 172.21.15.28:0/156673676 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f0026070 con 0x7f5100109dd0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.856+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f51001a5530 con 0x7f5100109dd0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.856+0000 7f50fcc00700 1 -- 172.21.15.28:0/156673676 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f50f000f460 con 0x7f5100109dd0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.856+0000 7f50fcc00700 1 -- 172.21.15.28:0/156673676 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f0009e60 con 0x7f5100109dd0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.856+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f51001ace10 con 0x7f5100109dd0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.859+0000 7f50fcc00700 1 -- 172.21.15.28:0/156673676 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f50f0029070 con 0x7f5100109dd0 2024-05-20T20:12:11.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.860+0000 7f51118d2b80 1 Processor -- start 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.860+0000 7f51118d2b80 1 -- start start 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.860+0000 7f50fcc00700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50e40b29f0 0x7f50e40b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af3077aff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077b530 0x55af30774fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30775a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f51118d2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55af306d2f40 con 0x55af3077b530 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f51118d2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55af306c2ae0 con 0x55af307755f0 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f51118d2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55af306c6b20 con 0x55af3077ac10 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f50fcc00700 1 -- 172.21.15.28:0/156673676 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f50f00e2ee0 con 0x7f5100109dd0 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f50ff400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077b530 0x55af30774fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f5105000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50e40b29f0 0x7f50e40b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af3077aff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f50ff400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077b530 0x55af30774fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42230/0 (socket says 172.21.15.28:42230) 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f50ff400700 1 -- 172.21.15.28:0/1678549329 learned_addr learned my addr 172.21.15.28:0/1678549329 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.861+0000 7f5105000700 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30775a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50ffe00700 1 -- 172.21.15.28:0/1678549329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 msgr2=0x55af30775a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50ffe00700 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30775a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50ffe00700 1 -- 172.21.15.28:0/1678549329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077b530 msgr2=0x55af30774fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50ffe00700 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077b530 0x55af30774fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50ffe00700 1 -- 172.21.15.28:0/1678549329 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55af3068b520 con 0x55af3077ac10 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50ffe00700 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af3077aff0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f50f00e05c0 tx=0x7f50f00a99d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ab59779bd013e266 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f5105000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50e40b29f0 0x7f50e40b4eb0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f50f8007af0 tx=0x7f50f8009bf0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50f5600700 1 -- 172.21.15.28:0/1678549329 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f00aa1e0 con 0x55af3077ac10 2024-05-20T20:12:11.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50f5600700 1 -- 172.21.15.28:0/1678549329 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f50f00ae590 con 0x55af3077ac10 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.862+0000 7f50f5600700 1 -- 172.21.15.28:0/1678549329 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f00ae160 con 0x55af3077ac10 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 -- 172.21.15.28:0/1678549329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 msgr2=0x55af3077aff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af3077aff0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f50f00e05c0 tx=0x7f50f00a99d0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 -- 172.21.15.28:0/1678549329 shutdown_connections 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30775a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af3077aff0 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 --2- 172.21.15.28:0/1678549329 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077b530 0x55af30774fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 -- 172.21.15.28:0/1678549329 >> 172.21.15.28:0/1678549329 conn(0x55af306a2ce0 msgr2=0x55af3076ff00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 -- 172.21.15.28:0/1678549329 shutdown_connections 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.863+0000 7f51118d2b80 1 -- 172.21.15.28:0/1678549329 wait complete. 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.864+0000 7f51118d2b80 1 -- start start 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.864+0000 7f51118d2b80 1 Processor -- start 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af307755f0 0x55af307811e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.115 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077ac10 0x55af3077e580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077eac0 0x55af3077ef40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f51118d2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55af305c90a0 con 0x55af307755f0 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f51118d2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55af30663860 con 0x55af3077ac10 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f51118d2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55af30781d60 con 0x55af3077eac0 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f51118d2b80 10 client.0 _ll_register_callbacks cb 0x55af3066cd70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af307755f0 0x55af307811e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af307755f0 0x55af307811e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42242/0 (socket says 172.21.15.28:42242) 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 -- 172.21.15.28:0/304963475 learned_addr learned my addr 172.21.15.28:0/304963475 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f5105000700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077eac0 0x55af3077ef40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ff400700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077ac10 0x55af3077e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 -- 172.21.15.28:0/304963475 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077eac0 msgr2=0x55af3077ef40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077eac0 0x55af3077ef40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 -- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077ac10 msgr2=0x55af3077e580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077ac10 0x55af3077e580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f50ffe00700 1 -- 172.21.15.28:0/304963475 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55af306129d0 con 0x55af307755f0 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.865+0000 7f5105000700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077eac0 0x55af3077ef40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f50ffe00700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af307755f0 0x55af307811e0 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f50f00e0620 tx=0x7f50f00b2fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=730506d551f4f871 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f50ea000700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f50ea000700 1 -- 172.21.15.28:0/304963475 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f00ae360 con 0x55af307755f0 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f50ea000700 1 -- 172.21.15.28:0/304963475 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f50f00aa440 con 0x55af307755f0 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f50f00ae590 con 0x55af307755f0 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f50ea000700 1 -- 172.21.15.28:0/304963475 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f50f00aa640 con 0x55af307755f0 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f51118d2b80 10 client.16503 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:11.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55af30693160 con 0x55af307755f0 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.866+0000 7f50ea000700 1 -- 172.21.15.28:0/304963475 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f50f00ae590 con 0x55af307755f0 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f50ea000700 1 -- 172.21.15.28:0/304963475 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f50f00a5d50 con 0x55af307755f0 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f51118d2b80 10 client.16503 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f51118d2b80 10 client.16503 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f51118d2b80 20 client.16503 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f51118d2b80 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af3078ead0 0x55af30790ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f51118d2b80 4 client.16503 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55af306d4e90 con 0x55af3078ead0 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f50ff400700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af3078ead0 0x55af30790ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f50ff400700 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af3078ead0 0x55af30790ec0 crc :-1 s=READY pgs=155 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ad4b00c279147eab server_cookie=3c5aaf755a0099ac in_seq=0 out_seq=0 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.867+0000 7f50ea000700 10 client.16503 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.868+0000 7f50ea000700 1 -- 172.21.15.28:0/304963475 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55af306d4e90 con 0x55af3078ead0 2024-05-20T20:12:11.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.868+0000 7f50ea000700 10 client.16503 handle_command_reply: tid=0 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.868+0000 7f51118d2b80 1 client.16503 shutdown 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.868+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af307755f0 msgr2=0x55af307811e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.868+0000 7f51118d2b80 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af307755f0 0x55af307811e0 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f50f00e0620 tx=0x7f50f00b2fb0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 shutdown_connections 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af3078ead0 0x55af30790ec0 crc :-1 s=READY pgs=155 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077eac0 0x55af3077ef40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af307755f0 0x55af307811e0 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 --2- 172.21.15.28:0/304963475 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077ac10 0x55af3077e580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f50ea000700 0 client.16503 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 >> 172.21.15.28:0/304963475 conn(0x55af306a2ce0 msgr2=0x55af3076f430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 shutdown_connections 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 1 -- 172.21.15.28:0/304963475 wait complete. 2024-05-20T20:12:11.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.869+0000 7f51118d2b80 20 client.16503 trim_cache size 0 max 16384 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.890+0000 7f51118d2b80 1 Processor -- start 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f51118d2b80 1 -- start start 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af307755f0 0x55af30774fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077ac10 0x55af30780290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077eac0 0x55af307807d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f51118d2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f51001b43b0 con 0x55af3077ac10 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f51118d2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f51001b5ef0 con 0x55af3077eac0 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f51118d2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f51001b4a80 con 0x55af307755f0 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af307755f0 0x55af30774fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f50ff400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077ac10 0x55af30780290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af307755f0 0x55af30774fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38028/0 (socket says 172.21.15.28:38028) 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f50ffe00700 1 -- 172.21.15.28:0/491856951 learned_addr learned my addr 172.21.15.28:0/491856951 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f5105000700 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077eac0 0x55af307807d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f50ffe00700 1 -- 172.21.15.28:0/491856951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077eac0 msgr2=0x55af307807d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.891+0000 7f50ffe00700 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077eac0 0x55af307807d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50ffe00700 1 -- 172.21.15.28:0/491856951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077ac10 msgr2=0x55af30780290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50ffe00700 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077ac10 0x55af30780290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50ffe00700 1 -- 172.21.15.28:0/491856951 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55af30770440 con 0x55af307755f0 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50ff400700 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077ac10 0x55af30780290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50ffe00700 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af307755f0 0x55af30774fd0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f50f00a7450 tx=0x7f50f00aaa10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c6008b0d835375fe server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50f4c00700 1 -- 172.21.15.28:0/491856951 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f00ae590 con 0x55af307755f0 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50f4c00700 1 -- 172.21.15.28:0/491856951 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f50f00c4070 con 0x55af307755f0 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.892+0000 7f50f4c00700 1 -- 172.21.15.28:0/491856951 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f00b2b20 con 0x55af307755f0 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 -- 172.21.15.28:0/491856951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af307755f0 msgr2=0x55af30774fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af307755f0 0x55af30774fd0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f50f00a7450 tx=0x7f50f00aaa10 comp rx=0 tx=0).stop 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 -- 172.21.15.28:0/491856951 shutdown_connections 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af3077eac0 0x55af307807d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af307755f0 0x55af30774fd0 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 --2- 172.21.15.28:0/491856951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077ac10 0x55af30780290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 -- 172.21.15.28:0/491856951 >> 172.21.15.28:0/491856951 conn(0x55af306a2ce0 msgr2=0x55af3077fb80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 -- 172.21.15.28:0/491856951 shutdown_connections 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 -- 172.21.15.28:0/491856951 wait complete. 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.893+0000 7f51118d2b80 1 -- start start 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 1 Processor -- start 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30783f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af307844c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077eac0 0x55af30784a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f51001b5820 con 0x55af3077eac0 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f51001b5150 con 0x55af307755f0 2024-05-20T20:12:11.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f51001b2370 con 0x55af3077ac10 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f51118d2b80 10 client.0 _ll_register_callbacks cb 0x55af3066cd70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f50ff400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af307844c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f50ffe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30783f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f50ff400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af307844c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38042/0 (socket says 172.21.15.28:38042) 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f50ff400700 1 -- 172.21.15.28:0/692362692 learned_addr learned my addr 172.21.15.28:0/692362692 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.894+0000 7f5105000700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077eac0 0x55af30784a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ffe00700 1 -- 172.21.15.28:0/692362692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 msgr2=0x55af307844c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ffe00700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af307844c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ffe00700 1 -- 172.21.15.28:0/692362692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077eac0 msgr2=0x55af30784a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ffe00700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077eac0 0x55af30784a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ffe00700 1 -- 172.21.15.28:0/692362692 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55af307f96b0 con 0x55af307755f0 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ff400700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af307844c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:11.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f5105000700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077eac0 0x55af30784a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ffe00700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30783f80 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f50f0003760 tx=0x7f50f00b2e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9aa420502ffcf2f0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ea000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ea000700 1 -- 172.21.15.28:0/692362692 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f50f00aa1e0 con 0x55af307755f0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f50f00c4070 con 0x55af307755f0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ea000700 1 -- 172.21.15.28:0/692362692 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f50f00a9650 con 0x55af307755f0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f50ea000700 1 -- 172.21.15.28:0/692362692 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f50f00ae590 con 0x55af307755f0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f51118d2b80 10 client.26813 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.895+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55af307f96b0 con 0x55af307755f0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f50ea000700 1 -- 172.21.15.28:0/692362692 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f50f00c4070 con 0x55af307755f0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f50ea000700 1 -- 172.21.15.28:0/692362692 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f50f00a4900 con 0x55af307755f0 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f51118d2b80 10 client.26813 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f51118d2b80 10 client.26813 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f51118d2b80 20 client.26813 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f51118d2b80 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af307fac10 0x55af307fd000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:11.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f51118d2b80 4 client.26813 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "b24c10ea", "release": true, "await": true}] 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "b24c10ea", "release": true, "await": true}) -- 0x55af306ce670 con 0x55af307fac10 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.896+0000 7f50ff400700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af307fac10 0x55af307fd000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.897+0000 7f50ff400700 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af307fac10 0x55af307fd000 crc :-1 s=READY pgs=156 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c0e8c65360c9235b server_cookie=e950ee1adb406644 in_seq=0 out_seq=0 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.897+0000 7f50ea000700 10 client.26813 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.900+0000 7f50ea000700 1 -- 172.21.15.28:0/692362692 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x55af306ce670 con 0x55af307fac10 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.900+0000 7f50ea000700 10 client.26813 handle_command_reply: tid=0 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.900+0000 7f51118d2b80 1 client.26813 shutdown 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.901+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 msgr2=0x55af30783f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.901+0000 7f51118d2b80 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30783f80 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f50f0003760 tx=0x7f50f00b2e80 comp rx=0 tx=0).stop 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.901+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 shutdown_connections 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.901+0000 7f51118d2b80 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55af307fac10 0x55af307fd000 crc :-1 s=READY pgs=156 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f51118d2b80 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55af3077eac0 0x55af30784a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f51118d2b80 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55af307755f0 0x55af30783f80 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f51118d2b80 1 --2- 172.21.15.28:0/692362692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55af3077ac10 0x55af307844c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f50ea000700 0 client.26813 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 >> 172.21.15.28:0/692362692 conn(0x55af306a2ce0 msgr2=0x55af3076fcc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 shutdown_connections 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 16, 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:12:11.126 INFO:teuthology.orchestra.run.smithi028.stdout: "b24c10ea": { 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 16, 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 140.8, 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:12:11.127 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:12:11.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f51118d2b80 1 -- 172.21.15.28:0/692362692 wait complete. 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.902+0000 7f51118d2b80 20 client.26813 trim_cache size 0 max 16384 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.903+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50e40b29f0 msgr2=0x7f50e40b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.903+0000 7f5107000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50e40b29f0 0x7f50e40b4eb0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f50f8007af0 tx=0x7f50f8009bf0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.903+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 msgr2=0x7f51001a0290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.903+0000 7f5107000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f51001a0290 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f50f0006010 tx=0x7f50f00055f0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.903+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 shutdown_connections 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.903+0000 7f5107000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f510010eeb0 0x7f51001a4b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.904+0000 7f5107000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5100109dd0 0x7f51001a0290 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.904+0000 7f5107000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50e40b29f0 0x7f50e40b4eb0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.904+0000 7f5107000700 1 --2- 172.21.15.28:0/156673676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f510010a780 0x7f51001a07d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.904+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 >> 172.21.15.28:0/156673676 conn(0x7f5100076d10 msgr2=0x7f5100078900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.904+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 shutdown_connections 2024-05-20T20:12:11.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:10.904+0000 7f5107000700 1 -- 172.21.15.28:0/156673676 wait complete. 2024-05-20T20:12:11.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:10 smithi154 ceph-mon[36314]: pgmap v200: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 222 MiB/s wr, 1.11k op/s 2024-05-20T20:12:11.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:10 smithi154 ceph-mon[36314]: osdmap e83: 12 total, 12 up, 12 in 2024-05-20T20:12:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:10 smithi033 ceph-mon[36179]: pgmap v200: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 222 MiB/s wr, 1.11k op/s 2024-05-20T20:12:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:10 smithi033 ceph-mon[36179]: osdmap e83: 12 total, 12 up, 12 in 2024-05-20T20:12:12.192 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set 'b24c10ea', total seconds elapsed: 33.3 2024-05-20T20:12:13.190 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:12 smithi028 ceph-mon[31178]: pgmap v202: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 95 MiB/s wr, 527 op/s 2024-05-20T20:12:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:12 smithi154 ceph-mon[36314]: pgmap v202: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 95 MiB/s wr, 527 op/s 2024-05-20T20:12:13.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:12 smithi033 ceph-mon[36179]: pgmap v202: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 95 MiB/s wr, 527 op/s 2024-05-20T20:12:13.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.575+0000 7f06c2c00700 1 -- 172.21.15.28:0/4087180846 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc108520 msgr2=0x7f06bc108900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.575+0000 7f06c2c00700 1 --2- 172.21.15.28:0/4087180846 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc108520 0x7f06bc108900 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f06ac009990 tx=0x7f06ac009ca0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.576+0000 7f06c2c00700 1 -- 172.21.15.28:0/4087180846 shutdown_connections 2024-05-20T20:12:13.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.576+0000 7f06c2c00700 1 --2- 172.21.15.28:0/4087180846 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06bc102e00 0x7f06bc10f2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.576+0000 7f06c2c00700 1 --2- 172.21.15.28:0/4087180846 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc108520 0x7f06bc108900 secure :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f06ac009990 tx=0x7f06ac009ca0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.576+0000 7f06c2c00700 1 --2- 172.21.15.28:0/4087180846 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 0x7f06bc1028c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.576+0000 7f06c2c00700 1 -- 172.21.15.28:0/4087180846 >> 172.21.15.28:0/4087180846 conn(0x7f06bc0793f0 msgr2=0x7f06bc079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:13.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.576+0000 7f06c2c00700 1 -- 172.21.15.28:0/4087180846 shutdown_connections 2024-05-20T20:12:13.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.576+0000 7f06c2c00700 1 -- 172.21.15.28:0/4087180846 wait complete. 2024-05-20T20:12:13.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 Processor -- start 2024-05-20T20:12:13.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 -- start start 2024-05-20T20:12:13.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 0x7f06bc1a04d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06bc102e00 0x7f06bc1a0a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc1a4cf0 0x7f06bc1a5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc112620 con 0x7f06bc102e00 2024-05-20T20:12:13.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1124a0 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.579+0000 7f06c2c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1127a0 con 0x7f06bc102440 2024-05-20T20:12:13.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 0x7f06bc1a04d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06c0c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc1a4cf0 0x7f06bc1a5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 0x7f06bc1a04d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38058/0 (socket says 172.21.15.28:38058) 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06bbe00700 1 -- 172.21.15.28:0/2720675033 learned_addr learned my addr 172.21.15.28:0/2720675033 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06c0c00700 1 -- 172.21.15.28:0/2720675033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 msgr2=0x7f06bc1a04d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06c0c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 0x7f06bc1a04d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06c0c00700 1 -- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06bc102e00 msgr2=0x7f06bc1a0a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06c0c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06bc102e00 0x7f06bc1a0a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.580+0000 7f06c0c00700 1 -- 172.21.15.28:0/2720675033 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f06bc1a57e0 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.581+0000 7f06bbe00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 0x7f06bc1a04d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:13.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.581+0000 7f06c0c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc1a4cf0 0x7f06bc1a5170 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f06b400cc50 tx=0x7f06b400cf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:13.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.582+0000 7f06b8c00700 1 -- 172.21.15.28:0/2720675033 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06b4016020 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.582+0000 7f06b8c00700 1 -- 172.21.15.28:0/2720675033 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f06b4012420 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.582+0000 7f06b8c00700 1 -- 172.21.15.28:0/2720675033 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06b40115a0 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.582+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f06bc1a5ad0 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.582+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f06bc1ad200 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.588+0000 7f06b8c00700 1 -- 172.21.15.28:0/2720675033 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f06b4019020 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.591+0000 7f06b8c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06a00b2af0 0x7f06a00b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.591+0000 7f06cd490b80 1 Processor -- start 2024-05-20T20:12:13.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.591+0000 7f06b8c00700 1 -- 172.21.15.28:0/2720675033 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f06b40e2360 con 0x7f06bc1a4cf0 2024-05-20T20:12:13.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.591+0000 7f06cd490b80 1 -- start start 2024-05-20T20:12:13.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.592+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cb610 0x55de9a8cb9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.592+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8c5b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.592+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8c65c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.592+0000 7f06cd490b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55de9a823e40 con 0x55de9a8cb610 2024-05-20T20:12:13.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.592+0000 7f06cd490b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55de9a8139e0 con 0x55de9a8cbf30 2024-05-20T20:12:13.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.592+0000 7f06cd490b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55de9a817a20 con 0x55de9a8c6140 2024-05-20T20:12:13.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.593+0000 7f06bbe00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06a00b2af0 0x7f06a00b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.593+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cb610 0x55de9a8cb9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.593+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cb610 0x55de9a8cb9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42292/0 (socket says 172.21.15.28:42292) 2024-05-20T20:12:13.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.593+0000 7f06bbe00700 1 -- 172.21.15.28:0/3676151500 learned_addr learned my addr 172.21.15.28:0/3676151500 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:13.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.593+0000 7f06c0c00700 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8c65c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.594+0000 7f06bb400700 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8c5b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.594+0000 7f06bbe00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06a00b2af0 0x7f06a00b4fb0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f06ac00bd00 tx=0x7f06ac0036c0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.594+0000 7f06c0c00700 1 -- 172.21.15.28:0/3676151500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 msgr2=0x55de9a8c5b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.594+0000 7f06c0c00700 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8c5b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.594+0000 7f06c0c00700 1 -- 172.21.15.28:0/3676151500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cb610 msgr2=0x55de9a8cb9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.594+0000 7f06c0c00700 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cb610 0x55de9a8cb9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.594+0000 7f06c0c00700 1 -- 172.21.15.28:0/3676151500 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55de9a7dc420 con 0x55de9a8c6140 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.595+0000 7f06bb400700 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8c5b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.595+0000 7f06c0c00700 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8c65c0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f06b40dfb50 tx=0x7f06b40a95e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ec53afc7a327aab7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.595+0000 7f06b1600700 1 -- 172.21.15.28:0/3676151500 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06b40c3070 con 0x55de9a8c6140 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.595+0000 7f06b1600700 1 -- 172.21.15.28:0/3676151500 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f06b40ada90 con 0x55de9a8c6140 2024-05-20T20:12:13.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.595+0000 7f06b1600700 1 -- 172.21.15.28:0/3676151500 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06b40a9940 con 0x55de9a8c6140 2024-05-20T20:12:13.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.596+0000 7f06cd490b80 1 -- 172.21.15.28:0/3676151500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 msgr2=0x55de9a8c65c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.596+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8c65c0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f06b40dfb50 tx=0x7f06b40a95e0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.597+0000 7f06cd490b80 1 -- 172.21.15.28:0/3676151500 shutdown_connections 2024-05-20T20:12:13.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.597+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8c65c0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.597+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cb610 0x55de9a8cb9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.597+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3676151500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8c5b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.597+0000 7f06cd490b80 1 -- 172.21.15.28:0/3676151500 >> 172.21.15.28:0/3676151500 conn(0x55de9a7f3be0 msgr2=0x55de9a8c0690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:13.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.597+0000 7f06cd490b80 1 -- 172.21.15.28:0/3676151500 shutdown_connections 2024-05-20T20:12:13.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.598+0000 7f06cd490b80 1 -- 172.21.15.28:0/3676151500 wait complete. 2024-05-20T20:12:13.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.598+0000 7f06cd490b80 1 -- start start 2024-05-20T20:12:13.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.598+0000 7f06cd490b80 1 Processor -- start 2024-05-20T20:12:13.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.599+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d1d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.599+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8cf0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.599+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8cf5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.599+0000 7f06cd490b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55de9a719fa0 con 0x55de9a8cbf30 2024-05-20T20:12:13.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.599+0000 7f06cd490b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55de9a7b4760 con 0x55de9a8cb610 2024-05-20T20:12:13.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.599+0000 7f06cd490b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55de9a8d29e0 con 0x55de9a8c6140 2024-05-20T20:12:13.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.599+0000 7f06cd490b80 10 client.0 _ll_register_callbacks cb 0x55de9a7bdc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:13.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.600+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d1d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.600+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d1d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38070/0 (socket says 172.21.15.28:38070) 2024-05-20T20:12:13.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.600+0000 7f06bbe00700 1 -- 172.21.15.28:0/2708352391 learned_addr learned my addr 172.21.15.28:0/2708352391 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:13.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.600+0000 7f06bb400700 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8cf0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.600+0000 7f06c0c00700 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8cf5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.601+0000 7f06bbe00700 1 -- 172.21.15.28:0/2708352391 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 msgr2=0x55de9a8cf0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.601+0000 7f06bbe00700 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8cf0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.601+0000 7f06bbe00700 1 -- 172.21.15.28:0/2708352391 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 msgr2=0x55de9a8cf5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.601+0000 7f06bbe00700 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8cf5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.601+0000 7f06bbe00700 1 -- 172.21.15.28:0/2708352391 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55de9a7638d0 con 0x55de9a8c6140 2024-05-20T20:12:13.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.602+0000 7f06bbe00700 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d1d10 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f06ac00bbc0 tx=0x7f06ac005fd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e6614e94635a313a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:13.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.602+0000 7f06a6000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:13.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.602+0000 7f06a6000700 1 -- 172.21.15.28:0/2708352391 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06ac02b040 con 0x55de9a8c6140 2024-05-20T20:12:13.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.602+0000 7f06a6000700 1 -- 172.21.15.28:0/2708352391 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f06ac00fdf0 con 0x55de9a8c6140 2024-05-20T20:12:13.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.603+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f06b40ada90 con 0x55de9a8c6140 2024-05-20T20:12:13.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.603+0000 7f06a6000700 1 -- 172.21.15.28:0/2708352391 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f06ac003860 con 0x55de9a8c6140 2024-05-20T20:12:13.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.603+0000 7f06cd490b80 10 client.26734 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:13.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.603+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55de9a7e4060 con 0x55de9a8c6140 2024-05-20T20:12:13.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.604+0000 7f06a6000700 1 -- 172.21.15.28:0/2708352391 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f06b40ada90 con 0x55de9a8c6140 2024-05-20T20:12:13.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.604+0000 7f06a6000700 1 -- 172.21.15.28:0/2708352391 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f06ac02f400 con 0x55de9a8c6140 2024-05-20T20:12:13.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.604+0000 7f06cd490b80 10 client.26734 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:13.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.604+0000 7f06cd490b80 10 client.26734 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:12:13.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.604+0000 7f06cd490b80 20 client.26734 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:13.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.605+0000 7f06cd490b80 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a8df9d0 0x55de9a8e1df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.605+0000 7f06cd490b80 4 client.26734 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:13.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.605+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55de9a825d90 con 0x55de9a8df9d0 2024-05-20T20:12:13.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.605+0000 7f06bb400700 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a8df9d0 0x55de9a8e1df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.606+0000 7f06a6000700 10 client.26734 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:12:13.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.606+0000 7f06bb400700 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a8df9d0 0x55de9a8e1df0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=db85096f084bf8bf server_cookie=6e58c8d854729184 in_seq=0 out_seq=0 2024-05-20T20:12:13.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.607+0000 7f06a6000700 1 -- 172.21.15.28:0/2708352391 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55de9a825d90 con 0x55de9a8df9d0 2024-05-20T20:12:13.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.607+0000 7f06a6000700 10 client.26734 handle_command_reply: tid=0 2024-05-20T20:12:13.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.607+0000 7f06cd490b80 1 client.26734 shutdown 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 msgr2=0x55de9a8d1d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d1d10 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f06ac00bbc0 tx=0x7f06ac005fd0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 shutdown_connections 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a8df9d0 0x55de9a8e1df0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8cf5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d1d10 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 --2- 172.21.15.28:0/2708352391 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8cf0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06a6000700 0 client.26734 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:12:13.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.608+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 >> 172.21.15.28:0/2708352391 conn(0x55de9a7f3be0 msgr2=0x55de9a8bfd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:13.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.609+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 shutdown_connections 2024-05-20T20:12:13.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.609+0000 7f06cd490b80 1 -- 172.21.15.28:0/2708352391 wait complete. 2024-05-20T20:12:13.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.609+0000 7f06cd490b80 20 client.26734 trim_cache size 0 max 16384 2024-05-20T20:12:13.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.633+0000 7f06cd490b80 1 Processor -- start 2024-05-20T20:12:13.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.633+0000 7f06cd490b80 1 -- start start 2024-05-20T20:12:13.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.634+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8c6140 0x55de9a8c5b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.634+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8cb610 0x55de9a8ce010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.634+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8cca20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.634+0000 7f06cd490b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1b4c10 con 0x55de9a8c6140 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.634+0000 7f06cd490b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1b6750 con 0x55de9a8cbf30 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.634+0000 7f06cd490b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1b52e0 con 0x55de9a8cb610 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.634+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8c6140 0x55de9a8c5b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8c6140 0x55de9a8c5b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42312/0 (socket says 172.21.15.28:42312) 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bbe00700 1 -- 172.21.15.28:0/63512401 learned_addr learned my addr 172.21.15.28:0/63512401 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bb400700 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8cb610 0x55de9a8ce010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06c0c00700 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8cca20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bb400700 1 -- 172.21.15.28:0/63512401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 msgr2=0x55de9a8cca20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bb400700 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8cca20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bb400700 1 -- 172.21.15.28:0/63512401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8c6140 msgr2=0x55de9a8c5b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bb400700 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8c6140 0x55de9a8c5b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.635+0000 7f06bb400700 1 -- 172.21.15.28:0/63512401 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55de9a8c2250 con 0x55de9a8cb610 2024-05-20T20:12:13.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.636+0000 7f06bb400700 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8cb610 0x55de9a8ce010 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x55de9a8e2630 tx=0x7f06a800d460 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e2c5061cdf18147b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:13.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.636+0000 7f06b0c00700 1 -- 172.21.15.28:0/63512401 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06a801e070 con 0x55de9a8cb610 2024-05-20T20:12:13.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.636+0000 7f06b0c00700 1 -- 172.21.15.28:0/63512401 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f06a801a2c0 con 0x55de9a8cb610 2024-05-20T20:12:13.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.636+0000 7f06b0c00700 1 -- 172.21.15.28:0/63512401 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06a80116a0 con 0x55de9a8cb610 2024-05-20T20:12:13.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.637+0000 7f06cd490b80 1 -- 172.21.15.28:0/63512401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8cb610 msgr2=0x55de9a8ce010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.637+0000 7f06cd490b80 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8cb610 0x55de9a8ce010 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x55de9a8e2630 tx=0x7f06a800d460 comp rx=0 tx=0).stop 2024-05-20T20:12:13.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 -- 172.21.15.28:0/63512401 shutdown_connections 2024-05-20T20:12:13.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cbf30 0x55de9a8cca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8c6140 0x55de9a8c5b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 --2- 172.21.15.28:0/63512401 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8cb610 0x55de9a8ce010 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 -- 172.21.15.28:0/63512401 >> 172.21.15.28:0/63512401 conn(0x55de9a7f3be0 msgr2=0x55de9a8cc310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:13.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 -- 172.21.15.28:0/63512401 shutdown_connections 2024-05-20T20:12:13.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 -- 172.21.15.28:0/63512401 wait complete. 2024-05-20T20:12:13.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.638+0000 7f06cd490b80 1 -- start start 2024-05-20T20:12:13.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.639+0000 7f06cd490b80 1 Processor -- start 2024-05-20T20:12:13.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d7c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8d81b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06cd490b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8d4c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06cd490b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1b6080 con 0x55de9a8cbf30 2024-05-20T20:12:13.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06cd490b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1b59b0 con 0x55de9a8cb610 2024-05-20T20:12:13.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06cd490b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f06bc1b2bd0 con 0x55de9a8c6140 2024-05-20T20:12:13.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06cd490b80 10 client.0 _ll_register_callbacks cb 0x55de9a7bdc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d7c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06bbe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d7c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38090/0 (socket says 172.21.15.28:38090) 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06bbe00700 1 -- 172.21.15.28:0/3575414101 learned_addr learned my addr 172.21.15.28:0/3575414101 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.640+0000 7f06bb400700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8d81b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.641+0000 7f06c0c00700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8d4c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.641+0000 7f06bbe00700 1 -- 172.21.15.28:0/3575414101 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 msgr2=0x55de9a8d81b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.641+0000 7f06bbe00700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8d81b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.641+0000 7f06bbe00700 1 -- 172.21.15.28:0/3575414101 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 msgr2=0x55de9a8d4c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:13.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.641+0000 7f06bbe00700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8d4c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.641+0000 7f06bbe00700 1 -- 172.21.15.28:0/3575414101 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55de9a8d5180 con 0x55de9a8c6140 2024-05-20T20:12:13.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.641+0000 7f06c0c00700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8d4c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:13.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.642+0000 7f06bbe00700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d7c70 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f06ac02f7d0 tx=0x7f06ac0037c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6cfebe6fdd18146c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:13.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.642+0000 7f06a6000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:13.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.642+0000 7f06a6000700 1 -- 172.21.15.28:0/3575414101 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06ac02b040 con 0x55de9a8c6140 2024-05-20T20:12:13.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.642+0000 7f06a6000700 1 -- 172.21.15.28:0/3575414101 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f06ac003a00 con 0x55de9a8c6140 2024-05-20T20:12:13.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.642+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f06a801a2c0 con 0x55de9a8c6140 2024-05-20T20:12:13.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.643+0000 7f06a6000700 1 -- 172.21.15.28:0/3575414101 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f06ac003c00 con 0x55de9a8c6140 2024-05-20T20:12:13.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.643+0000 7f06cd490b80 10 client.26740 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:13.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.643+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55de9a94ace0 con 0x55de9a8c6140 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.643+0000 7f06a6000700 1 -- 172.21.15.28:0/3575414101 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f06a801a2c0 con 0x55de9a8c6140 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.644+0000 7f06a6000700 1 -- 172.21.15.28:0/3575414101 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f06ac041030 con 0x55de9a8c6140 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.644+0000 7f06cd490b80 10 client.26740 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.644+0000 7f06cd490b80 10 client.26740 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.644+0000 7f06cd490b80 20 client.26740 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.644+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a94b330 0x55de9a94d750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.644+0000 7f06cd490b80 4 client.26740 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:12:13.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.644+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55de9a81f570 con 0x55de9a94b330 2024-05-20T20:12:13.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.645+0000 7f06bb400700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a94b330 0x55de9a94d750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:13.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.645+0000 7f06bb400700 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a94b330 0x55de9a94d750 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=6f0f6e097fc66569 server_cookie=b8235433ce4fbd1a in_seq=0 out_seq=0 2024-05-20T20:12:13.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.645+0000 7f06a6000700 10 client.26740 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:12:13.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06a6000700 1 -- 172.21.15.28:0/3575414101 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55de9a81f570 con 0x55de9a94b330 2024-05-20T20:12:13.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06a6000700 10 client.26740 handle_command_reply: tid=0 2024-05-20T20:12:13.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 client.26740 shutdown 2024-05-20T20:12:13.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 msgr2=0x55de9a8d7c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d7c70 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f06ac02f7d0 tx=0x7f06ac0037c0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 shutdown_connections 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55de9a94b330 0x55de9a94d750 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55de9a8cbf30 0x55de9a8d4c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55de9a8c6140 0x55de9a8d7c70 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.646+0000 7f06cd490b80 1 --2- 172.21.15.28:0/3575414101 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55de9a8cb610 0x55de9a8d81b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.647+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 >> 172.21.15.28:0/3575414101 conn(0x55de9a7f3be0 msgr2=0x55de9a8ccba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.647+0000 7f06a6000700 0 client.26740 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:12:13.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.647+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 shutdown_connections 2024-05-20T20:12:13.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.647+0000 7f06cd490b80 1 -- 172.21.15.28:0/3575414101 wait complete. 2024-05-20T20:12:13.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.647+0000 7f06cd490b80 20 client.26740 trim_cache size 0 max 16384 2024-05-20T20:12:13.649 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:12:13.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.648+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06a00b2af0 msgr2=0x7f06a00b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.648+0000 7f06c2c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06a00b2af0 0x7f06a00b4fb0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f06ac00bd00 tx=0x7f06ac0036c0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.648+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc1a4cf0 msgr2=0x7f06bc1a5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.648+0000 7f06c2c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc1a4cf0 0x7f06bc1a5170 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f06b400cc50 tx=0x7f06b400cf60 comp rx=0 tx=0).stop 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.649+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 shutdown_connections 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.649+0000 7f06c2c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06a00b2af0 0x7f06a00b4fb0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.649+0000 7f06c2c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f06bc1a4cf0 0x7f06bc1a5170 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.649+0000 7f06c2c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06bc102440 0x7f06bc1a04d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.649+0000 7f06c2c00700 1 --2- 172.21.15.28:0/2720675033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06bc102e00 0x7f06bc1a0a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:13.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.649+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 >> 172.21.15.28:0/2720675033 conn(0x7f06bc0793f0 msgr2=0x7f06bc06b460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:13.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.649+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 shutdown_connections 2024-05-20T20:12:13.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:13.650+0000 7f06c2c00700 1 -- 172.21.15.28:0/2720675033 wait complete. 2024-05-20T20:12:14.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 -- 172.21.15.28:0/1571620321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd01092b0 msgr2=0x7f3fd0109690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1571620321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd01092b0 0x7f3fd0109690 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7f3fc4009900 tx=0x7f3fc402a3f0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 -- 172.21.15.28:0/1571620321 shutdown_connections 2024-05-20T20:12:14.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1571620321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3fd0109c60 0x7f3fd010dca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1571620321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd010e380 0x7f3fd0115f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1571620321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd01092b0 0x7f3fd0109690 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 -- 172.21.15.28:0/1571620321 >> 172.21.15.28:0/1571620321 conn(0x7f3fd0076fe0 msgr2=0x7f3fd0079400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:14.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 -- 172.21.15.28:0/1571620321 shutdown_connections 2024-05-20T20:12:14.359 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.357+0000 7f3fd7600640 1 -- 172.21.15.28:0/1571620321 wait complete. 2024-05-20T20:12:14.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 Processor -- start 2024-05-20T20:12:14.360 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 -- start start 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3fd01092b0 0x7f3fd01a5420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd010e380 0x7f3fd01a5960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd019f5d0 0x7f3fd019fa80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd0119310 con 0x7f3fd019f5d0 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd0119190 con 0x7f3fd01092b0 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.358+0000 7f3fd7600640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd0119490 con 0x7f3fd010e380 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fd5600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd019f5d0 0x7f3fd019fa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd010e380 0x7f3fd01a5960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fd5600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd019f5d0 0x7f3fd019fa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42340/0 (socket says 172.21.15.28:42340) 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fd5600640 1 -- 172.21.15.28:0/1563860497 learned_addr learned my addr 172.21.15.28:0/1563860497 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd010e380 0x7f3fd01a5960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38116/0 (socket says 172.21.15.28:38116) 2024-05-20T20:12:14.361 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3fd01092b0 0x7f3fd01a5420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 -- 172.21.15.28:0/1563860497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3fd01092b0 msgr2=0x7f3fd01a5420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3fd01092b0 0x7f3fd01a5420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 -- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd019f5d0 msgr2=0x7f3fd019fa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd019f5d0 0x7f3fd019fa80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 -- 172.21.15.28:0/1563860497 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3fd01a00f0 con 0x7f3fd010e380 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fd5600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd019f5d0 0x7f3fd019fa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.359+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd010e380 0x7f3fd01a5960 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f3fc0005fd0 tx=0x7f3fc000dde0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.360+0000 7f3fcd600640 1 -- 172.21.15.28:0/1563860497 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc0009c00 con 0x7f3fd010e380 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.360+0000 7f3fcd600640 1 -- 172.21.15.28:0/1563860497 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3fc0009da0 con 0x7f3fd010e380 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.360+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3fd01a03e0 con 0x7f3fd010e380 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.360+0000 7f3fcd600640 1 -- 172.21.15.28:0/1563860497 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc0003e40 con 0x7f3fd010e380 2024-05-20T20:12:14.362 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.360+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3fd01ac680 con 0x7f3fd010e380 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 Processor -- start 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 -- start start 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fcd600640 1 -- 172.21.15.28:0/1563860497 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3fc0003fe0 con 0x7f3fd010e380 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813f070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8149930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc814c260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564cc80afb90 con 0x564cc813ec90 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564cc809f730 con 0x564cc813f5b0 2024-05-20T20:12:14.369 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.363+0000 7f3fda380740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564cc80a3770 con 0x564cc8149e70 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42354/0 (socket says 172.21.15.28:42354) 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 -- 172.21.15.28:0/3918734640 learned_addr learned my addr 172.21.15.28:0/3918734640 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8149930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc814c260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 -- 172.21.15.28:0/3918734640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 msgr2=0x564cc814c260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc814c260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 -- 172.21.15.28:0/3918734640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 msgr2=0x564cc8149930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8149930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd5600640 1 -- 172.21.15.28:0/3918734640 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564cc80388b0 con 0x564cc813ec90 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.364+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8149930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:14.370 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.365+0000 7f3fd5600640 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813f070 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f3fc8011850 tx=0x7f3fc8011d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=567ae7b9e41c8912 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.365+0000 7f3fbdc00640 1 -- 172.21.15.28:0/3918734640 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc8011330 con 0x564cc813ec90 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.365+0000 7f3fbdc00640 1 -- 172.21.15.28:0/3918734640 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3fc800fe50 con 0x564cc813ec90 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.365+0000 7f3fbdc00640 1 -- 172.21.15.28:0/3918734640 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc80193e0 con 0x564cc813ec90 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.366+0000 7f3fcd600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3fb80b2a90 0x7f3fb80b4f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.366+0000 7f3fcd600640 1 -- 172.21.15.28:0/1563860497 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f3fc00e20e0 con 0x7f3fd010e380 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.366+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3fb80b2a90 0x7f3fb80b4f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 -- 172.21.15.28:0/3918734640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 msgr2=0x564cc813f070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813f070 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f3fc8011850 tx=0x7f3fc8011d10 comp rx=0 tx=0).stop 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 -- 172.21.15.28:0/3918734640 shutdown_connections 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8149930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc814c260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 --2- 172.21.15.28:0/3918734640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813f070 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.371 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 -- 172.21.15.28:0/3918734640 >> 172.21.15.28:0/3918734640 conn(0x564cc813b1a0 msgr2=0x564cc813b580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:14.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.367+0000 7f3fda380740 1 -- 172.21.15.28:0/3918734640 shutdown_connections 2024-05-20T20:12:14.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.368+0000 7f3fda380740 1 -- 172.21.15.28:0/3918734640 wait complete. 2024-05-20T20:12:14.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.368+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3fb80b2a90 0x7f3fb80b4f50 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f3fc4005ab0 tx=0x7f3fc4005a20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:14.372 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.368+0000 7f3fda380740 1 -- start start 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.368+0000 7f3fda380740 1 Processor -- start 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fda380740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813ec90 0x564cc8149730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fda380740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8146a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fda380740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8146fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fda380740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564cc7d85790 con 0x564cc8149e70 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fda380740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564cc8047bc0 con 0x564cc813f5b0 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fda380740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564cc814caa0 con 0x564cc813ec90 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fcfe00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8146a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.373 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fd4c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813ec90 0x564cc8149730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fd4c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813ec90 0x564cc8149730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38126/0 (socket says 172.21.15.28:38126) 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fd4c00640 1 -- 172.21.15.28:0/1975594141 learned_addr learned my addr 172.21.15.28:0/1975594141 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.369+0000 7f3fda380740 10 client.0 _ll_register_callbacks cb 0x564cc8025400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fd5600640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8146fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fcfe00640 1 -- 172.21.15.28:0/1975594141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813ec90 msgr2=0x564cc8149730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813ec90 0x564cc8149730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fcfe00640 1 -- 172.21.15.28:0/1975594141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 msgr2=0x564cc8146fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8146fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fcfe00640 1 -- 172.21.15.28:0/1975594141 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564cc7db4210 con 0x564cc813f5b0 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fd5600640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8146fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8146a90 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f3fc00df7c0 tx=0x7f3fc000df90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2b500296f674b460 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.370+0000 7f3fb2000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:14.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.371+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813ec90 0x564cc8149730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:14.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.371+0000 7f3fb2000640 1 -- 172.21.15.28:0/1975594141 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc0020070 con 0x564cc813f5b0 2024-05-20T20:12:14.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.371+0000 7f3fb2000640 1 -- 172.21.15.28:0/1975594141 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f3fc00042d0 con 0x564cc813f5b0 2024-05-20T20:12:14.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.371+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3fc800fe50 con 0x564cc813f5b0 2024-05-20T20:12:14.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.371+0000 7f3fb2000640 1 -- 172.21.15.28:0/1975594141 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3fc00a8d50 con 0x564cc813f5b0 2024-05-20T20:12:14.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.371+0000 7f3fda380740 10 client.26843 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:14.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.371+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564cc7e80620 con 0x564cc813f5b0 2024-05-20T20:12:14.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.372+0000 7f3fb2000640 1 -- 172.21.15.28:0/1975594141 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3fc800fe50 con 0x564cc813f5b0 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.373+0000 7f3fb2000640 1 -- 172.21.15.28:0/1975594141 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3fc00dfa40 con 0x564cc813f5b0 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.373+0000 7f3fda380740 10 client.26843 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.373+0000 7f3fda380740 10 client.26843 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.373+0000 7f3fda380740 20 client.26843 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.373+0000 7f3fda380740 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc8157090 0x564cc8159530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.373+0000 7f3fda380740 4 client.26843 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.373+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564cc80b1ae0 con 0x564cc8157090 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.374+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc8157090 0x564cc8159530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.375+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc8157090 0x564cc8159530 crc :-1 s=READY pgs=157 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b648d7c7777c3e56 server_cookie=ff1eaee37260d016 in_seq=0 out_seq=0 2024-05-20T20:12:14.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.375+0000 7f3fb2000640 10 client.26843 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:14.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.375+0000 7f3fb2000640 1 -- 172.21.15.28:0/1975594141 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x564cc80b1ae0 con 0x564cc8157090 2024-05-20T20:12:14.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.375+0000 7f3fb2000640 10 client.26843 handle_command_reply: tid=0 2024-05-20T20:12:14.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.375+0000 7f3fda380740 1 client.26843 shutdown 2024-05-20T20:12:14.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.376+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 msgr2=0x564cc8146a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.376+0000 7f3fda380740 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8146a90 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f3fc00df7c0 tx=0x7f3fc000df90 comp rx=0 tx=0).stop 2024-05-20T20:12:14.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.377+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 shutdown_connections 2024-05-20T20:12:14.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.377+0000 7f3fda380740 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc8157090 0x564cc8159530 crc :-1 s=READY pgs=157 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.377+0000 7f3fda380740 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8146a90 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.377+0000 7f3fda380740 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8146fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.377+0000 7f3fda380740 1 --2- 172.21.15.28:0/1975594141 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813ec90 0x564cc8149730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.377+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 >> 172.21.15.28:0/1975594141 conn(0x564cc813b1a0 msgr2=0x564cc8140f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:14.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.377+0000 7f3fb2000640 0 client.26843 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:14.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.378+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 shutdown_connections 2024-05-20T20:12:14.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.378+0000 7f3fda380740 1 -- 172.21.15.28:0/1975594141 wait complete. 2024-05-20T20:12:14.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.378+0000 7f3fda380740 20 client.26843 trim_cache size 0 max 16384 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 Processor -- start 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 -- start start 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813aa90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8166a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc81489d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd00836f0 con 0x564cc813ec90 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd01ae070 con 0x564cc813f5b0 2024-05-20T20:12:14.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fda380740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd01acc00 con 0x564cc8149e70 2024-05-20T20:12:14.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.410+0000 7f3fd5600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc81489d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.411+0000 7f3fd5600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc81489d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38138/0 (socket says 172.21.15.28:38138) 2024-05-20T20:12:14.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.411+0000 7f3fd5600640 1 -- 172.21.15.28:0/4135439890 learned_addr learned my addr 172.21.15.28:0/4135439890 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:14.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.411+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8166a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.411+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813aa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.411+0000 7f3fd5600640 1 -- 172.21.15.28:0/4135439890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 msgr2=0x564cc8166a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:14.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.411+0000 7f3fd5600640 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8166a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.411+0000 7f3fd5600640 1 -- 172.21.15.28:0/4135439890 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 msgr2=0x564cc813aa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:14.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.412+0000 7f3fd5600640 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813aa90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.412+0000 7f3fd5600640 1 -- 172.21.15.28:0/4135439890 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564cc813afd0 con 0x564cc8149e70 2024-05-20T20:12:14.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.412+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8166a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:14.413 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.412+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813aa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:14.414 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.412+0000 7f3fd5600640 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc81489d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f3fc8000f80 tx=0x7f3fc8005c40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4df8dadae1eeaae2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:14.414 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.413+0000 7f3fbd200640 1 -- 172.21.15.28:0/4135439890 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc8026020 con 0x564cc8149e70 2024-05-20T20:12:14.414 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.413+0000 7f3fbd200640 1 -- 172.21.15.28:0/4135439890 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3fc8011260 con 0x564cc8149e70 2024-05-20T20:12:14.414 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.413+0000 7f3fbd200640 1 -- 172.21.15.28:0/4135439890 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc80052a0 con 0x564cc8149e70 2024-05-20T20:12:14.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 -- 172.21.15.28:0/4135439890 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 msgr2=0x564cc81489d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc81489d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f3fc8000f80 tx=0x7f3fc8005c40 comp rx=0 tx=0).stop 2024-05-20T20:12:14.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 -- 172.21.15.28:0/4135439890 shutdown_connections 2024-05-20T20:12:14.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813f5b0 0x564cc8166a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc8149e70 0x564cc81489d0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 --2- 172.21.15.28:0/4135439890 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc813ec90 0x564cc813aa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 -- 172.21.15.28:0/4135439890 >> 172.21.15.28:0/4135439890 conn(0x564cc813b1a0 msgr2=0x564cc81482c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:14.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.414+0000 7f3fda380740 1 -- 172.21.15.28:0/4135439890 shutdown_connections 2024-05-20T20:12:14.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.415+0000 7f3fda380740 1 -- 172.21.15.28:0/4135439890 wait complete. 2024-05-20T20:12:14.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.415+0000 7f3fda380740 1 -- start start 2024-05-20T20:12:14.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.415+0000 7f3fda380740 1 Processor -- start 2024-05-20T20:12:14.418 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fda380740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813ec90 0x564cc8151cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.418 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fda380740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813f5b0 0x564cc8152230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.418 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fda380740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8142d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fda380740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd01ad9a0 con 0x564cc8149e70 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fda380740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd01ad2d0 con 0x564cc813ec90 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fda380740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3fd00816f0 con 0x564cc813f5b0 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fda380740 10 client.0 _ll_register_callbacks cb 0x7f3fc00e5880 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fd4c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813ec90 0x564cc8151cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fd4c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813ec90 0x564cc8151cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49594/0 (socket says 172.21.15.28:49594) 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.417+0000 7f3fd4c00640 1 -- 172.21.15.28:0/407988026 learned_addr learned my addr 172.21.15.28:0/407988026 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.418+0000 7f3fd4c00640 1 -- 172.21.15.28:0/407988026 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813f5b0 msgr2=0x564cc8152230 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.418+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813f5b0 0x564cc8152230 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.418+0000 7f3fd4c00640 1 -- 172.21.15.28:0/407988026 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 msgr2=0x564cc8142d20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:12:14.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.418+0000 7f3fd5600640 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8142d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.418+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8142d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.418+0000 7f3fd4c00640 1 -- 172.21.15.28:0/407988026 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564cc8143260 con 0x564cc813ec90 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.418+0000 7f3fd5600640 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8142d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.419+0000 7f3fd4c00640 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813ec90 0x564cc8151cf0 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f3fc402a9e0 tx=0x7f3fc402a3f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a95a9426ce014188 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.419+0000 7f3fb2000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.419+0000 7f3fb2000640 1 -- 172.21.15.28:0/407988026 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3fc4035040 con 0x564cc813ec90 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.419+0000 7f3fb2000640 1 -- 172.21.15.28:0/407988026 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f3fc40035f0 con 0x564cc813ec90 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.419+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3fc8011260 con 0x564cc813ec90 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.419+0000 7f3fb2000640 1 -- 172.21.15.28:0/407988026 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3fc40037f0 con 0x564cc813ec90 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 10 client.26849 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:14.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564cc8144500 con 0x564cc813ec90 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fb2000640 1 -- 172.21.15.28:0/407988026 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3fc8011260 con 0x564cc813ec90 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fb2000640 1 -- 172.21.15.28:0/407988026 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3fc40548c0 con 0x564cc813ec90 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 10 client.26849 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 10 client.26849 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 20 client.26849 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc81d56c0 0x564cc81d7ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 4 client.26849 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:12:14.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.420+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564cc80ab2c0 con 0x564cc81d56c0 2024-05-20T20:12:14.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.421+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc81d56c0 0x564cc81d7ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:14.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.421+0000 7f3fb2000640 10 client.26849 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:14.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.422+0000 7f3fcfe00640 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc81d56c0 0x564cc81d7ab0 crc :-1 s=READY pgs=158 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=73f27ba799603418 server_cookie=7dae910ca8b19fa in_seq=0 out_seq=0 2024-05-20T20:12:14.442 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.441+0000 7f3fb2000640 1 -- 172.21.15.28:0/407988026 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x564cc80ab2c0 con 0x564cc81d56c0 2024-05-20T20:12:14.443 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.441+0000 7f3fb2000640 10 client.26849 handle_command_reply: tid=0 2024-05-20T20:12:14.443 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 client.26849 shutdown 2024-05-20T20:12:14.443 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813ec90 msgr2=0x564cc8151cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.443 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813ec90 0x564cc8151cf0 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f3fc402a9e0 tx=0x7f3fc402a3f0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.443 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 shutdown_connections 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564cc81d56c0 0x564cc81d7ab0 crc :-1 s=READY pgs=158 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564cc8149e70 0x564cc8142d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564cc813f5b0 0x564cc8152230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 --2- 172.21.15.28:0/407988026 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564cc813ec90 0x564cc8151cf0 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 >> 172.21.15.28:0/407988026 conn(0x564cc813b1a0 msgr2=0x564cc8142040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.442+0000 7f3fb2000640 0 client.26849 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.443+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 shutdown_connections 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.443+0000 7f3fda380740 1 -- 172.21.15.28:0/407988026 wait complete. 2024-05-20T20:12:14.444 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.443+0000 7f3fda380740 20 client.26849 trim_cache size 0 max 16384 2024-05-20T20:12:14.445 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:12:14.445 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:12:14.445 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:12:14.445 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.444+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3fb80b2a90 msgr2=0x7f3fb80b4f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.445 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.444+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3fb80b2a90 0x7f3fb80b4f50 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f3fc4005ab0 tx=0x7f3fc4005a20 comp rx=0 tx=0).stop 2024-05-20T20:12:14.445 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.444+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd010e380 msgr2=0x7f3fd01a5960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.444+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd010e380 0x7f3fd01a5960 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f3fc0005fd0 tx=0x7f3fc000dde0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.445+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 shutdown_connections 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.445+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3fb80b2a90 0x7f3fb80b4f50 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.445+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3fd019f5d0 0x7f3fd019fa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.445+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3fd010e380 0x7f3fd01a5960 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.445+0000 7f3fd7600640 1 --2- 172.21.15.28:0/1563860497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3fd01092b0 0x7f3fd01a5420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.445+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 >> 172.21.15.28:0/1563860497 conn(0x7f3fd0076fe0 msgr2=0x7f3fd00792e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:14.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.446+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 shutdown_connections 2024-05-20T20:12:14.447 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:14.447+0000 7f3fd7600640 1 -- 172.21.15.28:0/1563860497 wait complete. 2024-05-20T20:12:15.150 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:12:15.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:14 smithi028 ceph-mon[31178]: pgmap v203: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 94 MiB/s wr, 873 op/s 2024-05-20T20:12:15.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:14 smithi154 ceph-mon[36314]: pgmap v203: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 94 MiB/s wr, 873 op/s 2024-05-20T20:12:15.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:14 smithi033 ceph-mon[36179]: pgmap v203: 97 pgs: 97 active+clean; 37 GiB data, 8.0 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 94 MiB/s wr, 873 op/s 2024-05-20T20:12:15.757 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:17.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:16 smithi028 ceph-mon[31178]: pgmap v204: 97 pgs: 97 active+clean; 37 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 KiB/s rd, 76 MiB/s wr, 863 op/s 2024-05-20T20:12:17.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:16 smithi154 ceph-mon[36314]: pgmap v204: 97 pgs: 97 active+clean; 37 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 KiB/s rd, 76 MiB/s wr, 863 op/s 2024-05-20T20:12:17.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:16 smithi033 ceph-mon[36179]: pgmap v204: 97 pgs: 97 active+clean; 37 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 KiB/s rd, 76 MiB/s wr, 863 op/s 2024-05-20T20:12:18.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.285+0000 7f922fa00700 1 -- 172.21.15.28:0/3359892666 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228105780 msgr2=0x7f9228105c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.285+0000 7f922fa00700 1 --2- 172.21.15.28:0/3359892666 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228105780 0x7f9228105c00 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f9214009990 tx=0x7f9214009ca0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.285+0000 7f922fa00700 1 -- 172.21.15.28:0/3359892666 shutdown_connections 2024-05-20T20:12:18.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.285+0000 7f922fa00700 1 --2- 172.21.15.28:0/3359892666 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9228106140 0x7f9228112650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.285+0000 7f922fa00700 1 --2- 172.21.15.28:0/3359892666 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f922810b780 0x7f922810bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.285+0000 7f922fa00700 1 --2- 172.21.15.28:0/3359892666 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228105780 0x7f9228105c00 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.290 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.285+0000 7f922fa00700 1 -- 172.21.15.28:0/3359892666 >> 172.21.15.28:0/3359892666 conn(0x7f9228101270 msgr2=0x7f9228103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.286+0000 7f922fa00700 1 -- 172.21.15.28:0/3359892666 shutdown_connections 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.286+0000 7f922fa00700 1 -- 172.21.15.28:0/3359892666 wait complete. 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.287+0000 7f922fa00700 1 Processor -- start 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.288+0000 7f922fa00700 1 -- start start 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.288+0000 7f922fa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9228105780 0x7f92281124f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.288+0000 7f922fa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228106140 0x7f922810f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.288+0000 7f922fa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f922810b780 0x7f922810fb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.288+0000 7f922fa00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9228115a10 con 0x7f922810b780 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.288+0000 7f922fa00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9228115890 con 0x7f9228105780 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.288+0000 7f922fa00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9228115b90 con 0x7f9228106140 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228106140 0x7f922810f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228106140 0x7f922810f5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54664/0 (socket says 172.21.15.28:54664) 2024-05-20T20:12:18.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 -- 172.21.15.28:0/2457627331 learned_addr learned my addr 172.21.15.28:0/2457627331 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f922d000700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9228105780 0x7f92281124f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 -- 172.21.15.28:0/2457627331 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9228105780 msgr2=0x7f92281124f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9228105780 0x7f92281124f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 -- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f922810b780 msgr2=0x7f922810fb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f922810b780 0x7f922810fb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 -- 172.21.15.28:0/2457627331 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f92281ac6f0 con 0x7f9228106140 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.289+0000 7f9227e00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228106140 0x7f922810f5f0 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f9214005910 tx=0x7f921400b670 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.290+0000 7f9225600700 1 -- 172.21.15.28:0/2457627331 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9214027070 con 0x7f9228106140 2024-05-20T20:12:18.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.290+0000 7f9225600700 1 -- 172.21.15.28:0/2457627331 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9214022470 con 0x7f9228106140 2024-05-20T20:12:18.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.290+0000 7f9225600700 1 -- 172.21.15.28:0/2457627331 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9214017650 con 0x7f9228106140 2024-05-20T20:12:18.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.290+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f92281ac8c0 con 0x7f9228106140 2024-05-20T20:12:18.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.290+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f92281acc60 con 0x7f9228106140 2024-05-20T20:12:18.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.297+0000 7f9225600700 1 -- 172.21.15.28:0/2457627331 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f921402a070 con 0x7f9228106140 2024-05-20T20:12:18.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.297+0000 7f923a383b80 1 Processor -- start 2024-05-20T20:12:18.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.297+0000 7f923a383b80 1 -- start start 2024-05-20T20:12:18.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.298+0000 7f923a383b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f039f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.298+0000 7f923a383b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6efdb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.298+0000 7f923a383b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6efe5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.298+0000 7f923a383b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5612a6e5be40 con 0x5612a6f03f30 2024-05-20T20:12:18.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.298+0000 7f923a383b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5612a6e4b9e0 con 0x5612a6f03610 2024-05-20T20:12:18.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.298+0000 7f923a383b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5612a6e4fa20 con 0x5612a6efe140 2024-05-20T20:12:18.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.299+0000 7f922d000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6efdb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.299+0000 7f922d000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6efdb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:44442/0 (socket says 172.21.15.28:44442) 2024-05-20T20:12:18.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.299+0000 7f922d000700 1 -- 172.21.15.28:0/2468703625 learned_addr learned my addr 172.21.15.28:0/2468703625 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:18.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.300+0000 7f9225600700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f92100b2b40 0x7f92100b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.300+0000 7f9225600700 1 -- 172.21.15.28:0/2457627331 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f92140e3440 con 0x7f9228106140 2024-05-20T20:12:18.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.300+0000 7f922da00700 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f039f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.301+0000 7f9227e00700 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6efe5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.301+0000 7f922d000700 1 -- 172.21.15.28:0/2468703625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 msgr2=0x5612a6efe5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:18.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.301+0000 7f922d000700 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6efe5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.301+0000 7f922d000700 1 -- 172.21.15.28:0/2468703625 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 msgr2=0x5612a6f039f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:18.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.301+0000 7f922d000700 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f039f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.301+0000 7f922d000700 1 -- 172.21.15.28:0/2468703625 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5612a6e14420 con 0x5612a6f03f30 2024-05-20T20:12:18.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.301+0000 7f9227e00700 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6efe5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:18.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.303+0000 7f922da00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f92100b2b40 0x7f92100b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.303+0000 7f922d000700 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6efdb20 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f921c009d00 tx=0x7f921c011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b5565df41d0dcbba server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:18.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.304+0000 7f921a000700 1 -- 172.21.15.28:0/2468703625 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f921c004230 con 0x5612a6f03f30 2024-05-20T20:12:18.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.304+0000 7f921a000700 1 -- 172.21.15.28:0/2468703625 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f921c0043d0 con 0x5612a6f03f30 2024-05-20T20:12:18.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.304+0000 7f921a000700 1 -- 172.21.15.28:0/2468703625 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f921c0197e0 con 0x5612a6f03f30 2024-05-20T20:12:18.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.305+0000 7f923a383b80 1 -- 172.21.15.28:0/2468703625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 msgr2=0x5612a6efdb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.305+0000 7f923a383b80 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6efdb20 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f921c009d00 tx=0x7f921c011720 comp rx=0 tx=0).stop 2024-05-20T20:12:18.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f922da00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f92100b2b40 0x7f92100b5000 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f9220009690 tx=0x7f9220009600 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:18.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f923a383b80 1 -- 172.21.15.28:0/2468703625 shutdown_connections 2024-05-20T20:12:18.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f923a383b80 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6efe5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f923a383b80 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f039f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f923a383b80 1 --2- 172.21.15.28:0/2468703625 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6efdb20 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f923a383b80 1 -- 172.21.15.28:0/2468703625 >> 172.21.15.28:0/2468703625 conn(0x5612a6e2bbe0 msgr2=0x5612a6ef8690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:18.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f923a383b80 1 -- 172.21.15.28:0/2468703625 shutdown_connections 2024-05-20T20:12:18.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.306+0000 7f923a383b80 1 -- 172.21.15.28:0/2468703625 wait complete. 2024-05-20T20:12:18.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.307+0000 7f923a383b80 1 -- start start 2024-05-20T20:12:18.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.307+0000 7f923a383b80 1 Processor -- start 2024-05-20T20:12:18.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.308+0000 7f923a383b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f09b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.308+0000 7f923a383b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f06ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.308+0000 7f923a383b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6f07420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.308+0000 7f923a383b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5612a6d51fa0 con 0x5612a6f03f30 2024-05-20T20:12:18.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.308+0000 7f923a383b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5612a6dec760 con 0x5612a6f03610 2024-05-20T20:12:18.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.308+0000 7f923a383b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5612a6f0a9e0 con 0x5612a6efe140 2024-05-20T20:12:18.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.309+0000 7f922d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f09b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.309+0000 7f923a383b80 10 client.0 _ll_register_callbacks cb 0x5612a6df5c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:18.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.309+0000 7f922d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f09b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54686/0 (socket says 172.21.15.28:54686) 2024-05-20T20:12:18.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.309+0000 7f922d000700 1 -- 172.21.15.28:0/3137389521 learned_addr learned my addr 172.21.15.28:0/3137389521 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:18.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.309+0000 7f9227e00700 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f06ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.309+0000 7f922da00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6f07420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.310+0000 7f9227e00700 1 -- 172.21.15.28:0/3137389521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 msgr2=0x5612a6f09b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:18.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.310+0000 7f9227e00700 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f09b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.310+0000 7f9227e00700 1 -- 172.21.15.28:0/3137389521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 msgr2=0x5612a6f07420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:18.311 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.310+0000 7f9227e00700 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6f07420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.310+0000 7f9227e00700 1 -- 172.21.15.28:0/3137389521 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5612a6d9b8d0 con 0x5612a6f03610 2024-05-20T20:12:18.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.311+0000 7f9227e00700 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f06ee0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f92140e0b80 tx=0x7f92140aa6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e07c1f6ae7f84caf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:18.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.311+0000 7f920ea00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:18.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.311+0000 7f920ea00700 1 -- 172.21.15.28:0/3137389521 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f92140a9f10 con 0x5612a6f03610 2024-05-20T20:12:18.312 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.311+0000 7f920ea00700 1 -- 172.21.15.28:0/3137389521 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f92140b32f0 con 0x5612a6f03610 2024-05-20T20:12:18.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.311+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f921c0043d0 con 0x5612a6f03610 2024-05-20T20:12:18.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.312+0000 7f920ea00700 1 -- 172.21.15.28:0/3137389521 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f92140aa2b0 con 0x5612a6f03610 2024-05-20T20:12:18.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.312+0000 7f923a383b80 10 client.26861 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:18.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.312+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5612a6e1c060 con 0x5612a6f03610 2024-05-20T20:12:18.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.312+0000 7f920ea00700 1 -- 172.21.15.28:0/3137389521 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f921c0043d0 con 0x5612a6f03610 2024-05-20T20:12:18.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.313+0000 7f920ea00700 1 -- 172.21.15.28:0/3137389521 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f92140afce0 con 0x5612a6f03610 2024-05-20T20:12:18.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.313+0000 7f923a383b80 10 client.26861 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:18.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.313+0000 7f923a383b80 10 client.26861 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:12:18.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.314+0000 7f923a383b80 20 client.26861 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:18.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.314+0000 7f923a383b80 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f172d0 0x5612a6f196f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.314+0000 7f923a383b80 4 client.26861 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:18.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.314+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5612a6e5dd90 con 0x5612a6f172d0 2024-05-20T20:12:18.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.314+0000 7f922d000700 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f172d0 0x5612a6f196f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.315+0000 7f922d000700 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f172d0 0x5612a6f196f0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f2a544053598451f server_cookie=81dcd9248cf6c2cd in_seq=0 out_seq=0 2024-05-20T20:12:18.316 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.315+0000 7f920ea00700 10 client.26861 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:12:18.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.316+0000 7f920ea00700 1 -- 172.21.15.28:0/3137389521 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5612a6e5dd90 con 0x5612a6f172d0 2024-05-20T20:12:18.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.316+0000 7f920ea00700 10 client.26861 handle_command_reply: tid=0 2024-05-20T20:12:18.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.316+0000 7f923a383b80 1 client.26861 shutdown 2024-05-20T20:12:18.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 msgr2=0x5612a6f06ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f06ee0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f92140e0b80 tx=0x7f92140aa6a0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 shutdown_connections 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f172d0 0x5612a6f196f0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03f30 0x5612a6f07420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f09b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 --2- 172.21.15.28:0/3137389521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03610 0x5612a6f06ee0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 >> 172.21.15.28:0/3137389521 conn(0x5612a6e2bbe0 msgr2=0x5612a6ef7d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:18.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.317+0000 7f920ea00700 0 client.26861 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:12:18.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.318+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 shutdown_connections 2024-05-20T20:12:18.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.319+0000 7f923a383b80 1 -- 172.21.15.28:0/3137389521 wait complete. 2024-05-20T20:12:18.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.319+0000 7f923a383b80 20 client.26861 trim_cache size 0 max 16384 2024-05-20T20:12:18.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 Processor -- start 2024-05-20T20:12:18.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 -- start start 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6efe140 0x5612a6efdb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6f03610 0x5612a6f04a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 0x5612a6ef84f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f92281b4490 con 0x5612a6efe140 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f92281b5fd0 con 0x5612a6f03f30 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.345+0000 7f923a383b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f92281b4b60 con 0x5612a6f03610 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.346+0000 7f9227e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6f03610 0x5612a6f04a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.346+0000 7f9227e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6f03610 0x5612a6f04a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54696/0 (socket says 172.21.15.28:54696) 2024-05-20T20:12:18.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.346+0000 7f9227e00700 1 -- 172.21.15.28:0/1239718078 learned_addr learned my addr 172.21.15.28:0/1239718078 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:18.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.346+0000 7f9227e00700 1 -- 172.21.15.28:0/1239718078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 msgr2=0x5612a6ef84f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:12:18.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.346+0000 7f9227e00700 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 0x5612a6ef84f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.346+0000 7f9227e00700 1 -- 172.21.15.28:0/1239718078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6efe140 msgr2=0x5612a6efdb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:18.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.347+0000 7f922d000700 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6efe140 0x5612a6efdb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.347+0000 7f9227e00700 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6efe140 0x5612a6efdb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.347+0000 7f9227e00700 1 -- 172.21.15.28:0/1239718078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5612a6efa230 con 0x5612a6f03610 2024-05-20T20:12:18.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.348+0000 7f9227e00700 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6f03610 0x5612a6f04a20 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f92140aa730 tx=0x7f92140b1180 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2fd735d574c28496 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:18.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.348+0000 7f922d000700 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6efe140 0x5612a6efdb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:18.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.348+0000 7f9219600700 1 -- 172.21.15.28:0/1239718078 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f92140aeb50 con 0x5612a6f03610 2024-05-20T20:12:18.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f9219600700 1 -- 172.21.15.28:0/1239718078 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f92140aaea0 con 0x5612a6f03610 2024-05-20T20:12:18.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f9219600700 1 -- 172.21.15.28:0/1239718078 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f92140bfb60 con 0x5612a6f03610 2024-05-20T20:12:18.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f923a383b80 1 -- 172.21.15.28:0/1239718078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6f03610 msgr2=0x5612a6f04a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f923a383b80 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6f03610 0x5612a6f04a20 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f92140aa730 tx=0x7f92140b1180 comp rx=0 tx=0).stop 2024-05-20T20:12:18.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f923a383b80 1 -- 172.21.15.28:0/1239718078 shutdown_connections 2024-05-20T20:12:18.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f923a383b80 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 0x5612a6ef84f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f923a383b80 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6efe140 0x5612a6efdb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f923a383b80 1 --2- 172.21.15.28:0/1239718078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6f03610 0x5612a6f04a20 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.349+0000 7f923a383b80 1 -- 172.21.15.28:0/1239718078 >> 172.21.15.28:0/1239718078 conn(0x5612a6e2bbe0 msgr2=0x5612a6f04310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:18.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.350+0000 7f923a383b80 1 -- 172.21.15.28:0/1239718078 shutdown_connections 2024-05-20T20:12:18.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.350+0000 7f923a383b80 1 -- 172.21.15.28:0/1239718078 wait complete. 2024-05-20T20:12:18.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.350+0000 7f923a383b80 1 -- start start 2024-05-20T20:12:18.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.350+0000 7f923a383b80 1 Processor -- start 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.351+0000 7f923a383b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f10160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.351+0000 7f923a383b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03610 0x5612a6f1e650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.351+0000 7f923a383b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 0x5612a6f1eb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.351+0000 7f922d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f10160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.351+0000 7f922d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f10160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54698/0 (socket says 172.21.15.28:54698) 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.351+0000 7f922d000700 1 -- 172.21.15.28:0/2420735474 learned_addr learned my addr 172.21.15.28:0/2420735474 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.352+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f92281b5900 con 0x5612a6f03610 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.352+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f92281b5230 con 0x5612a6f03f30 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.352+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f92281b2450 con 0x5612a6efe140 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.352+0000 7f9227e00700 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03610 0x5612a6f1e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.352+0000 7f923a383b80 10 client.0 _ll_register_callbacks cb 0x5612a6df5c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:18.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.352+0000 7f922da00700 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 0x5612a6f1eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.353+0000 7f922d000700 1 -- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 msgr2=0x5612a6f1eb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:18.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.353+0000 7f922d000700 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 0x5612a6f1eb90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.353+0000 7f922d000700 1 -- 172.21.15.28:0/2420735474 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03610 msgr2=0x5612a6f1e650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:18.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.353+0000 7f922d000700 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03610 0x5612a6f1e650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.353+0000 7f922d000700 1 -- 172.21.15.28:0/2420735474 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5612a6f1f1d0 con 0x5612a6efe140 2024-05-20T20:12:18.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.353+0000 7f922d000700 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f10160 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f921c003fd0 tx=0x7f921c0096d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4e71abf3d7dae798 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:18.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.353+0000 7f920ea00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:18.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.354+0000 7f920ea00700 1 -- 172.21.15.28:0/2420735474 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f921c003680 con 0x5612a6efe140 2024-05-20T20:12:18.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.354+0000 7f920ea00700 1 -- 172.21.15.28:0/2420735474 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f921c003820 con 0x5612a6efe140 2024-05-20T20:12:18.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.354+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f92140aaea0 con 0x5612a6efe140 2024-05-20T20:12:18.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.354+0000 7f920ea00700 1 -- 172.21.15.28:0/2420735474 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f921c005b00 con 0x5612a6efe140 2024-05-20T20:12:18.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.355+0000 7f923a383b80 10 client.26773 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:18.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.355+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5612a6f92c20 con 0x5612a6efe140 2024-05-20T20:12:18.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.355+0000 7f920ea00700 1 -- 172.21.15.28:0/2420735474 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f92140aaea0 con 0x5612a6efe140 2024-05-20T20:12:18.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f920ea00700 1 -- 172.21.15.28:0/2420735474 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f921c004770 con 0x5612a6efe140 2024-05-20T20:12:18.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f923a383b80 10 client.26773 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:18.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f923a383b80 10 client.26773 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:12:18.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f923a383b80 20 client.26773 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:18.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f923a383b80 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f931c0 0x5612a6f955e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:18.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f923a383b80 4 client.26773 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:12:18.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5612a6e57570 con 0x5612a6f931c0 2024-05-20T20:12:18.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.356+0000 7f9227e00700 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f931c0 0x5612a6f955e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:18.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.359+0000 7f9227e00700 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f931c0 0x5612a6f955e0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=60c6a83ea1864914 server_cookie=93d2eeddab36c7b3 in_seq=0 out_seq=0 2024-05-20T20:12:18.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.359+0000 7f920ea00700 10 client.26773 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:12:18.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.360+0000 7f920ea00700 1 -- 172.21.15.28:0/2420735474 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5612a6e57570 con 0x5612a6f931c0 2024-05-20T20:12:18.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.360+0000 7f920ea00700 10 client.26773 handle_command_reply: tid=0 2024-05-20T20:12:18.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.360+0000 7f923a383b80 1 client.26773 shutdown 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.360+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 msgr2=0x5612a6f10160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.360+0000 7f923a383b80 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f10160 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f921c003fd0 tx=0x7f921c0096d0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 shutdown_connections 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5612a6f931c0 0x5612a6f955e0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612a6f03f30 0x5612a6f1eb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612a6efe140 0x5612a6f10160 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 --2- 172.21.15.28:0/2420735474 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612a6f03610 0x5612a6f1e650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 >> 172.21.15.28:0/2420735474 conn(0x5612a6e2bbe0 msgr2=0x5612a6f08460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f920ea00700 0 client.26773 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:12:18.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 shutdown_connections 2024-05-20T20:12:18.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 1 -- 172.21.15.28:0/2420735474 wait complete. 2024-05-20T20:12:18.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.361+0000 7f923a383b80 20 client.26773 trim_cache size 0 max 16384 2024-05-20T20:12:18.363 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:12:18.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.362+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f92100b2b40 msgr2=0x7f92100b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.362+0000 7f922fa00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f92100b2b40 0x7f92100b5000 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f9220009690 tx=0x7f9220009600 comp rx=0 tx=0).stop 2024-05-20T20:12:18.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.362+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228106140 msgr2=0x7f922810f5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:18.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.362+0000 7f922fa00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228106140 0x7f922810f5f0 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f9214005910 tx=0x7f921400b670 comp rx=0 tx=0).stop 2024-05-20T20:12:18.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.363+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 shutdown_connections 2024-05-20T20:12:18.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.363+0000 7f922fa00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f922810b780 0x7f922810fb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.363+0000 7f922fa00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f92100b2b40 0x7f92100b5000 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.363+0000 7f922fa00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9228105780 0x7f92281124f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.363+0000 7f922fa00700 1 --2- 172.21.15.28:0/2457627331 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9228106140 0x7f922810f5f0 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:18.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.363+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 >> 172.21.15.28:0/2457627331 conn(0x7f9228101270 msgr2=0x7f9228101970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:18.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.363+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 shutdown_connections 2024-05-20T20:12:18.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:18.364+0000 7f922fa00700 1 -- 172.21.15.28:0/2457627331 wait complete. 2024-05-20T20:12:19.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:18 smithi028 ceph-mon[31178]: pgmap v205: 97 pgs: 97 active+clean; 37 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 KiB/s rd, 51 MiB/s wr, 921 op/s 2024-05-20T20:12:19.135 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:12:19.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:18 smithi154 ceph-mon[36314]: pgmap v205: 97 pgs: 97 active+clean; 37 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 KiB/s rd, 51 MiB/s wr, 921 op/s 2024-05-20T20:12:19.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:18 smithi033 ceph-mon[36179]: pgmap v205: 97 pgs: 97 active+clean; 37 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 8.4 KiB/s rd, 51 MiB/s wr, 921 op/s 2024-05-20T20:12:19.747 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:21.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:20 smithi028 ceph-mon[31178]: pgmap v206: 97 pgs: 97 active+clean; 37 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 50 MiB/s wr, 1.49k op/s 2024-05-20T20:12:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:20 smithi154 ceph-mon[36314]: pgmap v206: 97 pgs: 97 active+clean; 37 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 50 MiB/s wr, 1.49k op/s 2024-05-20T20:12:21.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:20 smithi033 ceph-mon[36179]: pgmap v206: 97 pgs: 97 active+clean; 37 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 50 MiB/s wr, 1.49k op/s 2024-05-20T20:12:23.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:22 smithi028 ceph-mon[31178]: pgmap v207: 97 pgs: 97 active+clean; 37 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 51 MiB/s wr, 1.61k op/s 2024-05-20T20:12:23.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:22 smithi154 ceph-mon[36314]: pgmap v207: 97 pgs: 97 active+clean; 37 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 51 MiB/s wr, 1.61k op/s 2024-05-20T20:12:23.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:22 smithi033 ceph-mon[36179]: pgmap v207: 97 pgs: 97 active+clean; 37 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 51 MiB/s wr, 1.61k op/s 2024-05-20T20:12:25.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.228+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3921740015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406b220 msgr2=0x7f0ff406b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.228+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3921740015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406b220 0x7f0ff406b6a0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f0fe0009990 tx=0x7f0fe0009ca0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.228+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3921740015 shutdown_connections 2024-05-20T20:12:25.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.228+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3921740015 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0ff406bbe0 0x7f0ff41103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.228+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3921740015 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ff41095b0 0x7f0ff4109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.228+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3921740015 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406b220 0x7f0ff406b6a0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.228+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3921740015 >> 172.21.15.28:0/3921740015 conn(0x7f0ff40793f0 msgr2=0x7f0ff4079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:25.231 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:24 smithi028 ceph-mon[31178]: pgmap v208: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 66 MiB/s wr, 2.01k op/s 2024-05-20T20:12:25.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.229+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3921740015 shutdown_connections 2024-05-20T20:12:25.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.229+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3921740015 wait complete. 2024-05-20T20:12:25.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.231+0000 7f0ff9c00700 1 Processor -- start 2024-05-20T20:12:25.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.231+0000 7f0ff9c00700 1 -- start start 2024-05-20T20:12:25.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff9c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ff406b220 0x7f0ff419beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff9c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406bbe0 0x7f0ff419c3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff9c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0ff41095b0 0x7f0ff41a0780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff9c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff4115950 con 0x7f0ff406b220 2024-05-20T20:12:25.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff9c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff41157d0 con 0x7f0ff406bbe0 2024-05-20T20:12:25.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff9c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff4115ad0 con 0x7f0ff41095b0 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff2000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406bbe0 0x7f0ff419c3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff2000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406bbe0 0x7f0ff419c3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45080/0 (socket says 172.21.15.28:45080) 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.232+0000 7f0ff2000700 1 -- 172.21.15.28:0/3338020374 learned_addr learned my addr 172.21.15.28:0/3338020374 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff3400700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0ff41095b0 0x7f0ff41a0780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ff406b220 0x7f0ff419beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff2000700 1 -- 172.21.15.28:0/3338020374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0ff41095b0 msgr2=0x7f0ff41a0780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff2000700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0ff41095b0 0x7f0ff41a0780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff2000700 1 -- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ff406b220 msgr2=0x7f0ff419beb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff2000700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ff406b220 0x7f0ff419beb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff2000700 1 -- 172.21.15.28:0/3338020374 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0ff41a0ec0 con 0x7f0ff406bbe0 2024-05-20T20:12:25.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.233+0000 7f0ff2000700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406bbe0 0x7f0ff419c3f0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f0fe0000c00 tx=0x7f0fe0009510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:25.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.234+0000 7f0feb400700 1 -- 172.21.15.28:0/3338020374 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fe0027070 con 0x7f0ff406bbe0 2024-05-20T20:12:25.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.234+0000 7f0feb400700 1 -- 172.21.15.28:0/3338020374 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0fe0022470 con 0x7f0ff406bbe0 2024-05-20T20:12:25.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.234+0000 7f0feb400700 1 -- 172.21.15.28:0/3338020374 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fe0017650 con 0x7f0ff406bbe0 2024-05-20T20:12:25.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.234+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0ff41a1150 con 0x7f0ff406bbe0 2024-05-20T20:12:25.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.234+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0ff41a8a30 con 0x7f0ff406bbe0 2024-05-20T20:12:25.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.239+0000 7f0feb400700 1 -- 172.21.15.28:0/3338020374 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0fe002a070 con 0x7f0ff406bbe0 2024-05-20T20:12:25.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.241+0000 7f10045f3b80 1 Processor -- start 2024-05-20T20:12:25.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.242+0000 7f10045f3b80 1 -- start start 2024-05-20T20:12:25.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.242+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c90290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.242+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c8a260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.242+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8a880 0x564610c8ace0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.242+0000 7f10045f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564610be8e40 con 0x564610c8feb0 2024-05-20T20:12:25.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.242+0000 7f10045f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564610bd89e0 con 0x564610c8a880 2024-05-20T20:12:25.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.242+0000 7f10045f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564610bdca20 con 0x564610c907d0 2024-05-20T20:12:25.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.243+0000 7f0ff2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c8a260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.243+0000 7f0ff2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c8a260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54726/0 (socket says 172.21.15.28:54726) 2024-05-20T20:12:25.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.243+0000 7f0ff2a00700 1 -- 172.21.15.28:0/4153396264 learned_addr learned my addr 172.21.15.28:0/4153396264 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:25.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.243+0000 7f0ff3400700 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c90290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.243+0000 7f0ff2000700 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8a880 0x564610c8ace0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.244+0000 7f0ff2a00700 1 -- 172.21.15.28:0/4153396264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8a880 msgr2=0x564610c8ace0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.244+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8a880 0x564610c8ace0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.244+0000 7f0ff2a00700 1 -- 172.21.15.28:0/4153396264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 msgr2=0x564610c90290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.244+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c90290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.244+0000 7f0ff2a00700 1 -- 172.21.15.28:0/4153396264 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564610ba1420 con 0x564610c907d0 2024-05-20T20:12:25.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.244+0000 7f0feb400700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0fd80b2bc0 0x7f0fd80b5080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.244+0000 7f0feb400700 1 -- 172.21.15.28:0/3338020374 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f0fe00e3340 con 0x7f0ff406bbe0 2024-05-20T20:12:25.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.245+0000 7f0ff3400700 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c90290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:25.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.245+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c8a260 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f0fe4009bf0 tx=0x7f0fe4009f00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a07a1b345f3a2ad4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:25.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.245+0000 7f0fdfe00700 1 -- 172.21.15.28:0/4153396264 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fe4016070 con 0x564610c907d0 2024-05-20T20:12:25.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.245+0000 7f0fdfe00700 1 -- 172.21.15.28:0/4153396264 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0fe4004830 con 0x564610c907d0 2024-05-20T20:12:25.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.245+0000 7f0fdfe00700 1 -- 172.21.15.28:0/4153396264 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fe4005650 con 0x564610c907d0 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f0ff2000700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0fd80b2bc0 0x7f0fd80b5080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f10045f3b80 1 -- 172.21.15.28:0/4153396264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 msgr2=0x564610c8a260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f10045f3b80 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c8a260 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f0fe4009bf0 tx=0x7f0fe4009f00 comp rx=0 tx=0).stop 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f10045f3b80 1 -- 172.21.15.28:0/4153396264 shutdown_connections 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f10045f3b80 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8a880 0x564610c8ace0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f10045f3b80 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c90290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f10045f3b80 1 --2- 172.21.15.28:0/4153396264 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c8a260 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.246+0000 7f10045f3b80 1 -- 172.21.15.28:0/4153396264 >> 172.21.15.28:0/4153396264 conn(0x564610bb8be0 msgr2=0x564610c85010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:25.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.247+0000 7f0ff2000700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0fd80b2bc0 0x7f0fd80b5080 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0fe00a7620 tx=0x7f0fe0022f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:25.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.247+0000 7f10045f3b80 1 -- 172.21.15.28:0/4153396264 shutdown_connections 2024-05-20T20:12:25.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.247+0000 7f10045f3b80 1 -- 172.21.15.28:0/4153396264 wait complete. 2024-05-20T20:12:25.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.248+0000 7f10045f3b80 1 -- start start 2024-05-20T20:12:25.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.249+0000 7f10045f3b80 1 Processor -- start 2024-05-20T20:12:25.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8a880 0x564610c96430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8feb0 0x564610c937d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c93d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f10045f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564610adefa0 con 0x564610c8a880 2024-05-20T20:12:25.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f10045f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564610b79760 con 0x564610c8feb0 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f0ff2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8feb0 0x564610c937d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f0ff2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8feb0 0x564610c937d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45100/0 (socket says 172.21.15.28:45100) 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f10045f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564610c97290 con 0x564610c907d0 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f0ff2a00700 1 -- 172.21.15.28:0/2664911244 learned_addr learned my addr 172.21.15.28:0/2664911244 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.250+0000 7f10045f3b80 10 client.0 _ll_register_callbacks cb 0x564610b82c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.251+0000 7f0ff3400700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8a880 0x564610c96430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.251+0000 7f0ff2a00700 1 -- 172.21.15.28:0/2664911244 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 msgr2=0x564610c93d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.251+0000 7f0ff3400700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c93d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.251+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c93d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.251+0000 7f0ff2a00700 1 -- 172.21.15.28:0/2664911244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8a880 msgr2=0x564610c96430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.251+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8a880 0x564610c96430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.251+0000 7f0ff2a00700 1 -- 172.21.15.28:0/2664911244 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564610b288d0 con 0x564610c8feb0 2024-05-20T20:12:25.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.252+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0fe4004830 con 0x564610c8feb0 2024-05-20T20:12:25.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.252+0000 7f0ff3400700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c93d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:25.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.252+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8feb0 0x564610c937d0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f0fe4005f40 tx=0x7f0fe400eb80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c10641b1ff7069f0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:25.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.252+0000 7f0fdcc00700 10 client.26876 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:25.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.253+0000 7f0fdcc00700 1 -- 172.21.15.28:0/2664911244 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fe4028060 con 0x564610c8feb0 2024-05-20T20:12:25.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.253+0000 7f0fdcc00700 1 -- 172.21.15.28:0/2664911244 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f0fe40049e0 con 0x564610c8feb0 2024-05-20T20:12:25.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.253+0000 7f0fdcc00700 1 -- 172.21.15.28:0/2664911244 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0fe4016040 con 0x564610c8feb0 2024-05-20T20:12:25.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.253+0000 7f10045f3b80 10 client.26876 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:25.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.253+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x564610ba9060 con 0x564610c8feb0 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.254+0000 7f0fdcc00700 1 -- 172.21.15.28:0/2664911244 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0fe400f040 con 0x564610c8feb0 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.254+0000 7f10045f3b80 10 client.26876 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.254+0000 7f10045f3b80 10 client.26876 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.254+0000 7f10045f3b80 20 client.26876 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.254+0000 7f10045f3b80 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610cac2c0 0x564610cae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.254+0000 7f10045f3b80 4 client.26876 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.254+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564610bead90 con 0x564610cac2c0 2024-05-20T20:12:25.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.255+0000 7f0ff3400700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610cac2c0 0x564610cae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.256+0000 7f0ff3400700 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610cac2c0 0x564610cae6b0 crc :-1 s=READY pgs=159 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=169561e5f847574d server_cookie=be3d6fc4edf6ee46 in_seq=0 out_seq=0 2024-05-20T20:12:25.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.256+0000 7f0fdcc00700 10 client.26876 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:25.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.257+0000 7f0fdcc00700 1 -- 172.21.15.28:0/2664911244 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x564610bead90 con 0x564610cac2c0 2024-05-20T20:12:25.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.257+0000 7f0fdcc00700 10 client.26876 handle_command_reply: tid=0 2024-05-20T20:12:25.326 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.257+0000 7f10045f3b80 1 client.26876 shutdown 2024-05-20T20:12:25.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8feb0 msgr2=0x564610c937d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8feb0 0x564610c937d0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f0fe4005f40 tx=0x7f0fe400eb80 comp rx=0 tx=0).stop 2024-05-20T20:12:25.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 shutdown_connections 2024-05-20T20:12:25.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610cac2c0 0x564610cae6b0 crc :-1 s=READY pgs=159 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c907d0 0x564610c93d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8a880 0x564610c96430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 --2- 172.21.15.28:0/2664911244 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c8feb0 0x564610c937d0 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f0fdcc00700 0 client.26876 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.258+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 >> 172.21.15.28:0/2664911244 conn(0x564610bb8be0 msgr2=0x564610c89250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.259+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 shutdown_connections 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.259+0000 7f10045f3b80 1 -- 172.21.15.28:0/2664911244 wait complete. 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.259+0000 7f10045f3b80 20 client.26876 trim_cache size 0 max 16384 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.287+0000 7f10045f3b80 1 Processor -- start 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.287+0000 7f10045f3b80 1 -- start start 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610c8a260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c94370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610c90c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f10045f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff41abb60 con 0x564610c8feb0 2024-05-20T20:12:25.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f10045f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff41ad6a0 con 0x564610c907d0 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f10045f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff41ac230 con 0x564610c8a880 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f0ff3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610c8a260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f0ff3400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610c8a260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54746/0 (socket says 172.21.15.28:54746) 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f0ff3400700 1 -- 172.21.15.28:0/1262148272 learned_addr learned my addr 172.21.15.28:0/1262148272 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c94370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.288+0000 7f0ff3400700 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610c90c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.289+0000 7f0ff3400700 1 -- 172.21.15.28:0/1262148272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 msgr2=0x564610c90c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.289+0000 7f0ff3400700 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610c90c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.289+0000 7f0ff3400700 1 -- 172.21.15.28:0/1262148272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 msgr2=0x564610c94370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.289+0000 7f0ff3400700 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c94370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.289+0000 7f0ff3400700 1 -- 172.21.15.28:0/1262148272 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564610c8b2d0 con 0x564610c8a880 2024-05-20T20:12:25.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.290+0000 7f0ff3400700 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610c8a260 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f0fec004b10 tx=0x7f0fec009040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6bcb28ec1eac4f08 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.290+0000 7f0fdf400700 1 -- 172.21.15.28:0/1262148272 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fec01e040 con 0x564610c8a880 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.290+0000 7f0fdf400700 1 -- 172.21.15.28:0/1262148272 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0fec003c00 con 0x564610c8a880 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.291+0000 7f0fdf400700 1 -- 172.21.15.28:0/1262148272 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fec004310 con 0x564610c8a880 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.292+0000 7f10045f3b80 1 -- 172.21.15.28:0/1262148272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 msgr2=0x564610c8a260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.292+0000 7f10045f3b80 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610c8a260 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f0fec004b10 tx=0x7f0fec009040 comp rx=0 tx=0).stop 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.292+0000 7f10045f3b80 1 -- 172.21.15.28:0/1262148272 shutdown_connections 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.292+0000 7f10045f3b80 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610c90c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.292+0000 7f10045f3b80 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610c8a260 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.292+0000 7f10045f3b80 1 --2- 172.21.15.28:0/1262148272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610c94370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.292+0000 7f10045f3b80 1 -- 172.21.15.28:0/1262148272 >> 172.21.15.28:0/1262148272 conn(0x564610bb8be0 msgr2=0x564610c920c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.293+0000 7f10045f3b80 1 -- 172.21.15.28:0/1262148272 shutdown_connections 2024-05-20T20:12:25.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.293+0000 7f10045f3b80 1 -- 172.21.15.28:0/1262148272 wait complete. 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.293+0000 7f10045f3b80 1 -- start start 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 1 Processor -- start 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610cb1f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610caf2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610caf9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff41acfd0 con 0x564610c8feb0 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff41ac900 con 0x564610c907d0 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0ff41150e0 con 0x564610c8a880 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.294+0000 7f10045f3b80 10 client.0 _ll_register_callbacks cb 0x564610b82c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.295+0000 7f0ff3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610caf9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.295+0000 7f0ff3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610caf9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45122/0 (socket says 172.21.15.28:45122) 2024-05-20T20:12:25.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.295+0000 7f0ff3400700 1 -- 172.21.15.28:0/930348292 learned_addr learned my addr 172.21.15.28:0/930348292 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.295+0000 7f0ff3400700 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610cb1f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.296+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610caf2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.297+0000 7f0ff2a00700 1 -- 172.21.15.28:0/930348292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 msgr2=0x564610cb1f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.297+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610cb1f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.297+0000 7f0ff2a00700 1 -- 172.21.15.28:0/930348292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 msgr2=0x564610caf9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.297+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610caf9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.297+0000 7f0ff2a00700 1 -- 172.21.15.28:0/930348292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564610d15b00 con 0x564610c8feb0 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.297+0000 7f0ff2a00700 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610caf2a0 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7f0fe400e680 tx=0x7f0fe401c050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5966e18cfd29bc0f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.297+0000 7f0fdcc00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.298+0000 7f0fdcc00700 1 -- 172.21.15.28:0/930348292 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0fe403a060 con 0x564610c8feb0 2024-05-20T20:12:25.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.298+0000 7f0fdcc00700 1 -- 172.21.15.28:0/930348292 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f0fe4011880 con 0x564610c8feb0 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.298+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0fec003c00 con 0x564610c8feb0 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.298+0000 7f0fdcc00700 1 -- 172.21.15.28:0/930348292 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0fe4028020 con 0x564610c8feb0 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 10 client.16554 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564610d16c80 con 0x564610c8feb0 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f0fdcc00700 1 -- 172.21.15.28:0/930348292 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0fec003c00 con 0x564610c8feb0 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f0fdcc00700 1 -- 172.21.15.28:0/930348292 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0fe4034070 con 0x564610c8feb0 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 10 client.16554 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 10 client.16554 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 20 client.16554 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610d172d0 0x564610d196c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 4 client.16554 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.299+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x564610be4570 con 0x564610d172d0 2024-05-20T20:12:25.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.300+0000 7f0ff3400700 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610d172d0 0x564610d196c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.300+0000 7f0fdcc00700 10 client.16554 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.300+0000 7f0ff3400700 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610d172d0 0x564610d196c0 crc :-1 s=READY pgs=160 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=854c4ae49262e7d3 server_cookie=a19cddee7cebe568 in_seq=0 out_seq=0 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.301+0000 7f0fdcc00700 1 -- 172.21.15.28:0/930348292 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x564610be4570 con 0x564610d172d0 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.301+0000 7f0fdcc00700 10 client.16554 handle_command_reply: tid=0 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.301+0000 7f10045f3b80 1 client.16554 shutdown 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.301+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 msgr2=0x564610caf2a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.301+0000 7f10045f3b80 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610caf2a0 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7f0fe400e680 tx=0x7f0fe401c050 comp rx=0 tx=0).stop 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 shutdown_connections 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f10045f3b80 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564610d172d0 0x564610d196c0 crc :-1 s=READY pgs=160 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f10045f3b80 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564610c907d0 0x564610caf9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f10045f3b80 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564610c8a880 0x564610cb1f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f0fdcc00700 0 client.16554 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:25.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f10045f3b80 1 --2- 172.21.15.28:0/930348292 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564610c8feb0 0x564610caf2a0 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 >> 172.21.15.28:0/930348292 conn(0x564610bb8be0 msgr2=0x564610c935b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:25.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.302+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 shutdown_connections 2024-05-20T20:12:25.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.303+0000 7f10045f3b80 1 -- 172.21.15.28:0/930348292 wait complete. 2024-05-20T20:12:25.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.303+0000 7f10045f3b80 20 client.16554 trim_cache size 0 max 16384 2024-05-20T20:12:25.339 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0fd80b2bc0 msgr2=0x7f0fd80b5080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0fd80b2bc0 0x7f0fd80b5080 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0fe00a7620 tx=0x7f0fe0022f90 comp rx=0 tx=0).stop 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406bbe0 msgr2=0x7f0ff419c3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406bbe0 0x7f0ff419c3f0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f0fe0000c00 tx=0x7f0fe0009510 comp rx=0 tx=0).stop 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 shutdown_connections 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0ff41095b0 0x7f0ff41a0780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0ff406b220 0x7f0ff419beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0fd80b2bc0 0x7f0fd80b5080 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 --2- 172.21.15.28:0/3338020374 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0ff406bbe0 0x7f0ff419c3f0 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:25.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 >> 172.21.15.28:0/3338020374 conn(0x7f0ff40793f0 msgr2=0x7f0ff41012b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:25.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.304+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 shutdown_connections 2024-05-20T20:12:25.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:25.305+0000 7f0ff9c00700 1 -- 172.21.15.28:0/3338020374 wait complete. 2024-05-20T20:12:25.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:24 smithi154 ceph-mon[36314]: pgmap v208: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 66 MiB/s wr, 2.01k op/s 2024-05-20T20:12:25.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:24 smithi033 ceph-mon[36179]: pgmap v208: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 66 MiB/s wr, 2.01k op/s 2024-05-20T20:12:27.209 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:12:27.209 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:12:27.285 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:26 smithi028 ceph-mon[31178]: pgmap v209: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 59 MiB/s wr, 1.76k op/s 2024-05-20T20:12:27.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:26 smithi154 ceph-mon[36314]: pgmap v209: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 59 MiB/s wr, 1.76k op/s 2024-05-20T20:12:27.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:26 smithi033 ceph-mon[36179]: pgmap v209: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 59 MiB/s wr, 1.76k op/s 2024-05-20T20:12:27.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 -- 172.21.15.28:0/2148388482 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc52811c230 msgr2=0x7fc52811e620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 --2- 172.21.15.28:0/2148388482 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc52811c230 0x7fc52811e620 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7fc520009cd0 tx=0x7fc52002a9c0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 -- 172.21.15.28:0/2148388482 shutdown_connections 2024-05-20T20:12:27.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 --2- 172.21.15.28:0/2148388482 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc52811c230 0x7fc52811e620 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 --2- 172.21.15.28:0/2148388482 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc528111620 0x7fc528109f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 --2- 172.21.15.28:0/2148388482 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc528111240 0x7fc528109a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 -- 172.21.15.28:0/2148388482 >> 172.21.15.28:0/2148388482 conn(0x7fc52806db20 msgr2=0x7fc52806df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:27.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.653+0000 7fc52d200640 1 -- 172.21.15.28:0/2148388482 shutdown_connections 2024-05-20T20:12:27.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.654+0000 7fc52d200640 1 -- 172.21.15.28:0/2148388482 wait complete. 2024-05-20T20:12:27.655 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.655+0000 7fc52d200640 1 Processor -- start 2024-05-20T20:12:27.655 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.656+0000 7fc52d200640 1 -- start start 2024-05-20T20:12:27.657 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc52d200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc528111240 0x7fc5281a80d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.657 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc52d200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc52811c230 0x7fc5281a8610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.657 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc52d200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5281ac990 0x7fc5281ace40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.657 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc52d200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc528121b50 con 0x7fc528111240 2024-05-20T20:12:27.657 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc52d200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc5281219d0 con 0x7fc5281ac990 2024-05-20T20:12:27.657 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc52d200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc528121cd0 con 0x7fc52811c230 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc525400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc52811c230 0x7fc5281a8610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc525400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc52811c230 0x7fc5281a8610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58822/0 (socket says 172.21.15.28:58822) 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.657+0000 7fc525400640 1 -- 172.21.15.28:0/3680557579 learned_addr learned my addr 172.21.15.28:0/3680557579 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc526800640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5281ac990 0x7fc5281ace40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc525e00640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc528111240 0x7fc5281a80d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc526800640 1 -- 172.21.15.28:0/3680557579 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc52811c230 msgr2=0x7fc5281a8610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc526800640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc52811c230 0x7fc5281a8610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc526800640 1 -- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc528111240 msgr2=0x7fc5281a80d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc526800640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc528111240 0x7fc5281a80d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc526800640 1 -- 172.21.15.28:0/3680557579 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc5281ad480 con 0x7fc5281ac990 2024-05-20T20:12:27.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc525400640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc52811c230 0x7fc5281a8610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:27.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.658+0000 7fc525e00640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc528111240 0x7fc5281a80d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:27.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.659+0000 7fc526800640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5281ac990 0x7fc5281ace40 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7fc52002afb0 tx=0x7fc52002ff10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:27.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.659+0000 7fc51ea00640 1 -- 172.21.15.28:0/3680557579 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc520034050 con 0x7fc5281ac990 2024-05-20T20:12:27.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.659+0000 7fc51ea00640 1 -- 172.21.15.28:0/3680557579 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc520003e20 con 0x7fc5281ac990 2024-05-20T20:12:27.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.659+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc5281ad710 con 0x7fc5281ac990 2024-05-20T20:12:27.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.660+0000 7fc51ea00640 1 -- 172.21.15.28:0/3680557579 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc520039420 con 0x7fc5281ac990 2024-05-20T20:12:27.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.660+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc5281b4ed0 con 0x7fc5281ac990 2024-05-20T20:12:27.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.665+0000 7fc530034740 1 Processor -- start 2024-05-20T20:12:27.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc530034740 1 -- start start 2024-05-20T20:12:27.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc530034740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024bc90 0x55df6024c070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc530034740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024c5b0 0x55df60256930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.667 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc530034740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 0x55df60259260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc530034740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55df601bcb90 con 0x55df6024bc90 2024-05-20T20:12:27.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc530034740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55df601ac730 con 0x55df6024c5b0 2024-05-20T20:12:27.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc530034740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55df601b0770 con 0x55df60256e70 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc525e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024c5b0 0x55df60256930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc525e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024c5b0 0x55df60256930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36190/0 (socket says 172.21.15.28:36190) 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.667+0000 7fc525e00640 1 -- 172.21.15.28:0/3061942975 learned_addr learned my addr 172.21.15.28:0/3061942975 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc525e00640 1 -- 172.21.15.28:0/3061942975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 msgr2=0x55df60259260 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc525e00640 1 --2- 172.21.15.28:0/3061942975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 0x55df60259260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc525e00640 1 -- 172.21.15.28:0/3061942975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024bc90 msgr2=0x55df6024c070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc525e00640 1 --2- 172.21.15.28:0/3061942975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024bc90 0x55df6024c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc525e00640 1 -- 172.21.15.28:0/3061942975 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55df601458b0 con 0x55df6024c5b0 2024-05-20T20:12:27.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc51ea00640 1 -- 172.21.15.28:0/3680557579 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc520037020 con 0x7fc5281ac990 2024-05-20T20:12:27.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc525e00640 1 --2- 172.21.15.28:0/3061942975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024c5b0 0x55df60256930 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7fc51800bcc0 tx=0x7fc51800f730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f91a0eacd70e96a9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:27.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc513400640 1 -- 172.21.15.28:0/3061942975 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc518005020 con 0x55df6024c5b0 2024-05-20T20:12:27.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc513400640 1 -- 172.21.15.28:0/3061942975 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc5180047a0 con 0x55df6024c5b0 2024-05-20T20:12:27.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.668+0000 7fc513400640 1 -- 172.21.15.28:0/3061942975 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5180116a0 con 0x55df6024c5b0 2024-05-20T20:12:27.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.670+0000 7fc530034740 1 -- 172.21.15.28:0/3061942975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024c5b0 msgr2=0x55df60256930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.670+0000 7fc530034740 1 --2- 172.21.15.28:0/3061942975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024c5b0 0x55df60256930 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7fc51800bcc0 tx=0x7fc51800f730 comp rx=0 tx=0).stop 2024-05-20T20:12:27.671 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.670+0000 7fc530034740 1 -- 172.21.15.28:0/3061942975 shutdown_connections 2024-05-20T20:12:27.671 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.670+0000 7fc530034740 1 --2- 172.21.15.28:0/3061942975 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024c5b0 0x55df60256930 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.671 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.670+0000 7fc530034740 1 --2- 172.21.15.28:0/3061942975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 0x55df60259260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.671 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.670+0000 7fc530034740 1 --2- 172.21.15.28:0/3061942975 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024bc90 0x55df6024c070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.671 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.670+0000 7fc530034740 1 -- 172.21.15.28:0/3061942975 >> 172.21.15.28:0/3061942975 conn(0x55df602481a0 msgr2=0x55df60248580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.671+0000 7fc530034740 1 -- 172.21.15.28:0/3061942975 shutdown_connections 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.671+0000 7fc530034740 1 -- 172.21.15.28:0/3061942975 wait complete. 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.671+0000 7fc530034740 1 -- start start 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 1 Processor -- start 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60256880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024c5b0 0x55df60253b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df60256e70 0x55df60254180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55df5fe92790 con 0x55df60256e70 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55df60154bc0 con 0x55df6024bc90 2024-05-20T20:12:27.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55df60259aa0 con 0x55df6024c5b0 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.672+0000 7fc530034740 10 client.0 _ll_register_callbacks cb 0x55df60132400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.673+0000 7fc525e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024c5b0 0x55df60253b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.673+0000 7fc525400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60256880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.673+0000 7fc525e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024c5b0 0x55df60253b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58840/0 (socket says 172.21.15.28:58840) 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.673+0000 7fc525e00640 1 -- 172.21.15.28:0/3557270010 learned_addr learned my addr 172.21.15.28:0/3557270010 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.673+0000 7fc525400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60256880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36206/0 (socket says 172.21.15.28:36206) 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.673+0000 7fc51ea00640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc50c0b2a20 0x7fc50c0b4ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525400640 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df60256e70 0x55df60254180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc51ea00640 1 -- 172.21.15.28:0/3680557579 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fc520102ef0 con 0x7fc5281ac990 2024-05-20T20:12:27.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525e00640 1 -- 172.21.15.28:0/3557270010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 msgr2=0x55df60256880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:27.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525e00640 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60256880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.674 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525e00640 1 -- 172.21.15.28:0/3557270010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df60256e70 msgr2=0x55df60254180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:27.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525e00640 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df60256e70 0x55df60254180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525e00640 1 -- 172.21.15.28:0/3557270010 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55df5fec1210 con 0x55df6024c5b0 2024-05-20T20:12:27.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525400640 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df60256e70 0x55df60254180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc525e00640 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024c5b0 0x55df60253b70 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fc518004b80 tx=0x7fc518004bb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=98269987bbbb60ed server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.674+0000 7fc507e00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.675+0000 7fc526800640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc50c0b2a20 0x7fc50c0b4ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.675+0000 7fc507e00640 1 -- 172.21.15.28:0/3557270010 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc51802a060 con 0x55df6024c5b0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.675+0000 7fc507e00640 1 -- 172.21.15.28:0/3557270010 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fc518004f20 con 0x55df6024c5b0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.675+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc5180047a0 con 0x55df6024c5b0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.676+0000 7fc507e00640 1 -- 172.21.15.28:0/3557270010 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc518005120 con 0x55df6024c5b0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.676+0000 7fc526800640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc50c0b2a20 0x7fc50c0b4ee0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fc5200c61e0 tx=0x7fc520030000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.676+0000 7fc530034740 10 client.26797 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:27.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.676+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55df5ff8d620 con 0x55df6024c5b0 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.676+0000 7fc507e00640 1 -- 172.21.15.28:0/3557270010 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc5180047a0 con 0x55df6024c5b0 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.676+0000 7fc507e00640 1 -- 172.21.15.28:0/3557270010 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc51801b410 con 0x55df6024c5b0 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.677+0000 7fc530034740 10 client.26797 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.677+0000 7fc530034740 10 client.26797 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.677+0000 7fc530034740 20 client.26797 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.677+0000 7fc530034740 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df60266800 0x55df60268bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.677+0000 7fc530034740 4 client.26797 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:27.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.677+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55df601beae0 con 0x55df60266800 2024-05-20T20:12:27.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.677+0000 7fc525400640 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df60266800 0x55df60268bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.678+0000 7fc525400640 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df60266800 0x55df60268bf0 crc :-1 s=READY pgs=161 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e4e1caff3428e79f server_cookie=2fd5aa4ca9e913fa in_seq=0 out_seq=0 2024-05-20T20:12:27.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.678+0000 7fc507e00640 10 client.26797 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:27.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.679+0000 7fc507e00640 1 -- 172.21.15.28:0/3557270010 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55df601beae0 con 0x55df60266800 2024-05-20T20:12:27.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.679+0000 7fc507e00640 10 client.26797 handle_command_reply: tid=0 2024-05-20T20:12:27.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.679+0000 7fc530034740 1 client.26797 shutdown 2024-05-20T20:12:27.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024c5b0 msgr2=0x55df60253b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024c5b0 0x55df60253b70 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fc518004b80 tx=0x7fc518004bb0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 shutdown_connections 2024-05-20T20:12:27.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df60266800 0x55df60268bf0 crc :-1 s=READY pgs=161 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df60256e70 0x55df60254180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024c5b0 0x55df60253b70 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 --2- 172.21.15.28:0/3557270010 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60256880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 >> 172.21.15.28:0/3557270010 conn(0x55df602481a0 msgr2=0x55df6024df20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:27.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc507e00640 0 client.26797 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:27.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.680+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 shutdown_connections 2024-05-20T20:12:27.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.681+0000 7fc530034740 1 -- 172.21.15.28:0/3557270010 wait complete. 2024-05-20T20:12:27.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.681+0000 7fc530034740 20 client.26797 trim_cache size 0 max 16384 2024-05-20T20:12:27.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.709+0000 7fc530034740 1 Processor -- start 2024-05-20T20:12:27.709 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.709+0000 7fc530034740 1 -- start start 2024-05-20T20:12:27.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc530034740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60262b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc530034740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df602563b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc530034740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 0x55df6026e330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc530034740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc5281bd580 con 0x55df6024c5b0 2024-05-20T20:12:27.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc530034740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc5281bf0c0 con 0x55df6024bc90 2024-05-20T20:12:27.710 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc530034740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc5281bdc50 con 0x55df60256e70 2024-05-20T20:12:27.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc525e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df602563b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc525e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df602563b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60202/0 (socket says 172.21.15.28:60202) 2024-05-20T20:12:27.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.710+0000 7fc525e00640 1 -- 172.21.15.28:0/2042458640 learned_addr learned my addr 172.21.15.28:0/2042458640 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:27.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc525e00640 1 -- 172.21.15.28:0/2042458640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 msgr2=0x55df6026e330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:27.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc525e00640 1 --2- 172.21.15.28:0/2042458640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 0x55df6026e330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc525e00640 1 -- 172.21.15.28:0/2042458640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 msgr2=0x55df60262b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:27.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc525e00640 1 --2- 172.21.15.28:0/2042458640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60262b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc525e00640 1 -- 172.21.15.28:0/2042458640 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55df602544f0 con 0x55df6024c5b0 2024-05-20T20:12:27.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc525e00640 1 --2- 172.21.15.28:0/2042458640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df602563b0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fc518006010 tx=0x7fc51800bf90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b97e576bc54454ae server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:27.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc512a00640 1 -- 172.21.15.28:0/2042458640 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc51802a070 con 0x55df6024c5b0 2024-05-20T20:12:27.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc512a00640 1 -- 172.21.15.28:0/2042458640 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc518004ef0 con 0x55df6024c5b0 2024-05-20T20:12:27.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.711+0000 7fc512a00640 1 -- 172.21.15.28:0/2042458640 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc518034080 con 0x55df6024c5b0 2024-05-20T20:12:27.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.712+0000 7fc530034740 1 -- 172.21.15.28:0/2042458640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 msgr2=0x55df602563b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.712+0000 7fc530034740 1 --2- 172.21.15.28:0/2042458640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df602563b0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fc518006010 tx=0x7fc51800bf90 comp rx=0 tx=0).stop 2024-05-20T20:12:27.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.712+0000 7fc530034740 1 -- 172.21.15.28:0/2042458640 shutdown_connections 2024-05-20T20:12:27.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.712+0000 7fc530034740 1 --2- 172.21.15.28:0/2042458640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df60256e70 0x55df6026e330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.712+0000 7fc530034740 1 --2- 172.21.15.28:0/2042458640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df602563b0 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.712+0000 7fc530034740 1 --2- 172.21.15.28:0/2042458640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df6024bc90 0x55df60262b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.712+0000 7fc530034740 1 -- 172.21.15.28:0/2042458640 >> 172.21.15.28:0/2042458640 conn(0x55df602481a0 msgr2=0x55df6024ded0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.713+0000 7fc530034740 1 -- 172.21.15.28:0/2042458640 shutdown_connections 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.713+0000 7fc530034740 1 -- 172.21.15.28:0/2042458640 wait complete. 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.713+0000 7fc530034740 1 -- start start 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.713+0000 7fc530034740 1 Processor -- start 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.714+0000 7fc530034740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024bc90 0x55df6025a080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.714+0000 7fc530034740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df6025a5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.714+0000 7fc530034740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df60256e70 0x55df602cf1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.714+0000 7fc530034740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc5281be9f0 con 0x55df6024c5b0 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.714+0000 7fc530034740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc5281be320 con 0x55df60256e70 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.714+0000 7fc530034740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc5281bb580 con 0x55df6024bc90 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.714+0000 7fc530034740 10 client.0 _ll_register_callbacks cb 0x7fc518015070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.715+0000 7fc525e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df6025a5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.715+0000 7fc525e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df6025a5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60204/0 (socket says 172.21.15.28:60204) 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.715+0000 7fc525e00640 1 -- 172.21.15.28:0/935381261 learned_addr learned my addr 172.21.15.28:0/935381261 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.715+0000 7fc525400640 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df60256e70 0x55df602cf1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.715+0000 7fc525400640 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024bc90 0x55df6025a080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.716+0000 7fc525400640 1 -- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df60256e70 msgr2=0x55df602cf1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:27.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.716+0000 7fc525400640 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df60256e70 0x55df602cf1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.716+0000 7fc525400640 1 -- 172.21.15.28:0/935381261 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 msgr2=0x55df6025a5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:27.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.716+0000 7fc525400640 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df6025a5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.716+0000 7fc525400640 1 -- 172.21.15.28:0/935381261 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55df602cf730 con 0x55df6024bc90 2024-05-20T20:12:27.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.716+0000 7fc507e00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:27.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.716+0000 7fc525400640 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024bc90 0x55df6025a080 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fc514009880 tx=0x7fc514005f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a79540c9d14e8fd5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:27.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.717+0000 7fc507e00640 1 -- 172.21.15.28:0/935381261 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc514011050 con 0x55df6024bc90 2024-05-20T20:12:27.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.717+0000 7fc507e00640 1 -- 172.21.15.28:0/935381261 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fc51401a040 con 0x55df6024bc90 2024-05-20T20:12:27.717 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.717+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc518004ef0 con 0x55df6024bc90 2024-05-20T20:12:27.718 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.717+0000 7fc507e00640 1 -- 172.21.15.28:0/935381261 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc514011050 con 0x55df6024bc90 2024-05-20T20:12:27.718 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.717+0000 7fc530034740 10 client.26800 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:27.718 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.717+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55df602d0a10 con 0x55df6024bc90 2024-05-20T20:12:27.718 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc507e00640 1 -- 172.21.15.28:0/935381261 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc514005020 con 0x55df6024bc90 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc507e00640 1 -- 172.21.15.28:0/935381261 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc514027020 con 0x55df6024bc90 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc530034740 10 client.26800 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc530034740 10 client.26800 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc530034740 20 client.26800 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc530034740 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df602d1160 0x55df602d3580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc530034740 4 client.26800 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x7fc518004ef0 con 0x55df602d1160 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.718+0000 7fc525e00640 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df602d1160 0x55df602d3580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.719+0000 7fc507e00640 10 client.26800 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:27.719 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.719+0000 7fc525e00640 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df602d1160 0x55df602d3580 crc :-1 s=READY pgs=162 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b6674194906abab7 server_cookie=cc14d1772537d339 in_seq=0 out_seq=0 2024-05-20T20:12:27.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.741+0000 7fc507e00640 1 -- 172.21.15.28:0/935381261 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x7fc518004ef0 con 0x55df602d1160 2024-05-20T20:12:27.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.741+0000 7fc507e00640 10 client.26800 handle_command_reply: tid=0 2024-05-20T20:12:27.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.741+0000 7fc530034740 1 client.26800 shutdown 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024bc90 msgr2=0x55df6025a080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024bc90 0x55df6025a080 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fc514009880 tx=0x7fc514005f90 comp rx=0 tx=0).stop 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 shutdown_connections 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55df602d1160 0x55df602d3580 crc :-1 s=READY pgs=162 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55df60256e70 0x55df602cf1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55df6024c5b0 0x55df6025a5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 --2- 172.21.15.28:0/935381261 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55df6024bc90 0x55df6025a080 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 >> 172.21.15.28:0/935381261 conn(0x55df602481a0 msgr2=0x55df60261af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc507e00640 0 client.26800 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:27.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 shutdown_connections 2024-05-20T20:12:27.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 1 -- 172.21.15.28:0/935381261 wait complete. 2024-05-20T20:12:27.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.742+0000 7fc530034740 20 client.26800 trim_cache size 0 max 16384 2024-05-20T20:12:27.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:12:27.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:12:27.743 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:12:27.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.743+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc50c0b2a20 msgr2=0x7fc50c0b4ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.743+0000 7fc52d200640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc50c0b2a20 0x7fc50c0b4ee0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fc5200c61e0 tx=0x7fc520030000 comp rx=0 tx=0).stop 2024-05-20T20:12:27.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.743+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5281ac990 msgr2=0x7fc5281ace40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:27.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.743+0000 7fc52d200640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5281ac990 0x7fc5281ace40 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7fc52002afb0 tx=0x7fc52002ff10 comp rx=0 tx=0).stop 2024-05-20T20:12:27.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.743+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 shutdown_connections 2024-05-20T20:12:27.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.743+0000 7fc52d200640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5281ac990 0x7fc5281ace40 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.744+0000 7fc52d200640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc50c0b2a20 0x7fc50c0b4ee0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.744+0000 7fc52d200640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc52811c230 0x7fc5281a8610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.744+0000 7fc52d200640 1 --2- 172.21.15.28:0/3680557579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc528111240 0x7fc5281a80d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:27.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.744+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 >> 172.21.15.28:0/3680557579 conn(0x7fc52806db20 msgr2=0x7fc52810ef00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:27.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.744+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 shutdown_connections 2024-05-20T20:12:27.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:27.744+0000 7fc52d200640 1 -- 172.21.15.28:0/3680557579 wait complete. 2024-05-20T20:12:27.826 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:29.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:28 smithi028 ceph-mon[31178]: pgmap v210: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 64 MiB/s wr, 1.80k op/s 2024-05-20T20:12:29.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:28 smithi154 ceph-mon[36314]: pgmap v210: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 64 MiB/s wr, 1.80k op/s 2024-05-20T20:12:29.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:28 smithi033 ceph-mon[36179]: pgmap v210: 97 pgs: 97 active+clean; 37 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 64 MiB/s wr, 1.80k op/s 2024-05-20T20:12:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.766+0000 7f01c7600700 1 -- 172.21.15.28:0/630838408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01041a0 msgr2=0x7f01c01081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.766+0000 7f01c7600700 1 --2- 172.21.15.28:0/630838408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01041a0 0x7f01c01081c0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f01ac009920 tx=0x7f01ac009c30 comp rx=0 tx=0).stop 2024-05-20T20:12:30.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.773+0000 7f01c7600700 1 -- 172.21.15.28:0/630838408 shutdown_connections 2024-05-20T20:12:30.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.773+0000 7f01c7600700 1 --2- 172.21.15.28:0/630838408 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01c01088f0 0x7f01c0110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.773+0000 7f01c7600700 1 --2- 172.21.15.28:0/630838408 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01c01037f0 0x7f01c0103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.773+0000 7f01c7600700 1 --2- 172.21.15.28:0/630838408 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01041a0 0x7f01c01081c0 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.773+0000 7f01c7600700 1 -- 172.21.15.28:0/630838408 >> 172.21.15.28:0/630838408 conn(0x7f01c007a6a0 msgr2=0x7f01c007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:30.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.773+0000 7f01c7600700 1 -- 172.21.15.28:0/630838408 shutdown_connections 2024-05-20T20:12:30.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.773+0000 7f01c7600700 1 -- 172.21.15.28:0/630838408 wait complete. 2024-05-20T20:12:30.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.775+0000 7f01c7600700 1 Processor -- start 2024-05-20T20:12:30.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.775+0000 7f01c7600700 1 -- start start 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.776+0000 7f01c7600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01c01037f0 0x7f01c0074c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.776+0000 7f01c7600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01c01041a0 0x7f01c00751b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.777+0000 7f01c7600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01088f0 0x7f01c006df50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.777+0000 7f01c7600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f01c01137f0 con 0x7f01c01041a0 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.777+0000 7f01c7600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f01c0113670 con 0x7f01c01088f0 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.777+0000 7f01c7600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f01c0113970 con 0x7f01c01037f0 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.777+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01c01037f0 0x7f01c0074c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.777+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01c01037f0 0x7f01c0074c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58868/0 (socket says 172.21.15.28:58868) 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.777+0000 7f01c4c00700 1 -- 172.21.15.28:0/40010161 learned_addr learned my addr 172.21.15.28:0/40010161 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.778+0000 7f01c5600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01088f0 0x7f01c006df50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.778+0000 7f01bfe00700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01c01041a0 0x7f01c00751b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.781 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.779+0000 7f01c5600700 1 -- 172.21.15.28:0/40010161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01c01037f0 msgr2=0x7f01c0074c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.779+0000 7f01c5600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01c01037f0 0x7f01c0074c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.779+0000 7f01c5600700 1 -- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01c01041a0 msgr2=0x7f01c00751b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.779+0000 7f01c5600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01c01041a0 0x7f01c00751b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.779+0000 7f01c5600700 1 -- 172.21.15.28:0/40010161 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f01c006e850 con 0x7f01c01088f0 2024-05-20T20:12:30.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.780+0000 7f01c5600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01088f0 0x7f01c006df50 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f01b8009460 tx=0x7f01b800d3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:30.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.782+0000 7f01bd600700 1 -- 172.21.15.28:0/40010161 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01b8004db0 con 0x7f01c01088f0 2024-05-20T20:12:30.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.782+0000 7f01bd600700 1 -- 172.21.15.28:0/40010161 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f01b8012550 con 0x7f01c01088f0 2024-05-20T20:12:30.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.782+0000 7f01bd600700 1 -- 172.21.15.28:0/40010161 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01b801c670 con 0x7f01c01088f0 2024-05-20T20:12:30.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.782+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f01c006eb40 con 0x7f01c01088f0 2024-05-20T20:12:30.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.782+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f01c0072ac0 con 0x7f01c01088f0 2024-05-20T20:12:30.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.792+0000 7f01bd600700 1 -- 172.21.15.28:0/40010161 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f01b8012700 con 0x7f01c01088f0 2024-05-20T20:12:30.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.796+0000 7f01bd600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a80b2c00 0x7f01a80b50c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.798+0000 7f01bd600700 1 -- 172.21.15.28:0/40010161 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f01b80e63a0 con 0x7f01c01088f0 2024-05-20T20:12:30.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.799+0000 7f01c4c00700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a80b2c00 0x7f01a80b50c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.801+0000 7f01c4c00700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a80b2c00 0x7f01a80b50c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f01b4005950 tx=0x7f01b4009310 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:30.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.811+0000 7f01d2083b80 1 Processor -- start 2024-05-20T20:12:30.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.812+0000 7f01d2083b80 1 -- start start 2024-05-20T20:12:30.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.814+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5be310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.815+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5be310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.815+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5b82e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.815+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b8d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.815+0000 7f01d2083b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cb0f516ec0 con 0x55cb0f5be850 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.815+0000 7f01d2083b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cb0f506a60 con 0x55cb0f5b8900 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.815+0000 7f01d2083b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cb0f50aaa0 con 0x55cb0f5bdf30 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.815+0000 7f01bfe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5b82e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.816+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5be310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58876/0 (socket says 172.21.15.28:58876) 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.816+0000 7f01bfe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5b82e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60240/0 (socket says 172.21.15.28:60240) 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.816+0000 7f01bfe00700 1 -- 172.21.15.28:0/3891418176 learned_addr learned my addr 172.21.15.28:0/3891418176 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.816+0000 7f01c5600700 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b8d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.817+0000 7f01c5600700 1 -- 172.21.15.28:0/3891418176 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 msgr2=0x55cb0f5be310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.817+0000 7f01c5600700 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5be310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.817+0000 7f01c5600700 1 -- 172.21.15.28:0/3891418176 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 msgr2=0x55cb0f5b82e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.817+0000 7f01c5600700 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5b82e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.817+0000 7f01c5600700 1 -- 172.21.15.28:0/3891418176 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cb0f4cf4a0 con 0x55cb0f5b8900 2024-05-20T20:12:30.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.817+0000 7f01bfe00700 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5b82e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:30.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.818+0000 7f01c4c00700 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5be310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello state changed while learned_addr, mark_down or replacing must be happened just now 2024-05-20T20:12:30.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.818+0000 7f01c5600700 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b8d60 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f01b80e3b90 tx=0x7f01b80acf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=652b8db6081cc8a1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:30.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.819+0000 7f01b2000700 1 -- 172.21.15.28:0/3891418176 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01b80e3f10 con 0x55cb0f5b8900 2024-05-20T20:12:30.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.820+0000 7f01b2000700 1 -- 172.21.15.28:0/3891418176 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f01b80b4e50 con 0x55cb0f5b8900 2024-05-20T20:12:30.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.821+0000 7f01b2000700 1 -- 172.21.15.28:0/3891418176 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01b80232f0 con 0x55cb0f5b8900 2024-05-20T20:12:30.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.822+0000 7f01d2083b80 1 -- 172.21.15.28:0/3891418176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 msgr2=0x55cb0f5b8d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.822+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b8d60 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f01b80e3b90 tx=0x7f01b80acf60 comp rx=0 tx=0).stop 2024-05-20T20:12:30.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.824+0000 7f01d2083b80 1 -- 172.21.15.28:0/3891418176 shutdown_connections 2024-05-20T20:12:30.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.824+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b8d60 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.824+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5be310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.824+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3891418176 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5b82e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.824+0000 7f01d2083b80 1 -- 172.21.15.28:0/3891418176 >> 172.21.15.28:0/3891418176 conn(0x55cb0f4e6c60 msgr2=0x55cb0f5b3090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:30.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.824+0000 7f01d2083b80 1 -- 172.21.15.28:0/3891418176 shutdown_connections 2024-05-20T20:12:30.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.824+0000 7f01d2083b80 1 -- 172.21.15.28:0/3891418176 wait complete. 2024-05-20T20:12:30.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.826+0000 7f01d2083b80 1 -- start start 2024-05-20T20:12:30.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.826+0000 7f01d2083b80 1 Processor -- start 2024-05-20T20:12:30.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.828+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c4560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.828+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c1900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c4560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c4560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58884/0 (socket says 172.21.15.28:58884) 2024-05-20T20:12:30.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5c1e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01d2083b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cb0f40d020 con 0x55cb0f5be850 2024-05-20T20:12:30.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01d2083b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cb0f4a77e0 con 0x55cb0f5bdf30 2024-05-20T20:12:30.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01d2083b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cb0f5c52b0 con 0x55cb0f5b8900 2024-05-20T20:12:30.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01d2083b80 10 client.0 _ll_register_callbacks cb 0x55cb0f4b0cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:30.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01bfe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c1900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01bfe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c1900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36258/0 (socket says 172.21.15.28:36258) 2024-05-20T20:12:30.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.829+0000 7f01bfe00700 1 -- 172.21.15.28:0/3017799837 learned_addr learned my addr 172.21.15.28:0/3017799837 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.830+0000 7f01bfe00700 1 -- 172.21.15.28:0/3017799837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5b8900 msgr2=0x55cb0f5c4560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.831+0000 7f01bfe00700 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c4560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.831+0000 7f01bfe00700 1 -- 172.21.15.28:0/3017799837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 msgr2=0x55cb0f5c1e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.831+0000 7f01bfe00700 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5c1e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.831+0000 7f01bfe00700 1 -- 172.21.15.28:0/3017799837 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cb0f456950 con 0x55cb0f5bdf30 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.831+0000 7f01c4c00700 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c4560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.832+0000 7f01bfe00700 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c1900 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f01ac005ce0 tx=0x7f01ac005d10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ccba044e5dfcccae server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:30.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.832+0000 7f01a6a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:30.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.832+0000 7f01a6a00700 1 -- 172.21.15.28:0/3017799837 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01ac005590 con 0x55cb0f5bdf30 2024-05-20T20:12:30.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.833+0000 7f01a6a00700 1 -- 172.21.15.28:0/3017799837 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f01ac0046d0 con 0x55cb0f5bdf30 2024-05-20T20:12:30.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.833+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f01b80b4e50 con 0x55cb0f5bdf30 2024-05-20T20:12:30.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.834+0000 7f01a6a00700 1 -- 172.21.15.28:0/3017799837 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f01ac0048f0 con 0x55cb0f5bdf30 2024-05-20T20:12:30.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.836+0000 7f01d2083b80 10 client.26909 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:30.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.836+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cb0f4d70e0 con 0x55cb0f5bdf30 2024-05-20T20:12:30.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.837+0000 7f01a6a00700 1 -- 172.21.15.28:0/3017799837 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f01b80b4e50 con 0x55cb0f5bdf30 2024-05-20T20:12:30.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.837+0000 7f01a6a00700 1 -- 172.21.15.28:0/3017799837 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f01ac02b030 con 0x55cb0f5bdf30 2024-05-20T20:12:30.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.838+0000 7f01d2083b80 10 client.26909 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:30.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.839+0000 7f01d2083b80 10 client.26909 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:30.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.839+0000 7f01d2083b80 20 client.26909 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:30.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.839+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f5d1d90 0x55cb0f5d41b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.839+0000 7f01d2083b80 4 client.26909 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:30.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.839+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55cb0f518e10 con 0x55cb0f5d1d90 2024-05-20T20:12:30.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.839+0000 7f01c4c00700 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f5d1d90 0x55cb0f5d41b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.840+0000 7f01c4c00700 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f5d1d90 0x55cb0f5d41b0 crc :-1 s=READY pgs=163 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cb254f1d5048163f server_cookie=543d5890be0f29ec in_seq=0 out_seq=0 2024-05-20T20:12:30.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.840+0000 7f01a6a00700 10 client.26909 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:30.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.841+0000 7f01a6a00700 1 -- 172.21.15.28:0/3017799837 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55cb0f518e10 con 0x55cb0f5d1d90 2024-05-20T20:12:30.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.841+0000 7f01a6a00700 10 client.26909 handle_command_reply: tid=0 2024-05-20T20:12:30.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.841+0000 7f01d2083b80 1 client.26909 shutdown 2024-05-20T20:12:30.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5bdf30 msgr2=0x55cb0f5c1900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c1900 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f01ac005ce0 tx=0x7f01ac005d10 comp rx=0 tx=0).stop 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 shutdown_connections 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f5d1d90 0x55cb0f5d41b0 crc :-1 s=READY pgs=163 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01a6a00700 0 client.26909 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5c1e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c4560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3017799837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c1900 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.845+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 >> 172.21.15.28:0/3017799837 conn(0x55cb0f4e6c60 msgr2=0x55cb0f5b72d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.847+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 shutdown_connections 2024-05-20T20:12:30.847 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.847+0000 7f01d2083b80 1 -- 172.21.15.28:0/3017799837 wait complete. 2024-05-20T20:12:30.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.847+0000 7f01d2083b80 20 client.26909 trim_cache size 0 max 16384 2024-05-20T20:12:30.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.898+0000 7f01d2083b80 1 Processor -- start 2024-05-20T20:12:30.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.898+0000 7f01d2083b80 1 -- start start 2024-05-20T20:12:30.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.899+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b82e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.899+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5bc0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.899+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5bf4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.899+0000 7f01d2083b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f01c01b4420 con 0x55cb0f5be850 2024-05-20T20:12:30.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.899+0000 7f01d2083b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f01c01b5f60 con 0x55cb0f5b8900 2024-05-20T20:12:30.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.899+0000 7f01d2083b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f01c01b4af0 con 0x55cb0f5bdf30 2024-05-20T20:12:30.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.900+0000 7f01bfe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5bc0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.900+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b82e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.900+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b82e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36270/0 (socket says 172.21.15.28:36270) 2024-05-20T20:12:30.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.900+0000 7f01c4c00700 1 -- 172.21.15.28:0/1766141930 learned_addr learned my addr 172.21.15.28:0/1766141930 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:30.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.901+0000 7f01bfe00700 1 -- 172.21.15.28:0/1766141930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 msgr2=0x55cb0f5b82e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.901+0000 7f01bfe00700 1 --2- 172.21.15.28:0/1766141930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b82e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.901+0000 7f01bfe00700 1 -- 172.21.15.28:0/1766141930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 msgr2=0x55cb0f5bf4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.901+0000 7f01bfe00700 1 --2- 172.21.15.28:0/1766141930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5bf4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.901+0000 7f01bfe00700 1 -- 172.21.15.28:0/1766141930 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cb0f5b9750 con 0x55cb0f5bdf30 2024-05-20T20:12:30.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.902+0000 7f01bfe00700 1 --2- 172.21.15.28:0/1766141930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5bc0b0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f01ac00fb90 tx=0x7f01ac009c30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=81cb23cf780d9783 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:30.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.906+0000 7f01b1600700 1 -- 172.21.15.28:0/1766141930 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01ac004750 con 0x55cb0f5bdf30 2024-05-20T20:12:30.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.906+0000 7f01b1600700 1 -- 172.21.15.28:0/1766141930 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f01ac0233f0 con 0x55cb0f5bdf30 2024-05-20T20:12:30.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.906+0000 7f01b1600700 1 -- 172.21.15.28:0/1766141930 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01ac023860 con 0x55cb0f5bdf30 2024-05-20T20:12:30.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.907+0000 7f01d2083b80 1 -- 172.21.15.28:0/1766141930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 msgr2=0x55cb0f5bc0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.907+0000 7f01d2083b80 1 --2- 172.21.15.28:0/1766141930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5bc0b0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f01ac00fb90 tx=0x7f01ac009c30 comp rx=0 tx=0).stop 2024-05-20T20:12:30.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.907+0000 7f01d2083b80 1 -- 172.21.15.28:0/1766141930 shutdown_connections 2024-05-20T20:12:30.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.907+0000 7f01d2083b80 1 --2- 172.21.15.28:0/1766141930 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5be850 0x55cb0f5bf4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.907+0000 7f01d2083b80 1 --2- 172.21.15.28:0/1766141930 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5b82e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.907+0000 7f01d2083b80 1 --2- 172.21.15.28:0/1766141930 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5bc0b0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.907+0000 7f01d2083b80 1 -- 172.21.15.28:0/1766141930 >> 172.21.15.28:0/1766141930 conn(0x55cb0f4e6c60 msgr2=0x55cb0f5c0100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:30.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.908+0000 7f01d2083b80 1 -- 172.21.15.28:0/1766141930 shutdown_connections 2024-05-20T20:12:30.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.908+0000 7f01d2083b80 1 -- 172.21.15.28:0/1766141930 wait complete. 2024-05-20T20:12:30.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.911+0000 7f01d2083b80 1 -- start start 2024-05-20T20:12:30.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.911+0000 7f01d2083b80 1 Processor -- start 2024-05-20T20:12:30.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.912+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c7560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.913+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c7aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.913+0000 7f01d2083b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5be850 0x55cb0f5c7fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.913+0000 7f01d2083b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f01c01b5890 con 0x55cb0f5bdf30 2024-05-20T20:12:30.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.913+0000 7f01d2083b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f01c01b51c0 con 0x55cb0f5b8900 2024-05-20T20:12:30.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.913+0000 7f01d2083b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f01c01b2400 con 0x55cb0f5be850 2024-05-20T20:12:30.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.913+0000 7f01d2083b80 10 client.0 _ll_register_callbacks cb 0x55cb0f4b0cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:30.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.914+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c7560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.914+0000 7f01c4c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c7560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36272/0 (socket says 172.21.15.28:36272) 2024-05-20T20:12:30.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.915+0000 7f01c4c00700 1 -- 172.21.15.28:0/3277437775 learned_addr learned my addr 172.21.15.28:0/3277437775 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:30.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.914+0000 7f01c5600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5be850 0x55cb0f5c7fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.915+0000 7f01c4c00700 1 -- 172.21.15.28:0/3277437775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5be850 msgr2=0x55cb0f5c7fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.916+0000 7f01c4c00700 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5be850 0x55cb0f5c7fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.916+0000 7f01c4c00700 1 -- 172.21.15.28:0/3277437775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5bdf30 msgr2=0x55cb0f5c7aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:30.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.916+0000 7f01c4c00700 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c7aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.916+0000 7f01c4c00700 1 -- 172.21.15.28:0/3277437775 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cb0f645420 con 0x55cb0f5b8900 2024-05-20T20:12:30.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.917+0000 7f01c4c00700 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c7560 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f01b4011ab0 tx=0x7f01b40047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=978ff8e37bfcc345 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:30.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.922+0000 7f01a6a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:30.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.922+0000 7f01a6a00700 1 -- 172.21.15.28:0/3277437775 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01b402a8c0 con 0x55cb0f5b8900 2024-05-20T20:12:30.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.922+0000 7f01a6a00700 1 -- 172.21.15.28:0/3277437775 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f01b402aa60 con 0x55cb0f5b8900 2024-05-20T20:12:30.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.926+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55cb0f5125f0 con 0x55cb0f5b8900 2024-05-20T20:12:30.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.931+0000 7f01a6a00700 1 -- 172.21.15.28:0/3277437775 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f01b402ac60 con 0x55cb0f5b8900 2024-05-20T20:12:30.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.931+0000 7f01d2083b80 10 client.26915 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:30.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.931+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cb0f646740 con 0x55cb0f5b8900 2024-05-20T20:12:30.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.932+0000 7f01a6a00700 1 -- 172.21.15.28:0/3277437775 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55cb0f5125f0 con 0x55cb0f5b8900 2024-05-20T20:12:30.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.932+0000 7f01a6a00700 1 -- 172.21.15.28:0/3277437775 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f01b4031070 con 0x55cb0f5b8900 2024-05-20T20:12:30.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.932+0000 7f01d2083b80 10 client.26915 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:30.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.932+0000 7f01d2083b80 10 client.26915 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:30.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.932+0000 7f01d2083b80 20 client.26915 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:30.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.932+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f646e10 0x55cb0f649230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:30.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.932+0000 7f01d2083b80 4 client.26915 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:12:30.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.933+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x55cb0f3892d0 con 0x55cb0f646e10 2024-05-20T20:12:30.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.935+0000 7f01bfe00700 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f646e10 0x55cb0f649230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:30.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.942+0000 7f01bfe00700 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f646e10 0x55cb0f649230 crc :-1 s=READY pgs=164 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=aecee5f9e78028c4 server_cookie=e7c5ef4d48b2255 in_seq=0 out_seq=0 2024-05-20T20:12:30.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.942+0000 7f01a6a00700 10 client.26915 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:30.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.943+0000 7f01a6a00700 1 -- 172.21.15.28:0/3277437775 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x55cb0f3892d0 con 0x55cb0f646e10 2024-05-20T20:12:30.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.943+0000 7f01a6a00700 10 client.26915 handle_command_reply: tid=0 2024-05-20T20:12:30.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.943+0000 7f01d2083b80 1 client.26915 shutdown 2024-05-20T20:12:30.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.944+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 msgr2=0x55cb0f5c7560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.944+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c7560 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f01b4011ab0 tx=0x7f01b40047c0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.945+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 shutdown_connections 2024-05-20T20:12:30.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.945+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cb0f646e10 0x55cb0f649230 crc :-1 s=READY pgs=164 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.945+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cb0f5be850 0x55cb0f5c7fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.945+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cb0f5b8900 0x55cb0f5c7560 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.945+0000 7f01d2083b80 1 --2- 172.21.15.28:0/3277437775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cb0f5bdf30 0x55cb0f5c7aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.945+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 >> 172.21.15.28:0/3277437775 conn(0x55cb0f4e6c60 msgr2=0x55cb0f5bcae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:30.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.946+0000 7f01a6a00700 0 client.26915 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:30.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.946+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 shutdown_connections 2024-05-20T20:12:30.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.946+0000 7f01d2083b80 1 -- 172.21.15.28:0/3277437775 wait complete. 2024-05-20T20:12:30.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.946+0000 7f01d2083b80 20 client.26915 trim_cache size 0 max 16384 2024-05-20T20:12:30.948 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:12:30.948 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:12:30.949 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "555930d2-0ed7-4fa8-ae0e-df89b52d1b1a", 2024-05-20T20:12:30.949 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:12:30.949 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:12:30.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.949+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a80b2c00 msgr2=0x7f01a80b50c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.949+0000 7f01c7600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a80b2c00 0x7f01a80b50c0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f01b4005950 tx=0x7f01b4009310 comp rx=0 tx=0).stop 2024-05-20T20:12:30.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.949+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01088f0 msgr2=0x7f01c006df50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:30.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.949+0000 7f01c7600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01088f0 0x7f01c006df50 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f01b8009460 tx=0x7f01b800d3b0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 shutdown_connections 2024-05-20T20:12:30.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a80b2c00 0x7f01a80b50c0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01c01088f0 0x7f01c006df50 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01c01037f0 0x7f01c0074c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 --2- 172.21.15.28:0/40010161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01c01041a0 0x7f01c00751b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:30.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 >> 172.21.15.28:0/40010161 conn(0x7f01c007a6a0 msgr2=0x7f01c01079a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:30.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 shutdown_connections 2024-05-20T20:12:30.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:30.950+0000 7f01c7600700 1 -- 172.21.15.28:0/40010161 wait complete. 2024-05-20T20:12:31.164 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:31 smithi028 ceph-mon[31178]: pgmap v211: 97 pgs: 97 active+clean; 37 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 90 MiB/s wr, 2.23k op/s 2024-05-20T20:12:31.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:31 smithi154 ceph-mon[36314]: pgmap v211: 97 pgs: 97 active+clean; 37 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 90 MiB/s wr, 2.23k op/s 2024-05-20T20:12:31.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:31 smithi033 ceph-mon[36179]: pgmap v211: 97 pgs: 97 active+clean; 37 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 90 MiB/s wr, 2.23k op/s 2024-05-20T20:12:32.171 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:12:32.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:32 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:12:32.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:32 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:12:32.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:32 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:12:32.378 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:32 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:12:32.378 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:32 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:12:32.379 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:32 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:12:32.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:32 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:12:32.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:32 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:12:32.379 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:32 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:12:32.935 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:33.315 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:33 smithi028 ceph-mon[31178]: pgmap v212: 97 pgs: 97 active+clean; 37 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 82 MiB/s wr, 1.77k op/s 2024-05-20T20:12:33.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:33 smithi154 ceph-mon[36314]: pgmap v212: 97 pgs: 97 active+clean; 37 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 82 MiB/s wr, 1.77k op/s 2024-05-20T20:12:33.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:33 smithi033 ceph-mon[36179]: pgmap v212: 97 pgs: 97 active+clean; 37 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 82 MiB/s wr, 1.77k op/s 2024-05-20T20:12:34.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:34 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:12:34.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:34 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:12:34.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:34 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:12:34.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:34 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:12:34.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:34 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:12:34.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:34 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:12:35.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:35 smithi154 ceph-mon[36314]: pgmap v213: 97 pgs: 97 active+clean; 38 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 159 MiB/s wr, 2.50k op/s 2024-05-20T20:12:35.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:35 smithi033 ceph-mon[36179]: pgmap v213: 97 pgs: 97 active+clean; 38 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 159 MiB/s wr, 2.50k op/s 2024-05-20T20:12:35.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:35 smithi028 ceph-mon[31178]: pgmap v213: 97 pgs: 97 active+clean; 38 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 159 MiB/s wr, 2.50k op/s 2024-05-20T20:12:36.449 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:36 smithi028 ceph-mon[31178]: pgmap v214: 97 pgs: 97 active+clean; 38 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 139 MiB/s wr, 2.04k op/s 2024-05-20T20:12:36.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:36 smithi154 ceph-mon[36314]: pgmap v214: 97 pgs: 97 active+clean; 38 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 139 MiB/s wr, 2.04k op/s 2024-05-20T20:12:36.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:36 smithi033 ceph-mon[36179]: pgmap v214: 97 pgs: 97 active+clean; 38 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 139 MiB/s wr, 2.04k op/s 2024-05-20T20:12:39.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:38 smithi028 ceph-mon[31178]: pgmap v215: 97 pgs: 97 active+clean; 38 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 152 MiB/s wr, 2.16k op/s 2024-05-20T20:12:39.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:38 smithi154 ceph-mon[36314]: pgmap v215: 97 pgs: 97 active+clean; 38 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 152 MiB/s wr, 2.16k op/s 2024-05-20T20:12:39.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:38 smithi033 ceph-mon[36179]: pgmap v215: 97 pgs: 97 active+clean; 38 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 152 MiB/s wr, 2.16k op/s 2024-05-20T20:12:40.970 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 -- 172.21.15.28:0/3149340246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c004740 msgr2=0x7f519c004b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:40.973 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 --2- 172.21.15.28:0/3149340246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c004740 0x7f519c004b20 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f5198009930 tx=0x7f519802a420 comp rx=0 tx=0).stop 2024-05-20T20:12:40.973 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 -- 172.21.15.28:0/3149340246 shutdown_connections 2024-05-20T20:12:40.973 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 --2- 172.21.15.28:0/3149340246 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f519c005060 0x7f519c0a24c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.974 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 --2- 172.21.15.28:0/3149340246 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c0a2a00 0x7f519c0a4df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.974 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 --2- 172.21.15.28:0/3149340246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c004740 0x7f519c004b20 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.974 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 -- 172.21.15.28:0/3149340246 >> 172.21.15.28:0/3149340246 conn(0x7f519c00a740 msgr2=0x7f519c00ab50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:40.974 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.970+0000 7f51ab000640 1 -- 172.21.15.28:0/3149340246 shutdown_connections 2024-05-20T20:12:40.974 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.971+0000 7f51ab000640 1 -- 172.21.15.28:0/3149340246 wait complete. 2024-05-20T20:12:40.974 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.975+0000 7f51ab000640 1 Processor -- start 2024-05-20T20:12:40.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.975+0000 7f51ab000640 1 -- start start 2024-05-20T20:12:40.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.976+0000 7f51ab000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c004740 0x7f519c097130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.976+0000 7f51ab000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c005060 0x7f519c097670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.976+0000 7f51ab000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f519c0a2a00 0x7f519c097bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.976+0000 7f51ab000640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f519c0a81b0 con 0x7f519c004740 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.976+0000 7f51ab000640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f519c0a8030 con 0x7f519c0a2a00 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.976+0000 7f51ab000640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f519c0a8330 con 0x7f519c005060 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.977+0000 7f51a3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c004740 0x7f519c097130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.977+0000 7f51a3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c004740 0x7f519c097130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56486/0 (socket says 172.21.15.28:56486) 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.977+0000 7f51a3e00640 1 -- 172.21.15.28:0/4180152884 learned_addr learned my addr 172.21.15.28:0/4180152884 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.977+0000 7f51a9000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f519c0a2a00 0x7f519c097bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.977+0000 7f51a3400640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c005060 0x7f519c097670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a9000640 1 -- 172.21.15.28:0/4180152884 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c005060 msgr2=0x7f519c097670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:40.980 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a9000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c005060 0x7f519c097670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a9000640 1 -- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c004740 msgr2=0x7f519c097130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a9000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c004740 0x7f519c097130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a9000640 1 -- 172.21.15.28:0/4180152884 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f519c14fd10 con 0x7f519c0a2a00 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a3e00640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c004740 0x7f519c097130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a9000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f519c0a2a00 0x7f519c097bb0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f51a4068330 tx=0x7f51a406df20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.978+0000 7f51a0c00640 1 -- 172.21.15.28:0/4180152884 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51a406ae30 con 0x7f519c0a2a00 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.979+0000 7f51a0c00640 1 -- 172.21.15.28:0/4180152884 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f51a406d020 con 0x7f519c0a2a00 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.979+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f519c14ffa0 con 0x7f519c0a2a00 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.979+0000 7f51a0c00640 1 -- 172.21.15.28:0/4180152884 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51a406c550 con 0x7f519c0a2a00 2024-05-20T20:12:40.981 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.979+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f519c150470 con 0x7f519c0a2a00 2024-05-20T20:12:40.982 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.983+0000 7f51ade18740 1 Processor -- start 2024-05-20T20:12:40.983 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.984+0000 7f51a0c00640 1 -- 172.21.15.28:0/4180152884 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f51a40742d0 con 0x7f519c0a2a00 2024-05-20T20:12:40.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.984+0000 7f51ade18740 1 -- start start 2024-05-20T20:12:40.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.984+0000 7f51ade18740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa5b250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.984+0000 7f51ade18740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa5b790 0x56388fa65b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51ade18740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa66050 0x56388fa68440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51ade18740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56388f9cbb90 con 0x56388fa66050 2024-05-20T20:12:40.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51ade18740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56388f9bb730 con 0x56388fa5b790 2024-05-20T20:12:40.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51ade18740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56388f9bf770 con 0x56388fa5ae70 2024-05-20T20:12:40.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa5b250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa5b250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39702/0 (socket says 172.21.15.28:39702) 2024-05-20T20:12:40.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa5b790 0x56388fa65b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3400640 1 -- 172.21.15.28:0/2529617099 learned_addr learned my addr 172.21.15.28:0/2529617099 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:40.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3e00640 1 -- 172.21.15.28:0/2529617099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 msgr2=0x56388fa5b250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3e00640 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa5b250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3e00640 1 -- 172.21.15.28:0/2529617099 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa66050 msgr2=0x56388fa68440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3e00640 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa66050 0x56388fa68440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3e00640 1 -- 172.21.15.28:0/2529617099 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56388f9548b0 con 0x56388fa5b790 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.985+0000 7f51a3400640 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa5b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.986+0000 7f51a3e00640 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa5b790 0x56388fa65b10 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f5198004770 tx=0x7f5198004610 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b77ad4e8e47919b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.986+0000 7f51a0c00640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f518c0b29e0 0x7f518c0b4ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.986+0000 7f51a0c00640 1 -- 172.21.15.28:0/4180152884 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f51a4146540 con 0x7f519c0a2a00 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.986+0000 7f5195600640 1 -- 172.21.15.28:0/2529617099 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5198040070 con 0x56388fa5b790 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.986+0000 7f5195600640 1 -- 172.21.15.28:0/2529617099 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5198005020 con 0x56388fa5b790 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.986+0000 7f5195600640 1 -- 172.21.15.28:0/2529617099 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5198035040 con 0x56388fa5b790 2024-05-20T20:12:40.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51a3400640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f518c0b29e0 0x7f518c0b4ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51ade18740 1 -- 172.21.15.28:0/2529617099 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa5b790 msgr2=0x56388fa65b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:40.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51ade18740 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa5b790 0x56388fa65b10 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f5198004770 tx=0x7f5198004610 comp rx=0 tx=0).stop 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51ade18740 1 -- 172.21.15.28:0/2529617099 shutdown_connections 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51ade18740 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa5b790 0x56388fa65b10 secure :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f5198004770 tx=0x7f5198004610 comp rx=0 tx=0).stop 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51ade18740 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa66050 0x56388fa68440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51ade18740 1 --2- 172.21.15.28:0/2529617099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa5b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.987+0000 7f51ade18740 1 -- 172.21.15.28:0/2529617099 >> 172.21.15.28:0/2529617099 conn(0x56388fa57380 msgr2=0x56388fa57760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.988+0000 7f51a3400640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f518c0b29e0 0x7f518c0b4ea0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f5190005d00 tx=0x7f5190005c70 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.989+0000 7f51ade18740 1 -- 172.21.15.28:0/2529617099 shutdown_connections 2024-05-20T20:12:40.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.989+0000 7f51ade18740 1 -- 172.21.15.28:0/2529617099 wait complete. 2024-05-20T20:12:40.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.990+0000 7f51ade18740 1 -- start start 2024-05-20T20:12:40.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.990+0000 7f51ade18740 1 Processor -- start 2024-05-20T20:12:40.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51ade18740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa62ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51ade18740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa66050 0x56388fa63410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51ade18740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa63950 0x56388fa71f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51ade18740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56388f6a1790 con 0x56388fa63950 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51ade18740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56388f963bc0 con 0x56388fa66050 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51ade18740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56388fa68c20 con 0x56388fa5ae70 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51ade18740 10 client.0 _ll_register_callbacks cb 0x56388f941400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51a3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa66050 0x56388fa63410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51a3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa66050 0x56388fa63410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45300/0 (socket says 172.21.15.28:45300) 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51a3400640 1 -- 172.21.15.28:0/1029248756 learned_addr learned my addr 172.21.15.28:0/1029248756 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:40.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.991+0000 7f51a9000640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa63950 0x56388fa71f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.992+0000 7f51a3e00640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa62ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.992+0000 7f51a3400640 1 -- 172.21.15.28:0/1029248756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 msgr2=0x56388fa62ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:40.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.992+0000 7f51a3400640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa62ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.992+0000 7f51a3400640 1 -- 172.21.15.28:0/1029248756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa63950 msgr2=0x56388fa71f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:40.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.992+0000 7f51a3400640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa63950 0x56388fa71f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:40.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.992+0000 7f51a3400640 1 -- 172.21.15.28:0/1029248756 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56388f6d0210 con 0x56388fa66050 2024-05-20T20:12:40.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.992+0000 7f51a3e00640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa62ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:40.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.993+0000 7f51a3400640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa66050 0x56388fa63410 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f519000c010 tx=0x7f5190004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8faed73b00afc6fb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:40.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.993+0000 7f518a000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:40.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.993+0000 7f518a000640 1 -- 172.21.15.28:0/1029248756 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5190024070 con 0x56388fa66050 2024-05-20T20:12:40.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.993+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5198005020 con 0x56388fa66050 2024-05-20T20:12:40.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.993+0000 7f518a000640 1 -- 172.21.15.28:0/1029248756 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f5190020810 con 0x56388fa66050 2024-05-20T20:12:40.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.994+0000 7f518a000640 1 -- 172.21.15.28:0/1029248756 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5190030050 con 0x56388fa66050 2024-05-20T20:12:40.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.994+0000 7f51ade18740 10 client.26927 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:40.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.994+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56388f79c620 con 0x56388fa66050 2024-05-20T20:12:40.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.994+0000 7f518a000640 1 -- 172.21.15.28:0/1029248756 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5198005020 con 0x56388fa66050 2024-05-20T20:12:40.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.995+0000 7f518a000640 1 -- 172.21.15.28:0/1029248756 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5190010a60 con 0x56388fa66050 2024-05-20T20:12:40.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.995+0000 7f51ade18740 10 client.26927 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:40.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.995+0000 7f51ade18740 10 client.26927 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:40.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.995+0000 7f51ade18740 20 client.26927 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:40.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.995+0000 7f51ade18740 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fa760c0 0x56388fa78550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:40.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.995+0000 7f51ade18740 4 client.26927 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:40.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.995+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56388f9cdae0 con 0x56388fa760c0 2024-05-20T20:12:40.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.996+0000 7f51a3e00640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fa760c0 0x56388fa78550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:40.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.997+0000 7f51a3e00640 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fa760c0 0x56388fa78550 crc :-1 s=READY pgs=165 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cde4836a7350dff8 server_cookie=13aa00b0ad7eda57 in_seq=0 out_seq=0 2024-05-20T20:12:40.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.997+0000 7f518a000640 10 client.26927 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:40.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:40.999+0000 7f518a000640 1 -- 172.21.15.28:0/1029248756 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56388f9cdae0 con 0x56388fa760c0 2024-05-20T20:12:40.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.000+0000 7f518a000640 10 client.26927 handle_command_reply: tid=0 2024-05-20T20:12:40.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.000+0000 7f51ade18740 1 client.26927 shutdown 2024-05-20T20:12:41.000 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.000+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa66050 msgr2=0x56388fa63410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.000 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.000+0000 7f51ade18740 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa66050 0x56388fa63410 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f519000c010 tx=0x7f5190004730 comp rx=0 tx=0).stop 2024-05-20T20:12:41.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.001+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 shutdown_connections 2024-05-20T20:12:41.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.001+0000 7f51ade18740 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fa760c0 0x56388fa78550 crc :-1 s=READY pgs=165 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.001+0000 7f518a000640 0 client.26927 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:41.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.001+0000 7f51ade18740 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa66050 0x56388fa63410 secure :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f519000c010 tx=0x7f5190004730 comp rx=0 tx=0).stop 2024-05-20T20:12:41.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.001+0000 7f51ade18740 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa63950 0x56388fa71f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.001+0000 7f51ade18740 1 --2- 172.21.15.28:0/1029248756 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa62ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.001 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.001+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 >> 172.21.15.28:0/1029248756 conn(0x56388fa57380 msgr2=0x56388fa5d100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.002+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 shutdown_connections 2024-05-20T20:12:41.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.002+0000 7f51ade18740 1 -- 172.21.15.28:0/1029248756 wait complete. 2024-05-20T20:12:41.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.002+0000 7f51ade18740 20 client.26927 trim_cache size 0 max 16384 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 Processor -- start 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 -- start start 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa61e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa63950 0x56388fa71cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa760c0 0x56388fa721f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f519c158e80 con 0x56388fa760c0 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f519c15a9c0 con 0x56388fa63950 2024-05-20T20:12:41.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51ade18740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f519c159550 con 0x56388fa5ae70 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.025+0000 7f51a3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa63950 0x56388fa71cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa63950 0x56388fa71cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45310/0 (socket says 172.21.15.28:45310) 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 -- 172.21.15.28:0/730496458 learned_addr learned my addr 172.21.15.28:0/730496458 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a9000640 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa760c0 0x56388fa721f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3e00640 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa61e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 -- 172.21.15.28:0/730496458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 msgr2=0x56388fa61e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa61e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 -- 172.21.15.28:0/730496458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa760c0 msgr2=0x56388fa721f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa760c0 0x56388fa721f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 -- 172.21.15.28:0/730496458 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56388fa61420 con 0x56388fa63950 2024-05-20T20:12:41.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.026+0000 7f51a3400640 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa63950 0x56388fa71cb0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f5190010fd0 tx=0x7f519000b870 comp rx=0 tx=0).ready entity=mon.2 client_cookie=24d052b13e892339 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.027+0000 7f5194c00640 1 -- 172.21.15.28:0/730496458 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5190024040 con 0x56388fa63950 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.027+0000 7f5194c00640 1 -- 172.21.15.28:0/730496458 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5190004030 con 0x56388fa63950 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.027+0000 7f5194c00640 1 -- 172.21.15.28:0/730496458 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5190028620 con 0x56388fa63950 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.028+0000 7f51ade18740 1 -- 172.21.15.28:0/730496458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa63950 msgr2=0x56388fa71cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.028+0000 7f51ade18740 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa63950 0x56388fa71cb0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f5190010fd0 tx=0x7f519000b870 comp rx=0 tx=0).stop 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.028+0000 7f51ade18740 1 -- 172.21.15.28:0/730496458 shutdown_connections 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.028+0000 7f51ade18740 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa63950 0x56388fa71cb0 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.028+0000 7f51ade18740 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa760c0 0x56388fa721f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.028+0000 7f51ade18740 1 --2- 172.21.15.28:0/730496458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa5ae70 0x56388fa61e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.028+0000 7f51ade18740 1 -- 172.21.15.28:0/730496458 >> 172.21.15.28:0/730496458 conn(0x56388fa57380 msgr2=0x56388fa56320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.030+0000 7f51ade18740 1 -- 172.21.15.28:0/730496458 shutdown_connections 2024-05-20T20:12:41.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.030+0000 7f51ade18740 1 -- 172.21.15.28:0/730496458 wait complete. 2024-05-20T20:12:41.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.030+0000 7f51ade18740 1 -- start start 2024-05-20T20:12:41.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 1 Processor -- start 2024-05-20T20:12:41.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa5ae70 0x56388fa6e090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa63950 0x56388fa6e5d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa760c0 0x56388fa5f720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f519c15a2f0 con 0x56388fa5ae70 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f519c159c20 con 0x56388fa760c0 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f519c156e80 con 0x56388fa63950 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.031+0000 7f51ade18740 10 client.0 _ll_register_callbacks cb 0x7f519000b7c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa5ae70 0x56388fa6e090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa5ae70 0x56388fa6e090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56520/0 (socket says 172.21.15.28:56520) 2024-05-20T20:12:41.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 -- 172.21.15.28:0/4194824413 learned_addr learned my addr 172.21.15.28:0/4194824413 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a9000640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa760c0 0x56388fa5f720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3400640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa63950 0x56388fa6e5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 -- 172.21.15.28:0/4194824413 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa63950 msgr2=0x56388fa6e5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa63950 0x56388fa6e5d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 -- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa760c0 msgr2=0x56388fa5f720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa760c0 0x56388fa5f720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.032+0000 7f51a3e00640 1 -- 172.21.15.28:0/4194824413 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56388fa5fef0 con 0x56388fa5ae70 2024-05-20T20:12:41.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.033+0000 7f51a9000640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa760c0 0x56388fa5f720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:41.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.034+0000 7f51a3e00640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa5ae70 0x56388fa6e090 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f5198005ec0 tx=0x7f5198009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9aa312d0a5c6cd94 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.035+0000 7f518a000640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:12:41.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.036+0000 7f518a000640 1 -- 172.21.15.28:0/4194824413 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5198040040 con 0x56388fa5ae70 2024-05-20T20:12:41.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.037+0000 7f518a000640 1 -- 172.21.15.28:0/4194824413 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f5198054030 con 0x56388fa5ae70 2024-05-20T20:12:41.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.042+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x56388f9c72c0 con 0x56388fa5ae70 2024-05-20T20:12:41.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.042+0000 7f518a000640 1 -- 172.21.15.28:0/4194824413 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5198040040 con 0x56388fa5ae70 2024-05-20T20:12:41.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.042+0000 7f51ade18740 10 client.16590 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:41.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.042+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56388fae0780 con 0x56388fa5ae70 2024-05-20T20:12:41.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.045+0000 7f518a000640 1 -- 172.21.15.28:0/4194824413 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x56388f9c72c0 con 0x56388fa5ae70 2024-05-20T20:12:41.045 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.045+0000 7f518a000640 1 -- 172.21.15.28:0/4194824413 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5198004b10 con 0x56388fa5ae70 2024-05-20T20:12:41.045 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.046+0000 7f51ade18740 10 client.16590 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:41.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.046+0000 7f51ade18740 10 client.16590 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:41.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.046+0000 7f51ade18740 20 client.16590 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:41.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.046+0000 7f51ade18740 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fae0d40 0x56388fae3160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.048+0000 7f51ade18740 4 client.16590 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:12:41.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.048+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56388f9414b0 con 0x56388fae0d40 2024-05-20T20:12:41.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.048+0000 7f51a3400640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fae0d40 0x56388fae3160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.050+0000 7f518a000640 10 client.16590 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:41.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.050+0000 7f51a3400640 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fae0d40 0x56388fae3160 crc :-1 s=READY pgs=166 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a534c26f40c1b7bc server_cookie=d0f43a21e05b9aa7 in_seq=0 out_seq=0 2024-05-20T20:12:41.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:40 smithi028 ceph-mon[31178]: pgmap v216: 97 pgs: 97 active+clean; 39 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 221 MiB/s wr, 2.82k op/s 2024-05-20T20:12:41.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.071+0000 7f518a000640 1 -- 172.21.15.28:0/4194824413 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x56388f9414b0 con 0x56388fae0d40 2024-05-20T20:12:41.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.071+0000 7f518a000640 10 client.16590 handle_command_reply: tid=0 2024-05-20T20:12:41.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.071+0000 7f51ade18740 1 client.16590 shutdown 2024-05-20T20:12:41.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa5ae70 msgr2=0x56388fa6e090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa5ae70 0x56388fa6e090 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7f5198005ec0 tx=0x7f5198009260 comp rx=0 tx=0).stop 2024-05-20T20:12:41.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 shutdown_connections 2024-05-20T20:12:41.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56388fae0d40 0x56388fae3160 crc :-1 s=READY pgs=166 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f518a000640 0 client.16590 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:41.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56388fa760c0 0x56388fa5f720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56388fa63950 0x56388fa6e5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 --2- 172.21.15.28:0/4194824413 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56388fa5ae70 0x56388fa6e090 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.072+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 >> 172.21.15.28:0/4194824413 conn(0x56388fa57380 msgr2=0x56388fa5da30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.073+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 shutdown_connections 2024-05-20T20:12:41.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.073+0000 7f51ade18740 1 -- 172.21.15.28:0/4194824413 wait complete. 2024-05-20T20:12:41.073 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.073+0000 7f51ade18740 20 client.16590 trim_cache size 0 max 16384 2024-05-20T20:12:41.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:12:41.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:12:41.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:12:41.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.075+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f518c0b29e0 msgr2=0x7f518c0b4ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.074 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.075+0000 7f51ab000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f518c0b29e0 0x7f518c0b4ea0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f5190005d00 tx=0x7f5190005c70 comp rx=0 tx=0).stop 2024-05-20T20:12:41.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.075+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f519c0a2a00 msgr2=0x7f519c097bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.075+0000 7f51ab000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f519c0a2a00 0x7f519c097bb0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f51a4068330 tx=0x7f51a406df20 comp rx=0 tx=0).stop 2024-05-20T20:12:41.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 shutdown_connections 2024-05-20T20:12:41.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f519c0a2a00 0x7f519c097bb0 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f518c0b29e0 0x7f518c0b4ea0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f519c005060 0x7f519c097670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 --2- 172.21.15.28:0/4180152884 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f519c004740 0x7f519c097130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 >> 172.21.15.28:0/4180152884 conn(0x7f519c00a740 msgr2=0x7f519c0a3ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 shutdown_connections 2024-05-20T20:12:41.076 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:41.076+0000 7f51ab000640 1 -- 172.21.15.28:0/4180152884 wait complete. 2024-05-20T20:12:41.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:40 smithi154 ceph-mon[36314]: pgmap v216: 97 pgs: 97 active+clean; 39 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 221 MiB/s wr, 2.82k op/s 2024-05-20T20:12:41.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:40 smithi033 ceph-mon[36179]: pgmap v216: 97 pgs: 97 active+clean; 39 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 221 MiB/s wr, 2.82k op/s 2024-05-20T20:12:41.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.631+0000 7fb2ba400700 1 -- 172.21.15.28:0/3002280148 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 msgr2=0x7fb2b4107c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.631+0000 7fb2ba400700 1 --2- 172.21.15.28:0/3002280148 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 0x7fb2b4107c80 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fb2ac00b0a0 tx=0x7fb2ac00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.632+0000 7fb2ba400700 1 -- 172.21.15.28:0/3002280148 shutdown_connections 2024-05-20T20:12:41.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.632+0000 7fb2ba400700 1 --2- 172.21.15.28:0/3002280148 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 0x7fb2b4107c80 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.632+0000 7fb2ba400700 1 --2- 172.21.15.28:0/3002280148 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b41023c0 0x7fb2b41027a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.633+0000 7fb2ba400700 1 --2- 172.21.15.28:0/3002280148 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb2b4102ce0 0x7fb2b41071b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.633+0000 7fb2ba400700 1 -- 172.21.15.28:0/3002280148 >> 172.21.15.28:0/3002280148 conn(0x7fb2b40fde00 msgr2=0x7fb2b4100220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.634+0000 7fb2ba400700 1 -- 172.21.15.28:0/3002280148 shutdown_connections 2024-05-20T20:12:41.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.635+0000 7fb2ba400700 1 -- 172.21.15.28:0/3002280148 wait complete. 2024-05-20T20:12:41.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.636+0000 7fb2ba400700 1 Processor -- start 2024-05-20T20:12:41.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.636+0000 7fb2ba400700 1 -- start start 2024-05-20T20:12:41.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.636+0000 7fb2ba400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb2b41023c0 0x7fb2b41a00f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2ba400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b4102ce0 0x7fb2b41a0630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2ba400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 0x7fb2b41a4970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2ba400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b4119c40 con 0x7fb2b41023c0 2024-05-20T20:12:41.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2ba400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b4119ac0 con 0x7fb2b4102ce0 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2ba400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b4119dc0 con 0x7fb2b41077d0 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b4102ce0 0x7fb2b41a0630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b4102ce0 0x7fb2b41a0630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45336/0 (socket says 172.21.15.28:45336) 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 -- 172.21.15.28:0/2754096411 learned_addr learned my addr 172.21.15.28:0/2754096411 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 0x7fb2b41a4970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b9000700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb2b41023c0 0x7fb2b41a00f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 -- 172.21.15.28:0/2754096411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 msgr2=0x7fb2b41a4970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 0x7fb2b41a4970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 -- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb2b41023c0 msgr2=0x7fb2b41a00f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb2b41023c0 0x7fb2b41a00f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.637+0000 7fb2b3e00700 1 -- 172.21.15.28:0/2754096411 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb2b41a50b0 con 0x7fb2b4102ce0 2024-05-20T20:12:41.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.638+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b4102ce0 0x7fb2b41a0630 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fb2a000d750 tx=0x7fb2a000da60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.638+0000 7fb2b1600700 1 -- 172.21.15.28:0/2754096411 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a0017070 con 0x7fb2b4102ce0 2024-05-20T20:12:41.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.638+0000 7fb2b1600700 1 -- 172.21.15.28:0/2754096411 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb2a000fd40 con 0x7fb2b4102ce0 2024-05-20T20:12:41.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.638+0000 7fb2b1600700 1 -- 172.21.15.28:0/2754096411 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a000e350 con 0x7fb2b4102ce0 2024-05-20T20:12:41.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.638+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb2b41a53a0 con 0x7fb2b4102ce0 2024-05-20T20:12:41.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.639+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb2b41acc20 con 0x7fb2b4102ce0 2024-05-20T20:12:41.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.650+0000 7fb2b1600700 1 -- 172.21.15.28:0/2754096411 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb2a000e4f0 con 0x7fb2b4102ce0 2024-05-20T20:12:41.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.650+0000 7fb2c63e1b80 1 Processor -- start 2024-05-20T20:12:41.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.651+0000 7fb2c63e1b80 1 -- start start 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.651+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1020 0x5576ae3d1400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.652+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1940 0x5576ae3cb4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.652+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbf30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.652+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5576ae329e40 con 0x5576ae3cbad0 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.652+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5576ae3199e0 con 0x5576ae3d1020 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.652+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5576ae31da20 con 0x5576ae3d1940 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.652+0000 7fb2b9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1940 0x5576ae3cb4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1940 0x5576ae3cb4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39740/0 (socket says 172.21.15.28:39740) 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9000700 1 -- 172.21.15.28:0/2087998360 learned_addr learned my addr 172.21.15.28:0/2087998360 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b1600700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb29c0b2af0 0x7fb29c0b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b1600700 1 -- 172.21.15.28:0/2754096411 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fb2a00e24a0 con 0x7fb2b4102ce0 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9000700 1 -- 172.21.15.28:0/2087998360 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1020 msgr2=0x5576ae3d1400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.652+0000 7fb2b9a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1020 0x5576ae3d1400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9000700 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1020 0x5576ae3d1400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9000700 1 -- 172.21.15.28:0/2087998360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 msgr2=0x5576ae3cbf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.653+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9000700 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9000700 1 -- 172.21.15.28:0/2087998360 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5576ae2e2420 con 0x5576ae3d1940 2024-05-20T20:12:41.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1020 0x5576ae3d1400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.655+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.656+0000 7fb2b9000700 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1940 0x5576ae3cb4b0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7fb2a8009d30 tx=0x7fb2a80116c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2893c13f5699c63 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.656+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb29c0b2af0 0x7fb29c0b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.657+0000 7fb2a6000700 1 -- 172.21.15.28:0/2087998360 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a8012450 con 0x5576ae3d1940 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.657+0000 7fb2a6000700 1 -- 172.21.15.28:0/2087998360 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb2a8012aa0 con 0x5576ae3d1940 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.658+0000 7fb2a6000700 1 -- 172.21.15.28:0/2087998360 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a800f920 con 0x5576ae3d1940 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.658+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2087998360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1940 msgr2=0x5576ae3cb4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.658+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1940 0x5576ae3cb4b0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7fb2a8009d30 tx=0x7fb2a80116c0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.658+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb29c0b2af0 0x7fb29c0b4fb0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fb2ac00b780 tx=0x7fb2ac0099e0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.659+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2087998360 shutdown_connections 2024-05-20T20:12:41.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.659+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.659+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1020 0x5576ae3d1400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.659+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2087998360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1940 0x5576ae3cb4b0 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.659+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2087998360 >> 172.21.15.28:0/2087998360 conn(0x5576ae2f9be0 msgr2=0x5576ae3c6150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.659+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2087998360 shutdown_connections 2024-05-20T20:12:41.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.659+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2087998360 wait complete. 2024-05-20T20:12:41.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.660+0000 7fb2c63e1b80 1 -- start start 2024-05-20T20:12:41.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.660+0000 7fb2c63e1b80 1 Processor -- start 2024-05-20T20:12:41.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.661+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3d74e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.661+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d7a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3d4a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5576ae21ffa0 con 0x5576ae3cbad0 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5576ae2ba760 con 0x5576ae3d1940 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5576ae3d8500 con 0x5576ae3d1020 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2c63e1b80 10 client.0 _ll_register_callbacks cb 0x5576ae2c3c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d7a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d7a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39748/0 (socket says 172.21.15.28:39748) 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.662+0000 7fb2b3e00700 1 -- 172.21.15.28:0/534657541 learned_addr learned my addr 172.21.15.28:0/534657541 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:41.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.663+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3d4a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.663+0000 7fb2b9000700 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3d74e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.663+0000 7fb2b3e00700 1 -- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 msgr2=0x5576ae3d4a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.663+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3d4a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.663+0000 7fb2b3e00700 1 -- 172.21.15.28:0/534657541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 msgr2=0x5576ae3d74e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.663+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3d74e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.663+0000 7fb2b3e00700 1 -- 172.21.15.28:0/534657541 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5576ae2698d0 con 0x5576ae3d1020 2024-05-20T20:12:41.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.664+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d7a20 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fb2a00a6b40 tx=0x7fb2a000fcb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5549d2836e118298 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.664+0000 7fb29aa00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:41.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.664+0000 7fb29aa00700 1 -- 172.21.15.28:0/534657541 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a0020070 con 0x5576ae3d1020 2024-05-20T20:12:41.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.664+0000 7fb29aa00700 1 -- 172.21.15.28:0/534657541 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fb2a00b22f0 con 0x5576ae3d1020 2024-05-20T20:12:41.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.664+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb2a8012aa0 con 0x5576ae3d1020 2024-05-20T20:12:41.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.665+0000 7fb29aa00700 1 -- 172.21.15.28:0/534657541 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb2a00a93e0 con 0x5576ae3d1020 2024-05-20T20:12:41.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.666+0000 7fb2c63e1b80 10 client.26842 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:41.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.666+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5576ae2ea060 con 0x5576ae3d1020 2024-05-20T20:12:41.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.667+0000 7fb29aa00700 1 -- 172.21.15.28:0/534657541 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb2a8012aa0 con 0x5576ae3d1020 2024-05-20T20:12:41.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.668+0000 7fb29aa00700 1 -- 172.21.15.28:0/534657541 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb2a00aac30 con 0x5576ae3d1020 2024-05-20T20:12:41.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.669+0000 7fb2c63e1b80 10 client.26842 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:41.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.669+0000 7fb2c63e1b80 10 client.26842 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:41.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.669+0000 7fb2c63e1b80 20 client.26842 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:41.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.669+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae3e4d30 0x5576ae3e7120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.670+0000 7fb2c63e1b80 4 client.26842 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:41.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.670+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5576ae32bd90 con 0x5576ae3e4d30 2024-05-20T20:12:41.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.670+0000 7fb2b9000700 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae3e4d30 0x5576ae3e7120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.676+0000 7fb2b9000700 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae3e4d30 0x5576ae3e7120 crc :-1 s=READY pgs=167 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b4d0b2390f897623 server_cookie=c5db81fc84fb23ee in_seq=0 out_seq=0 2024-05-20T20:12:41.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.676+0000 7fb29aa00700 10 client.26842 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:41.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.677+0000 7fb29aa00700 1 -- 172.21.15.28:0/534657541 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5576ae32bd90 con 0x5576ae3e4d30 2024-05-20T20:12:41.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.677+0000 7fb29aa00700 10 client.26842 handle_command_reply: tid=0 2024-05-20T20:12:41.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.677+0000 7fb2c63e1b80 1 client.26842 shutdown 2024-05-20T20:12:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.678+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 msgr2=0x5576ae3d7a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.679+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d7a20 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fb2a00a6b40 tx=0x7fb2a000fcb0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.679+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 shutdown_connections 2024-05-20T20:12:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.679+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae3e4d30 0x5576ae3e7120 crc :-1 s=READY pgs=167 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.679+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3d4a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.679+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3cbad0 0x5576ae3d74e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.679+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/534657541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d7a20 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.679+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 >> 172.21.15.28:0/534657541 conn(0x5576ae2f9be0 msgr2=0x5576ae3ca4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.680+0000 7fb29aa00700 0 client.26842 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.680+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 shutdown_connections 2024-05-20T20:12:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.681+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/534657541 wait complete. 2024-05-20T20:12:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.681+0000 7fb2c63e1b80 20 client.26842 trim_cache size 0 max 16384 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 Processor -- start 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 -- start start 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d1de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3d1940 0x5576ae3d2320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b41b4410 con 0x5576ae3d1940 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b41b5f50 con 0x5576ae3cbad0 2024-05-20T20:12:41.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.722+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b41b4ae0 con 0x5576ae3d1020 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d1de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbeb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:45360/0 (socket says 172.21.15.28:45360) 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b9000700 1 -- 172.21.15.28:0/1569512674 learned_addr learned my addr 172.21.15.28:0/1569512674 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d1de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39752/0 (socket says 172.21.15.28:39752) 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3d1940 0x5576ae3d2320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 -- 172.21.15.28:0/1569512674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3cbad0 msgr2=0x5576ae3cbeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 -- 172.21.15.28:0/1569512674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3d1940 msgr2=0x5576ae3d2320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3d1940 0x5576ae3d2320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 -- 172.21.15.28:0/1569512674 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5576ae3d2860 con 0x5576ae3d1020 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3d1940 0x5576ae3d2320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.723+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d1de0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fb2a000de60 tx=0x7fb2a00a9170 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1abfde956a5d34ce server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.724+0000 7fb2a5600700 1 -- 172.21.15.28:0/1569512674 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a0020070 con 0x5576ae3d1020 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.724+0000 7fb2a5600700 1 -- 172.21.15.28:0/1569512674 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb2a00b22f0 con 0x5576ae3d1020 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.724+0000 7fb2a5600700 1 -- 172.21.15.28:0/1569512674 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a00beb60 con 0x5576ae3d1020 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/1569512674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 msgr2=0x5576ae3d1de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d1de0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fb2a000de60 tx=0x7fb2a00a9170 comp rx=0 tx=0).stop 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/1569512674 shutdown_connections 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3d1940 0x5576ae3d2320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3cbad0 0x5576ae3cbeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/1569512674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3d1020 0x5576ae3d1de0 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/1569512674 >> 172.21.15.28:0/1569512674 conn(0x5576ae2f9be0 msgr2=0x5576ae3c5c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/1569512674 shutdown_connections 2024-05-20T20:12:41.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/1569512674 wait complete. 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.725+0000 7fb2c63e1b80 1 -- start start 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.726+0000 7fb2c63e1b80 1 Processor -- start 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.727+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3cbad0 0x5576ae3dd810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.727+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3ddd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.727+0000 7fb2c63e1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3da7e0 0x5576ae3dac60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.727+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b41b5880 con 0x5576ae3da7e0 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.727+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b41b51b0 con 0x5576ae3d1940 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.727+0000 7fb2c63e1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb2b41b23d0 con 0x5576ae3cbad0 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.727+0000 7fb2c63e1b80 10 client.0 _ll_register_callbacks cb 0x5576ae2c3c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.728+0000 7fb2b3e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3ddd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.728+0000 7fb2b9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3cbad0 0x5576ae3dd810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.728+0000 7fb2b9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3cbad0 0x5576ae3dd810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39768/0 (socket says 172.21.15.28:39768) 2024-05-20T20:12:41.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.728+0000 7fb2b9000700 1 -- 172.21.15.28:0/2514499200 learned_addr learned my addr 172.21.15.28:0/2514499200 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:41.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.729+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3da7e0 0x5576ae3dac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.730+0000 7fb2b9000700 1 -- 172.21.15.28:0/2514499200 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 msgr2=0x5576ae3ddd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.730+0000 7fb2b9000700 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3ddd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.730+0000 7fb2b9000700 1 -- 172.21.15.28:0/2514499200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3da7e0 msgr2=0x5576ae3dac60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.730+0000 7fb2b9000700 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3da7e0 0x5576ae3dac60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.730+0000 7fb2b9000700 1 -- 172.21.15.28:0/2514499200 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5576ae3db1a0 con 0x5576ae3cbad0 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.730+0000 7fb2b9a00700 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3da7e0 0x5576ae3dac60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.731+0000 7fb2b9000700 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3cbad0 0x5576ae3dd810 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7fb2a8009d30 tx=0x7fb2a8009750 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4953dbde9bc1bba1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:41.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.731+0000 7fb29aa00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.731+0000 7fb29aa00700 1 -- 172.21.15.28:0/2514499200 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb2a8004220 con 0x5576ae3cbad0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.731+0000 7fb29aa00700 1 -- 172.21.15.28:0/2514499200 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fb2a80043c0 con 0x5576ae3cbad0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.731+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb2a00b22f0 con 0x5576ae3cbad0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.735+0000 7fb29aa00700 1 -- 172.21.15.28:0/2514499200 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb2a8004920 con 0x5576ae3cbad0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.735+0000 7fb2c63e1b80 10 client.26848 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.735+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5576ae461470 con 0x5576ae3cbad0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb29aa00700 1 -- 172.21.15.28:0/2514499200 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb2a00b22f0 con 0x5576ae3cbad0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb29aa00700 1 -- 172.21.15.28:0/2514499200 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb2a8030070 con 0x5576ae3cbad0 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb2c63e1b80 10 client.26848 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb2c63e1b80 10 client.26848 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb2c63e1b80 20 client.26848 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae4618f0 0x5576ae463d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb2c63e1b80 4 client.26848 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:12:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.736+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x5576ae325570 con 0x5576ae4618f0 2024-05-20T20:12:41.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.751+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae4618f0 0x5576ae463d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:41.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.753+0000 7fb2b3e00700 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae4618f0 0x5576ae463d10 crc :-1 s=READY pgs=168 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9b88e5b7032176fb server_cookie=86fd41c59ddf6999 in_seq=0 out_seq=0 2024-05-20T20:12:41.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.753+0000 7fb29aa00700 10 client.26848 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:41.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.753+0000 7fb29aa00700 1 -- 172.21.15.28:0/2514499200 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x5576ae325570 con 0x5576ae4618f0 2024-05-20T20:12:41.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.753+0000 7fb29aa00700 10 client.26848 handle_command_reply: tid=0 2024-05-20T20:12:41.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.754+0000 7fb2c63e1b80 1 client.26848 shutdown 2024-05-20T20:12:41.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3cbad0 msgr2=0x5576ae3dd810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3cbad0 0x5576ae3dd810 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7fb2a8009d30 tx=0x7fb2a8009750 comp rx=0 tx=0).stop 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 shutdown_connections 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5576ae4618f0 0x5576ae463d10 crc :-1 s=READY pgs=168 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5576ae3da7e0 0x5576ae3dac60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5576ae3cbad0 0x5576ae3dd810 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 --2- 172.21.15.28:0/2514499200 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5576ae3d1940 0x5576ae3ddd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 >> 172.21.15.28:0/2514499200 conn(0x5576ae2f9be0 msgr2=0x5576ae3c5c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.758+0000 7fb29aa00700 0 client.26848 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.759+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 shutdown_connections 2024-05-20T20:12:41.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.759+0000 7fb2c63e1b80 1 -- 172.21.15.28:0/2514499200 wait complete. 2024-05-20T20:12:41.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.759+0000 7fb2c63e1b80 20 client.26848 trim_cache size 0 max 16384 2024-05-20T20:12:41.761 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:12:41.761 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:12:41.761 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:12:41.761 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:12:41.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.760+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb29c0b2af0 msgr2=0x7fb29c0b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.760+0000 7fb2ba400700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb29c0b2af0 0x7fb29c0b4fb0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fb2ac00b780 tx=0x7fb2ac0099e0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.760+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b4102ce0 msgr2=0x7fb2b41a0630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.760+0000 7fb2ba400700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b4102ce0 0x7fb2b41a0630 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fb2a000d750 tx=0x7fb2a000da60 comp rx=0 tx=0).stop 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 shutdown_connections 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb2b41077d0 0x7fb2b41a4970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb2b41023c0 0x7fb2b41a00f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb29c0b2af0 0x7fb29c0b4fb0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 --2- 172.21.15.28:0/2754096411 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb2b4102ce0 0x7fb2b41a0630 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 >> 172.21.15.28:0/2754096411 conn(0x7fb2b40fde00 msgr2=0x7fb2b4105180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 shutdown_connections 2024-05-20T20:12:41.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:41.761+0000 7fb2ba400700 1 -- 172.21.15.28:0/2754096411 wait complete. 2024-05-20T20:12:43.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:42 smithi028 ceph-mon[31178]: pgmap v217: 97 pgs: 97 active+clean; 39 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 207 MiB/s wr, 2.35k op/s 2024-05-20T20:12:43.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:42 smithi154 ceph-mon[36314]: pgmap v217: 97 pgs: 97 active+clean; 39 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 207 MiB/s wr, 2.35k op/s 2024-05-20T20:12:43.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:42 smithi033 ceph-mon[36179]: pgmap v217: 97 pgs: 97 active+clean; 39 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 207 MiB/s wr, 2.35k op/s 2024-05-20T20:12:43.686 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:12:43.687 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:12:44.449 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:45.197 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:44 smithi028 ceph-mon[31178]: pgmap v218: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 293 MiB/s wr, 3.00k op/s 2024-05-20T20:12:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:44 smithi154 ceph-mon[36314]: pgmap v218: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 293 MiB/s wr, 3.00k op/s 2024-05-20T20:12:45.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:44 smithi033 ceph-mon[36179]: pgmap v218: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 293 MiB/s wr, 3.00k op/s 2024-05-20T20:12:47.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:46 smithi028 ceph-mon[31178]: pgmap v219: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 212 MiB/s wr, 2.02k op/s 2024-05-20T20:12:47.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:46 smithi154 ceph-mon[36314]: pgmap v219: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 212 MiB/s wr, 2.02k op/s 2024-05-20T20:12:47.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:46 smithi033 ceph-mon[36179]: pgmap v219: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 212 MiB/s wr, 2.02k op/s 2024-05-20T20:12:49.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.241+0000 7f10a4600700 1 -- 172.21.15.28:0/3983360031 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c11cd20 msgr2=0x7f109c11f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:49.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.241+0000 7f10a4600700 1 --2- 172.21.15.28:0/3983360031 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c11cd20 0x7f109c11f110 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7f109400b0a0 tx=0x7f109400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.250 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:49 smithi028 ceph-mon[31178]: pgmap v220: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 232 MiB/s wr, 2.15k op/s 2024-05-20T20:12:49.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.243+0000 7f10a4600700 1 -- 172.21.15.28:0/3983360031 shutdown_connections 2024-05-20T20:12:49.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.243+0000 7f10a4600700 1 --2- 172.21.15.28:0/3983360031 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c11cd20 0x7f109c11f110 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.243+0000 7f10a4600700 1 --2- 172.21.15.28:0/3983360031 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f109c111d20 0x7f109c10a540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.243+0000 7f10a4600700 1 --2- 172.21.15.28:0/3983360031 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f109c10aa80 0x7f109c10af00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.243+0000 7f10a4600700 1 -- 172.21.15.28:0/3983360031 >> 172.21.15.28:0/3983360031 conn(0x7f109c06e6b0 msgr2=0x7f109c06eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:49.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.247+0000 7f10a4600700 1 -- 172.21.15.28:0/3983360031 shutdown_connections 2024-05-20T20:12:49.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.247+0000 7f10a4600700 1 -- 172.21.15.28:0/3983360031 wait complete. 2024-05-20T20:12:49.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.249+0000 7f10a4600700 1 Processor -- start 2024-05-20T20:12:49.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.264+0000 7f10a4600700 1 -- start start 2024-05-20T20:12:49.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.265+0000 7f10a4600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c10aa80 0x7f109c1ae6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:49.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.265+0000 7f10a4600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f109c111d20 0x7f109c1aec10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:49.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.265+0000 7f10a4600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f109c11cd20 0x7f109c1a8850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:49.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.265+0000 7f10a4600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f109c1225b0 con 0x7f109c111d20 2024-05-20T20:12:49.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.265+0000 7f10a4600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f109c122430 con 0x7f109c10aa80 2024-05-20T20:12:49.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.265+0000 7f10a4600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f109c122730 con 0x7f109c11cd20 2024-05-20T20:12:49.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.271+0000 7f10a2600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f109c11cd20 0x7f109c1a8850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:49.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.271+0000 7f10a1c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c10aa80 0x7f109c1ae6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:49.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.273+0000 7f10a1c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c10aa80 0x7f109c1ae6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51136/0 (socket says 172.21.15.28:51136) 2024-05-20T20:12:49.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.273+0000 7f10a1c00700 1 -- 172.21.15.28:0/662483055 learned_addr learned my addr 172.21.15.28:0/662483055 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:49.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.272+0000 7f10a2600700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f109c11cd20 0x7f109c1a8850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55630/0 (socket says 172.21.15.28:55630) 2024-05-20T20:12:49.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.273+0000 7f10a1200700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f109c111d20 0x7f109c1aec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:49.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.274+0000 7f10a1200700 1 -- 172.21.15.28:0/662483055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f109c11cd20 msgr2=0x7f109c1a8850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:49.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.274+0000 7f10a1200700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f109c11cd20 0x7f109c1a8850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.275+0000 7f10a1200700 1 -- 172.21.15.28:0/662483055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c10aa80 msgr2=0x7f109c1ae6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:49.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.275+0000 7f10a1200700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c10aa80 0x7f109c1ae6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.275+0000 7f10a1200700 1 -- 172.21.15.28:0/662483055 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f109c1a9030 con 0x7f109c111d20 2024-05-20T20:12:49.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.276+0000 7f10a1200700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f109c111d20 0x7f109c1aec10 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f108800c970 tx=0x7f108800cc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:49.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.277+0000 7f109a000700 1 -- 172.21.15.28:0/662483055 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1088004a30 con 0x7f109c111d20 2024-05-20T20:12:49.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.277+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f109c1a9320 con 0x7f109c111d20 2024-05-20T20:12:49.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.277+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f109c1b5530 con 0x7f109c111d20 2024-05-20T20:12:49.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.278+0000 7f109a000700 1 -- 172.21.15.28:0/662483055 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f10880054d0 con 0x7f109c111d20 2024-05-20T20:12:49.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.279+0000 7f109a000700 1 -- 172.21.15.28:0/662483055 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1088004a30 con 0x7f109c111d20 2024-05-20T20:12:49.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.281+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f107c005330 con 0x7f109c111d20 2024-05-20T20:12:49.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.286+0000 7f109a000700 1 -- 172.21.15.28:0/662483055 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1088004bd0 con 0x7f109c111d20 2024-05-20T20:12:49.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.289+0000 7f109a000700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f10840b2cf0 0x7f10840b51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:49.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.289+0000 7f109a000700 1 -- 172.21.15.28:0/662483055 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f10880e1f90 con 0x7f109c111d20 2024-05-20T20:12:49.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.301+0000 7f109a000700 1 -- 172.21.15.28:0/662483055 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f1088014030 con 0x7f109c111d20 2024-05-20T20:12:49.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.301+0000 7f10a1c00700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f10840b2cf0 0x7f10840b51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:49.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.307+0000 7f10a1c00700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f10840b2cf0 0x7f10840b51b0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f1090009ca0 tx=0x7f1090009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:49.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:49 smithi154 ceph-mon[36314]: pgmap v220: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 232 MiB/s wr, 2.15k op/s 2024-05-20T20:12:49.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:49 smithi033 ceph-mon[36179]: pgmap v220: 97 pgs: 97 active+clean; 40 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 232 MiB/s wr, 2.15k op/s 2024-05-20T20:12:49.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.904+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f107c005620 con 0x7f109c111d20 2024-05-20T20:12:49.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.906+0000 7f109a000700 1 -- 172.21.15.28:0/662483055 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f10880a9980 con 0x7f109c111d20 2024-05-20T20:12:49.911 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:12:49.911 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:12:49.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.911+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f10840b2cf0 msgr2=0x7f10840b51b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:49.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.911+0000 7f10a4600700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f10840b2cf0 0x7f10840b51b0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f1090009ca0 tx=0x7f1090009260 comp rx=0 tx=0).stop 2024-05-20T20:12:49.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.911+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f109c111d20 msgr2=0x7f109c1aec10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:49.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.911+0000 7f10a4600700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f109c111d20 0x7f109c1aec10 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f108800c970 tx=0x7f108800cc80 comp rx=0 tx=0).stop 2024-05-20T20:12:49.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.913+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 shutdown_connections 2024-05-20T20:12:49.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.913+0000 7f10a4600700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f109c11cd20 0x7f109c1a8850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.913+0000 7f10a4600700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f10840b2cf0 0x7f10840b51b0 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.913+0000 7f10a4600700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f109c10aa80 0x7f109c1ae6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.913+0000 7f10a4600700 1 --2- 172.21.15.28:0/662483055 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f109c111d20 0x7f109c1aec10 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:49.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.913+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 >> 172.21.15.28:0/662483055 conn(0x7f109c06e6b0 msgr2=0x7f109c11ad10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:49.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.914+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 shutdown_connections 2024-05-20T20:12:49.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:49.914+0000 7f10a4600700 1 -- 172.21.15.28:0/662483055 wait complete. 2024-05-20T20:12:49.918 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:12:50.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:50 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/662483055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:50.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:50 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/662483055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:50.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:50 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/662483055' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:50.822 INFO:tasks.quiescer.fs.[cephfs]:Total duration (46.6) is longer than `split_if_longer` (35.0), will split into two consecutive quiesces 2024-05-20T20:12:50.823 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 23.3 2024-05-20T20:12:50.823 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:12:51.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:51 smithi028 ceph-mon[31178]: pgmap v221: 97 pgs: 97 active+clean; 41 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 305 MiB/s wr, 2.48k op/s 2024-05-20T20:12:51.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:51 smithi154 ceph-mon[36314]: pgmap v221: 97 pgs: 97 active+clean; 41 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 305 MiB/s wr, 2.48k op/s 2024-05-20T20:12:51.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:51 smithi033 ceph-mon[36179]: pgmap v221: 97 pgs: 97 active+clean; 41 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 305 MiB/s wr, 2.48k op/s 2024-05-20T20:12:51.754 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:12:51.926 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:53.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:53 smithi028 ceph-mon[31178]: pgmap v222: 97 pgs: 97 active+clean; 41 GiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 242 MiB/s wr, 1.75k op/s 2024-05-20T20:12:53.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:53 smithi154 ceph-mon[36314]: pgmap v222: 97 pgs: 97 active+clean; 41 GiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 242 MiB/s wr, 1.75k op/s 2024-05-20T20:12:53.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:53 smithi033 ceph-mon[36179]: pgmap v222: 97 pgs: 97 active+clean; 41 GiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 242 MiB/s wr, 1.75k op/s 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.290+0000 7fc5bd200640 1 -- 172.21.15.28:0/1078297397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8102a60 msgr2=0x7fc5b8102ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.290+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1078297397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8102a60 0x7fc5b8102ee0 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8009900 tx=0x7fc5a802a420 comp rx=0 tx=0).stop 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.290+0000 7fc5bd200640 1 -- 172.21.15.28:0/1078297397 shutdown_connections 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.290+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1078297397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8102a60 0x7fc5b8102ee0 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.290+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1078297397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 0x7fc5b810f930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.290+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1078297397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8108a70 0x7fc5b8108e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.290+0000 7fc5bd200640 1 -- 172.21.15.28:0/1078297397 >> 172.21.15.28:0/1078297397 conn(0x7fc5b8079ee0 msgr2=0x7fc5b807a2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.291+0000 7fc5bd200640 1 -- 172.21.15.28:0/1078297397 shutdown_connections 2024-05-20T20:12:54.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.291+0000 7fc5bd200640 1 -- 172.21.15.28:0/1078297397 wait complete. 2024-05-20T20:12:54.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.293+0000 7fc5bd200640 1 Processor -- start 2024-05-20T20:12:54.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.293+0000 7fc5bd200640 1 -- start start 2024-05-20T20:12:54.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.294+0000 7fc5bd200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8102a60 0x7fc5b81992a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.295+0000 7fc5bd200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 0x7fc5b81997e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.295+0000 7fc5bd200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8108a70 0x7fc5b819db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.295+0000 7fc5b5400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 0x7fc5b81997e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.295+0000 7fc5b5400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 0x7fc5b81997e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55640/0 (socket says 172.21.15.28:55640) 2024-05-20T20:12:54.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.295+0000 7fc5b5e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8102a60 0x7fc5b81992a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.296+0000 7fc5b5e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8102a60 0x7fc5b81992a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40676/0 (socket says 172.21.15.28:40676) 2024-05-20T20:12:54.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.296+0000 7fc5b5400640 1 -- 172.21.15.28:0/1178149749 learned_addr learned my addr 172.21.15.28:0/1178149749 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:54.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.296+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8112e20 con 0x7fc5b8102a60 2024-05-20T20:12:54.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.297+0000 7fc5b6800640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8108a70 0x7fc5b819db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.297+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8112ca0 con 0x7fc5b8108a70 2024-05-20T20:12:54.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.297+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8112fa0 con 0x7fc5b8103420 2024-05-20T20:12:54.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b5400640 1 -- 172.21.15.28:0/1178149749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8108a70 msgr2=0x7fc5b819db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b5400640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8108a70 0x7fc5b819db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b5400640 1 -- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8102a60 msgr2=0x7fc5b81992a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b5400640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8102a60 0x7fc5b81992a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b5400640 1 -- 172.21.15.28:0/1178149749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc5b819e290 con 0x7fc5b8103420 2024-05-20T20:12:54.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8102a60 0x7fc5b81992a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:54.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b6800640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8108a70 0x7fc5b819db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:54.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.298+0000 7fc5b5400640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 0x7fc5b81997e0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8006010 tx=0x7fc5a803cee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:54.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.299+0000 7fc5aea00640 1 -- 172.21.15.28:0/1178149749 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5a8049070 con 0x7fc5b8103420 2024-05-20T20:12:54.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.299+0000 7fc5aea00640 1 -- 172.21.15.28:0/1178149749 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc5a8031430 con 0x7fc5b8103420 2024-05-20T20:12:54.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.299+0000 7fc5aea00640 1 -- 172.21.15.28:0/1178149749 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5a8035070 con 0x7fc5b8103420 2024-05-20T20:12:54.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.299+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc5b819e520 con 0x7fc5b8103420 2024-05-20T20:12:54.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.299+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc5b8074c80 con 0x7fc5b8103420 2024-05-20T20:12:54.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.304+0000 7fc5c0025740 1 Processor -- start 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.304+0000 7fc5c0025740 1 -- start start 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.305+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b093f070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.305+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b0949930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.305+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b0949e70 0x55c0b094c260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.305+0000 7fc5c0025740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c0b08afb90 con 0x55c0b0949e70 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.305+0000 7fc5c0025740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c0b089f730 con 0x55c0b093f5b0 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5c0025740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c0b08a3770 con 0x55c0b093ec90 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5b6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b093f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5b6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b093f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55656/0 (socket says 172.21.15.28:55656) 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5b6800640 1 -- 172.21.15.28:0/2463943865 learned_addr learned my addr 172.21.15.28:0/2463943865 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b0949930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5b5e00640 1 -- 172.21.15.28:0/2463943865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 msgr2=0x55c0b093f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b093f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.306+0000 7fc5b5e00640 1 -- 172.21.15.28:0/2463943865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b0949e70 msgr2=0x55c0b094c260 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.307+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b0949e70 0x55c0b094c260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.307+0000 7fc5b5e00640 1 -- 172.21.15.28:0/2463943865 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c0b08388b0 con 0x55c0b093f5b0 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.307+0000 7fc5b6800640 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b093f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.307+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b0949930 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7fc5a4009d70 tx=0x7fc5a400e5a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ae5ecfc4ebbc9dff server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.307+0000 7fc5a3400640 1 -- 172.21.15.28:0/2463943865 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5a4017070 con 0x55c0b093f5b0 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.308+0000 7fc5a3400640 1 -- 172.21.15.28:0/2463943865 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc5a4013920 con 0x55c0b093f5b0 2024-05-20T20:12:54.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.308+0000 7fc5aea00640 1 -- 172.21.15.28:0/1178149749 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc5a804c070 con 0x7fc5b8103420 2024-05-20T20:12:54.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5c0025740 1 -- 172.21.15.28:0/2463943865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 msgr2=0x55c0b0949930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5a3400640 1 -- 172.21.15.28:0/2463943865 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5a4017070 con 0x55c0b093f5b0 2024-05-20T20:12:54.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b0949930 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7fc5a4009d70 tx=0x7fc5a400e5a0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5c0025740 1 -- 172.21.15.28:0/2463943865 shutdown_connections 2024-05-20T20:12:54.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b0949930 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b0949e70 0x55c0b094c260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2463943865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b093f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.309+0000 7fc5c0025740 1 -- 172.21.15.28:0/2463943865 >> 172.21.15.28:0/2463943865 conn(0x55c0b093b1a0 msgr2=0x55c0b093b580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:54.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.312+0000 7fc5aea00640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc59c0b2a10 0x7fc59c0b4ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.312+0000 7fc5c0025740 1 -- 172.21.15.28:0/2463943865 shutdown_connections 2024-05-20T20:12:54.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.313+0000 7fc5aea00640 1 -- 172.21.15.28:0/1178149749 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fc5a8106930 con 0x7fc5b8103420 2024-05-20T20:12:54.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.313+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc59c0b2a10 0x7fc59c0b4ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.313+0000 7fc5c0025740 1 -- 172.21.15.28:0/2463943865 wait complete. 2024-05-20T20:12:54.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.314+0000 7fc5c0025740 1 -- start start 2024-05-20T20:12:54.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.315+0000 7fc5c0025740 1 Processor -- start 2024-05-20T20:12:54.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.316+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b095a750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.316+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 0x55c0b095ac90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.316+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b095b1d0 0x55c0b0948900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.317+0000 7fc5c0025740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c0b0585790 con 0x55c0b093f5b0 2024-05-20T20:12:54.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.317+0000 7fc5c0025740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c0b0847bc0 con 0x55c0b095b1d0 2024-05-20T20:12:54.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.317+0000 7fc5c0025740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c0b094c9e0 con 0x55c0b093ec90 2024-05-20T20:12:54.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.317+0000 7fc5c0025740 10 client.0 _ll_register_callbacks cb 0x55c0b0825400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:54.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.317+0000 7fc5b6800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b095b1d0 0x55c0b0948900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.317+0000 7fc5b6800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b095b1d0 0x55c0b0948900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51170/0 (socket says 172.21.15.28:51170) 2024-05-20T20:12:54.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.317+0000 7fc5b6800640 1 -- 172.21.15.28:0/3639046773 learned_addr learned my addr 172.21.15.28:0/3639046773 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:54.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.318+0000 7fc5b6800640 1 -- 172.21.15.28:0/3639046773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 msgr2=0x55c0b095a750 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:12:54.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.318+0000 7fc5b6800640 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b095a750 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.318+0000 7fc5b6800640 1 -- 172.21.15.28:0/3639046773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 msgr2=0x55c0b095ac90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:54.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.318+0000 7fc5b6800640 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 0x55c0b095ac90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.318+0000 7fc5b6800640 1 -- 172.21.15.28:0/3639046773 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c0b05b4210 con 0x55c0b095b1d0 2024-05-20T20:12:54.320 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.320+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc5a4013920 con 0x55c0b095b1d0 2024-05-20T20:12:54.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.320+0000 7fc597e00640 10 client.26963 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:54.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.320+0000 7fc5b6800640 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b095b1d0 0x55c0b0948900 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7fc5b000db70 tx=0x7fc5b000b700 comp rx=0 tx=0).ready entity=mon.2 client_cookie=19a99b4a357e5f03 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:54.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.321+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc59c0b2a10 0x7fc59c0b4ed0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fc5a400efd0 tx=0x7fc5a4009900 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:54.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.321+0000 7fc597e00640 1 -- 172.21.15.28:0/3639046773 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5b001b060 con 0x55c0b095b1d0 2024-05-20T20:12:54.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.321+0000 7fc597e00640 1 -- 172.21.15.28:0/3639046773 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fc5b0016640 con 0x55c0b095b1d0 2024-05-20T20:12:54.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.321+0000 7fc597e00640 1 -- 172.21.15.28:0/3639046773 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc5b0016880 con 0x55c0b095b1d0 2024-05-20T20:12:54.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.322+0000 7fc5c0025740 10 client.26963 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:54.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.322+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x55c0b0680620 con 0x55c0b095b1d0 2024-05-20T20:12:54.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.328+0000 7fc597e00640 1 -- 172.21.15.28:0/3639046773 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc5b000e420 con 0x55c0b095b1d0 2024-05-20T20:12:54.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.329+0000 7fc5c0025740 10 client.26963 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:54.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.329+0000 7fc5c0025740 10 client.26963 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:54.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.329+0000 7fc5c0025740 20 client.26963 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:54.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.329+0000 7fc5c0025740 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b0962530 0x55c0b09649b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.329+0000 7fc5c0025740 4 client.26963 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:54.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.329+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c0b08b1ae0 con 0x55c0b0962530 2024-05-20T20:12:54.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.333+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b0962530 0x55c0b09649b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.341 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.336+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b0962530 0x55c0b09649b0 crc :-1 s=READY pgs=169 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=590d4bbb0b99f6ff server_cookie=618e75129917fb29 in_seq=0 out_seq=0 2024-05-20T20:12:54.341 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.337+0000 7fc597e00640 10 client.26963 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:54.341 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.341+0000 7fc597e00640 1 -- 172.21.15.28:0/3639046773 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55c0b08b1ae0 con 0x55c0b0962530 2024-05-20T20:12:54.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.341+0000 7fc597e00640 10 client.26963 handle_command_reply: tid=0 2024-05-20T20:12:54.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.341+0000 7fc5c0025740 1 client.26963 shutdown 2024-05-20T20:12:54.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.341+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b095b1d0 msgr2=0x55c0b0948900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.341+0000 7fc5c0025740 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b095b1d0 0x55c0b0948900 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7fc5b000db70 tx=0x7fc5b000b700 comp rx=0 tx=0).stop 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 shutdown_connections 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc5c0025740 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b0962530 0x55c0b09649b0 crc :-1 s=READY pgs=169 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc597e00640 0 client.26963 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc5c0025740 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b095b1d0 0x55c0b0948900 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc5c0025740 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 0x55c0b095ac90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc5c0025740 1 --2- 172.21.15.28:0/3639046773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b095a750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 >> 172.21.15.28:0/3639046773 conn(0x55c0b093b1a0 msgr2=0x55c0b0940fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.342+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 shutdown_connections 2024-05-20T20:12:54.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.343+0000 7fc5c0025740 1 -- 172.21.15.28:0/3639046773 wait complete. 2024-05-20T20:12:54.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.345+0000 7fc5c0025740 20 client.26963 trim_cache size 0 max 16384 2024-05-20T20:12:54.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.411+0000 7fc5c0025740 1 Processor -- start 2024-05-20T20:12:54.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.416+0000 7fc5c0025740 1 -- start start 2024-05-20T20:12:54.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.418+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093ec90 0x55c0b096a2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.418+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 0x55c0b09487f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.418+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b095b1d0 0x55c0b096a800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.418+0000 7fc5c0025740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8072bf0 con 0x55c0b093f5b0 2024-05-20T20:12:54.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.418+0000 7fc5c0025740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8074730 con 0x55c0b093ec90 2024-05-20T20:12:54.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.418+0000 7fc5c0025740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b80732c0 con 0x55c0b095b1d0 2024-05-20T20:12:54.423 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:12:54.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.421+0000 7fc5b5400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 0x55c0b09487f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.421+0000 7fc5b6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b095b1d0 0x55c0b096a800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.424 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.422+0000 7fc5b6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b095b1d0 0x55c0b096a800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55670/0 (socket says 172.21.15.28:55670) 2024-05-20T20:12:54.424 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.422+0000 7fc5b6800640 1 -- 172.21.15.28:0/2804132160 learned_addr learned my addr 172.21.15.28:0/2804132160 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:54.424 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.422+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093ec90 0x55c0b096a2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.426 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.424+0000 7fc5b6800640 1 -- 172.21.15.28:0/2804132160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093ec90 msgr2=0x55c0b096a2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:54.426 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.424+0000 7fc5b6800640 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093ec90 0x55c0b096a2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.426 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.424+0000 7fc5b6800640 1 -- 172.21.15.28:0/2804132160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 msgr2=0x55c0b09487f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:54.427 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.424+0000 7fc5b6800640 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 0x55c0b09487f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.427 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.424+0000 7fc5b6800640 1 -- 172.21.15.28:0/2804132160 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c0b0958ca0 con 0x55c0b095b1d0 2024-05-20T20:12:54.434 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.434+0000 7fc5b6800640 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b095b1d0 0x55c0b096a800 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7fc5b000bf00 tx=0x7fc5b0010000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1a4097302409e21a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:54.435 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.434+0000 7fc5a2a00640 1 -- 172.21.15.28:0/2804132160 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5b001b040 con 0x55c0b095b1d0 2024-05-20T20:12:54.441 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.435+0000 7fc5a2a00640 1 -- 172.21.15.28:0/2804132160 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc5b002d070 con 0x55c0b095b1d0 2024-05-20T20:12:54.441 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.435+0000 7fc5a2a00640 1 -- 172.21.15.28:0/2804132160 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5b0016440 con 0x55c0b095b1d0 2024-05-20T20:12:54.441 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.440+0000 7fc5c0025740 1 -- 172.21.15.28:0/2804132160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b095b1d0 msgr2=0x55c0b096a800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.441 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.440+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b095b1d0 0x55c0b096a800 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7fc5b000bf00 tx=0x7fc5b0010000 comp rx=0 tx=0).stop 2024-05-20T20:12:54.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.445+0000 7fc5c0025740 1 -- 172.21.15.28:0/2804132160 shutdown_connections 2024-05-20T20:12:54.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.445+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b095b1d0 0x55c0b096a800 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.445+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b093f5b0 0x55c0b09487f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.445+0000 7fc5c0025740 1 --2- 172.21.15.28:0/2804132160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093ec90 0x55c0b096a2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.445+0000 7fc5c0025740 1 -- 172.21.15.28:0/2804132160 >> 172.21.15.28:0/2804132160 conn(0x55c0b093b1a0 msgr2=0x55c0b093b580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:54.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.445+0000 7fc5c0025740 1 -- 172.21.15.28:0/2804132160 shutdown_connections 2024-05-20T20:12:54.447 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.446+0000 7fc5c0025740 1 -- 172.21.15.28:0/2804132160 wait complete. 2024-05-20T20:12:54.447 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.447+0000 7fc5c0025740 1 -- start start 2024-05-20T20:12:54.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.447+0000 7fc5c0025740 1 Processor -- start 2024-05-20T20:12:54.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b09c5490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b09c59d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5c0025740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b095b1d0 0x55c0b0943480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5c0025740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8074060 con 0x55c0b095b1d0 2024-05-20T20:12:54.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5c0025740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8073990 con 0x55c0b093f5b0 2024-05-20T20:12:54.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5c0025740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc5b8070bf0 con 0x55c0b093ec90 2024-05-20T20:12:54.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5c0025740 10 client.0 _ll_register_callbacks cb 0x7fc5b806c2d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:54.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5b5400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b09c59d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5b5400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b09c59d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51182/0 (socket says 172.21.15.28:51182) 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5b5400640 1 -- 172.21.15.28:0/4128743314 learned_addr learned my addr 172.21.15.28:0/4128743314 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.449+0000 7fc5b6800640 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b095b1d0 0x55c0b0943480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc5b5400640 1 -- 172.21.15.28:0/4128743314 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 msgr2=0x55c0b09c5490 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc5b5400640 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b09c5490 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc5b5400640 1 -- 172.21.15.28:0/4128743314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b095b1d0 msgr2=0x55c0b0943480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc5b5400640 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b095b1d0 0x55c0b0943480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc5b5400640 1 -- 172.21.15.28:0/4128743314 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c0b0943b70 con 0x55c0b093f5b0 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc5b6800640 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b095b1d0 0x55c0b0943480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc5b5400640 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b09c59d0 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8003fd0 tx=0x7fc5a80c9bf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6036832eb4e461e0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.450+0000 7fc597e00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:54.452 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.451+0000 7fc597e00640 1 -- 172.21.15.28:0/4128743314 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc5a80c9390 con 0x55c0b093f5b0 2024-05-20T20:12:54.453 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.451+0000 7fc597e00640 1 -- 172.21.15.28:0/4128743314 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7fc5a80318e0 con 0x55c0b093f5b0 2024-05-20T20:12:54.453 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.451+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc5b002d070 con 0x55c0b093f5b0 2024-05-20T20:12:54.455 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.451+0000 7fc597e00640 1 -- 172.21.15.28:0/4128743314 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc5a80cd710 con 0x55c0b093f5b0 2024-05-20T20:12:54.455 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.455+0000 7fc5c0025740 10 client.26969 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:54.455 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.455+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c0b09ccc40 con 0x55c0b093f5b0 2024-05-20T20:12:54.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.455+0000 7fc597e00640 1 -- 172.21.15.28:0/4128743314 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc5b002d070 con 0x55c0b093f5b0 2024-05-20T20:12:54.457 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.457+0000 7fc597e00640 1 -- 172.21.15.28:0/4128743314 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc5a8043030 con 0x55c0b093f5b0 2024-05-20T20:12:54.457 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.457+0000 7fc5c0025740 10 client.26969 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:54.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.457+0000 7fc5c0025740 10 client.26969 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:12:54.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.457+0000 7fc5c0025740 20 client.26969 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:54.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.457+0000 7fc5c0025740 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b09cd310 0x55c0b09cf700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:54.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.457+0000 7fc5c0025740 4 client.26969 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:12:54.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.457+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c0b08ab2c0 con 0x55c0b09cd310 2024-05-20T20:12:54.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.461+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b09cd310 0x55c0b09cf700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:54.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.465+0000 7fc5b5e00640 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b09cd310 0x55c0b09cf700 crc :-1 s=READY pgs=170 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=aacfcb09b4f949a5 server_cookie=842530013a52b661 in_seq=0 out_seq=0 2024-05-20T20:12:54.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.465+0000 7fc597e00640 10 client.26969 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:54.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.486+0000 7fc597e00640 1 -- 172.21.15.28:0/4128743314 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55c0b08ab2c0 con 0x55c0b09cd310 2024-05-20T20:12:54.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.487+0000 7fc597e00640 10 client.26969 handle_command_reply: tid=0 2024-05-20T20:12:54.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.487+0000 7fc5c0025740 1 client.26969 shutdown 2024-05-20T20:12:54.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.487+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 msgr2=0x55c0b09c59d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.487+0000 7fc5c0025740 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b09c59d0 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8003fd0 tx=0x7fc5a80c9bf0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 shutdown_connections 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c0b09cd310 0x55c0b09cf700 crc :-1 s=READY pgs=170 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c0b093f5b0 0x55c0b09c59d0 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c0b095b1d0 0x55c0b0943480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 --2- 172.21.15.28:0/4128743314 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c0b093ec90 0x55c0b09c5490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 >> 172.21.15.28:0/4128743314 conn(0x55c0b093b1a0 msgr2=0x55c0b09554b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc597e00640 0 client.26969 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 shutdown_connections 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 1 -- 172.21.15.28:0/4128743314 wait complete. 2024-05-20T20:12:54.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.488+0000 7fc5c0025740 20 client.26969 trim_cache size 0 max 16384 2024-05-20T20:12:54.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:12:54.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:12:54.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc59c0b2a10 msgr2=0x7fc59c0b4ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc59c0b2a10 0x7fc59c0b4ed0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fc5a400efd0 tx=0x7fc5a4009900 comp rx=0 tx=0).stop 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 msgr2=0x7fc5b81997e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 0x7fc5b81997e0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fc5a8006010 tx=0x7fc5a803cee0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 shutdown_connections 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc5b8108a70 0x7fc5b819db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc59c0b2a10 0x7fc59c0b4ed0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc5b8103420 0x7fc5b81997e0 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 --2- 172.21.15.28:0/1178149749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc5b8102a60 0x7fc5b81992a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:54.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 >> 172.21.15.28:0/1178149749 conn(0x7fc5b8079ee0 msgr2=0x7fc5b81001f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:54.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 shutdown_connections 2024-05-20T20:12:54.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:12:54.489+0000 7fc5bd200640 1 -- 172.21.15.28:0/1178149749 wait complete. 2024-05-20T20:12:55.189 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:55 smithi028 ceph-mon[31178]: pgmap v223: 97 pgs: 97 active+clean; 42 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 327 MiB/s wr, 2.14k op/s 2024-05-20T20:12:55.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:55 smithi154 ceph-mon[36314]: pgmap v223: 97 pgs: 97 active+clean; 42 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 327 MiB/s wr, 2.14k op/s 2024-05-20T20:12:55.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:55 smithi033 ceph-mon[36179]: pgmap v223: 97 pgs: 97 active+clean; 42 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 327 MiB/s wr, 2.14k op/s 2024-05-20T20:12:56.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.654+0000 7ff532800700 1 -- 172.21.15.28:0/1612680190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff52c10ffa0 msgr2=0x7ff52c112390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:56.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.654+0000 7ff532800700 1 --2- 172.21.15.28:0/1612680190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff52c10ffa0 0x7ff52c112390 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7ff52400b0a0 tx=0x7ff52400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:12:56.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.655+0000 7ff532800700 1 -- 172.21.15.28:0/1612680190 shutdown_connections 2024-05-20T20:12:56.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.655+0000 7ff532800700 1 --2- 172.21.15.28:0/1612680190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff52c10ffa0 0x7ff52c112390 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:56.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.655+0000 7ff532800700 1 --2- 172.21.15.28:0/1612680190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff52c075770 0x7ff52c075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:56.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.655+0000 7ff532800700 1 --2- 172.21.15.28:0/1612680190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c076090 0x7ff52c10fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:56.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.655+0000 7ff532800700 1 -- 172.21.15.28:0/1612680190 >> 172.21.15.28:0/1612680190 conn(0x7ff52c07a6a0 msgr2=0x7ff52c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:56.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.656+0000 7ff532800700 1 -- 172.21.15.28:0/1612680190 shutdown_connections 2024-05-20T20:12:56.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.659+0000 7ff532800700 1 -- 172.21.15.28:0/1612680190 wait complete. 2024-05-20T20:12:56.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.661+0000 7ff532800700 1 Processor -- start 2024-05-20T20:12:56.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.662+0000 7ff532800700 1 -- start start 2024-05-20T20:12:56.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.663+0000 7ff532800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff52c075770 0x7ff52c11b8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:56.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.663+0000 7ff532800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff52c076090 0x7ff52c11be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:56.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.664+0000 7ff532800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c10ffa0 0x7ff52c1149d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:56.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.664+0000 7ff532800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff52c1050d0 con 0x7ff52c075770 2024-05-20T20:12:56.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.664+0000 7ff532800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff52c104f50 con 0x7ff52c076090 2024-05-20T20:12:56.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.664+0000 7ff532800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff52c105250 con 0x7ff52c10ffa0 2024-05-20T20:12:56.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.664+0000 7ff52be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c10ffa0 0x7ff52c1149d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:56.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.664+0000 7ff52be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c10ffa0 0x7ff52c1149d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55688/0 (socket says 172.21.15.28:55688) 2024-05-20T20:12:56.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.664+0000 7ff52be00700 1 -- 172.21.15.28:0/2161322424 learned_addr learned my addr 172.21.15.28:0/2161322424 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:56.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.665+0000 7ff52b400700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff52c075770 0x7ff52c11b8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.665+0000 7ff52be00700 1 -- 172.21.15.28:0/2161322424 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff52c076090 msgr2=0x7ff52c11be30 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:12:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.665+0000 7ff52be00700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff52c076090 0x7ff52c11be30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.665+0000 7ff52be00700 1 -- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff52c075770 msgr2=0x7ff52c11b8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.665+0000 7ff52be00700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff52c075770 0x7ff52c11b8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:56.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.665+0000 7ff52be00700 1 -- 172.21.15.28:0/2161322424 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff52c1151b0 con 0x7ff52c10ffa0 2024-05-20T20:12:56.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.666+0000 7ff52be00700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c10ffa0 0x7ff52c1149d0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7ff52400b070 tx=0x7ff524004d30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:56.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.668+0000 7ff523e00700 1 -- 172.21.15.28:0/2161322424 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff524021070 con 0x7ff52c10ffa0 2024-05-20T20:12:56.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.668+0000 7ff523e00700 1 -- 172.21.15.28:0/2161322424 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff524007c40 con 0x7ff52c10ffa0 2024-05-20T20:12:56.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.669+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff52c115440 con 0x7ff52c10ffa0 2024-05-20T20:12:56.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.669+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff52c119420 con 0x7ff52c10ffa0 2024-05-20T20:12:56.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.670+0000 7ff523e00700 1 -- 172.21.15.28:0/2161322424 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff524021070 con 0x7ff52c10ffa0 2024-05-20T20:12:56.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.674+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff50c005330 con 0x7ff52c10ffa0 2024-05-20T20:12:56.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.679+0000 7ff523e00700 1 -- 172.21.15.28:0/2161322424 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ff524004400 con 0x7ff52c10ffa0 2024-05-20T20:12:56.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.682+0000 7ff523e00700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff5100b2cf0 0x7ff5100b51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:56.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.682+0000 7ff523e00700 1 -- 172.21.15.28:0/2161322424 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7ff52400f020 con 0x7ff52c10ffa0 2024-05-20T20:12:56.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.685+0000 7ff523e00700 1 -- 172.21.15.28:0/2161322424 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7ff52402a020 con 0x7ff52c10ffa0 2024-05-20T20:12:56.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.686+0000 7ff52b400700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff5100b2cf0 0x7ff5100b51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:56.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:56.690+0000 7ff52b400700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff5100b2cf0 0x7ff5100b51b0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7ff518009bf0 tx=0x7ff518009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:57.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.221+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7ff50c0058f0 con 0x7ff52c10ffa0 2024-05-20T20:12:57.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.222+0000 7ff523e00700 1 -- 172.21.15.28:0/2161322424 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7ff5240aafa0 con 0x7ff52c10ffa0 2024-05-20T20:12:57.223 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:12:57.223 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:12:57.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.224+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff5100b2cf0 msgr2=0x7ff5100b51b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.224+0000 7ff532800700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff5100b2cf0 0x7ff5100b51b0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7ff518009bf0 tx=0x7ff518009260 comp rx=0 tx=0).stop 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.224+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c10ffa0 msgr2=0x7ff52c1149d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.224+0000 7ff532800700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c10ffa0 0x7ff52c1149d0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7ff52400b070 tx=0x7ff524004d30 comp rx=0 tx=0).stop 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 shutdown_connections 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff52c10ffa0 0x7ff52c1149d0 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff52c075770 0x7ff52c11b8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff5100b2cf0 0x7ff5100b51b0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 --2- 172.21.15.28:0/2161322424 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff52c076090 0x7ff52c11be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 >> 172.21.15.28:0/2161322424 conn(0x7ff52c07a6a0 msgr2=0x7ff52c1007b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 shutdown_connections 2024-05-20T20:12:57.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:57.225+0000 7ff532800700 1 -- 172.21.15.28:0/2161322424 wait complete. 2024-05-20T20:12:57.227 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:12:57.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:57 smithi154 ceph-mon[36314]: pgmap v224: 97 pgs: 97 active+clean; 42 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 240 MiB/s wr, 1.39k op/s 2024-05-20T20:12:57.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:57 smithi033 ceph-mon[36179]: pgmap v224: 97 pgs: 97 active+clean; 42 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 240 MiB/s wr, 1.39k op/s 2024-05-20T20:12:57.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:57 smithi028 ceph-mon[31178]: pgmap v224: 97 pgs: 97 active+clean; 42 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 240 MiB/s wr, 1.39k op/s 2024-05-20T20:12:58.221 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:58 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2161322424' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:58.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:58 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2161322424' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:58.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:58 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2161322424' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:12:59.196 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:12:59 smithi028 ceph-mon[31178]: pgmap v225: 97 pgs: 97 active+clean; 43 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 261 MiB/s wr, 1.48k op/s 2024-05-20T20:12:59.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:12:59 smithi154 ceph-mon[36314]: pgmap v225: 97 pgs: 97 active+clean; 43 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 261 MiB/s wr, 1.48k op/s 2024-05-20T20:12:59.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:12:59 smithi033 ceph-mon[36179]: pgmap v225: 97 pgs: 97 active+clean; 43 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 261 MiB/s wr, 1.48k op/s 2024-05-20T20:12:59.515 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.512+0000 7f0127a00700 1 -- 172.21.15.28:0/93688348 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 msgr2=0x7f012010f2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.515 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.514+0000 7f0127a00700 1 --2- 172.21.15.28:0/93688348 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 0x7f012010f2f0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f011000b0a0 tx=0x7f011000b3b0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.518 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.516+0000 7f0127a00700 1 -- 172.21.15.28:0/93688348 shutdown_connections 2024-05-20T20:12:59.518 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.517+0000 7f0127a00700 1 --2- 172.21.15.28:0/93688348 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 0x7f012010f2f0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.518 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.517+0000 7f0127a00700 1 --2- 172.21.15.28:0/93688348 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0120108520 0x7f0120108900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.518 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.517+0000 7f0127a00700 1 --2- 172.21.15.28:0/93688348 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 0x7f01201028c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.518 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.517+0000 7f0127a00700 1 -- 172.21.15.28:0/93688348 >> 172.21.15.28:0/93688348 conn(0x7f01200793f0 msgr2=0x7f0120079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:59.518 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.517+0000 7f0127a00700 1 -- 172.21.15.28:0/93688348 shutdown_connections 2024-05-20T20:12:59.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.517+0000 7f0127a00700 1 -- 172.21.15.28:0/93688348 wait complete. 2024-05-20T20:12:59.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.519+0000 7f0127a00700 1 Processor -- start 2024-05-20T20:12:59.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.520+0000 7f0127a00700 1 -- start start 2024-05-20T20:12:59.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.521+0000 7f0127a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 0x7f0120198890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.521+0000 7f0127a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 0x7f0120198dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.521+0000 7f0127a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0120108520 0x7f012019d8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.521+0000 7f0125000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 0x7f0120198890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.521+0000 7f0125000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 0x7f0120198890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42696/0 (socket says 172.21.15.28:42696) 2024-05-20T20:12:59.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.522+0000 7f0125000700 1 -- 172.21.15.28:0/1809960350 learned_addr learned my addr 172.21.15.28:0/1809960350 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:59.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.522+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f01201126a0 con 0x7f0120108520 2024-05-20T20:12:59.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.522+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0120112520 con 0x7f0120102440 2024-05-20T20:12:59.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.522+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0120112820 con 0x7f0120102e00 2024-05-20T20:12:59.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.529+0000 7f0125000700 1 -- 172.21.15.28:0/1809960350 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 msgr2=0x7f0120198dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.522+0000 7f011fe00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 0x7f0120198dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.523+0000 7f0125a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0120108520 0x7f012019d8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.530+0000 7f0125000700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 0x7f0120198dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.530+0000 7f0125000700 1 -- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0120108520 msgr2=0x7f012019d8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.530+0000 7f0125000700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0120108520 0x7f012019d8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.530+0000 7f0125000700 1 -- 172.21.15.28:0/1809960350 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f012019de20 con 0x7f0120102440 2024-05-20T20:12:59.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.532+0000 7f011fe00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 0x7f0120198dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:59.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.532+0000 7f0125a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0120108520 0x7f012019d8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:59.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.538+0000 7f0125000700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 0x7f0120198890 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f011800b530 tx=0x7f011800b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:59.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.540+0000 7f011d600700 1 -- 172.21.15.28:0/1809960350 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0118015020 con 0x7f0120102440 2024-05-20T20:12:59.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.540+0000 7f011d600700 1 -- 172.21.15.28:0/1809960350 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0118011ce0 con 0x7f0120102440 2024-05-20T20:12:59.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.540+0000 7f011d600700 1 -- 172.21.15.28:0/1809960350 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0118010d20 con 0x7f0120102440 2024-05-20T20:12:59.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.540+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f012019e110 con 0x7f0120102440 2024-05-20T20:12:59.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.540+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f012019e600 con 0x7f0120102440 2024-05-20T20:12:59.556 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.555+0000 7f011d600700 1 -- 172.21.15.28:0/1809960350 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0118018020 con 0x7f0120102440 2024-05-20T20:12:59.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.559+0000 7f011d600700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01040b2b80 0x7f01040b5040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.559+0000 7f013243fb80 1 Processor -- start 2024-05-20T20:12:59.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.559+0000 7f013243fb80 1 -- start start 2024-05-20T20:12:59.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.560+0000 7f013243fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d1eb0 0x5651333d2290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.560+0000 7f013243fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d27d0 0x5651333cc260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.560+0000 7f013243fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333ccce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.560+0000 7f013243fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56513332ae40 con 0x5651333cc880 2024-05-20T20:12:59.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.560+0000 7f013243fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56513331a9e0 con 0x5651333d27d0 2024-05-20T20:12:59.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.560+0000 7f013243fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56513331ea20 con 0x5651333d1eb0 2024-05-20T20:12:59.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.559+0000 7f011d600700 1 -- 172.21.15.28:0/1809960350 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f01180e1c50 con 0x7f0120102440 2024-05-20T20:12:59.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.561+0000 7f011fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d27d0 0x5651333cc260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.561+0000 7f011fe00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01040b2b80 0x7f01040b5040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.561+0000 7f011fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d27d0 0x5651333cc260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42704/0 (socket says 172.21.15.28:42704) 2024-05-20T20:12:59.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.561+0000 7f011fe00700 1 -- 172.21.15.28:0/2627666760 learned_addr learned my addr 172.21.15.28:0/2627666760 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:59.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.562+0000 7f0125a00700 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333ccce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.563+0000 7f0125000700 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d1eb0 0x5651333d2290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.563+0000 7f011fe00700 1 -- 172.21.15.28:0/2627666760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d1eb0 msgr2=0x5651333d2290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:59.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.563+0000 7f011fe00700 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d1eb0 0x5651333d2290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.563+0000 7f011fe00700 1 -- 172.21.15.28:0/2627666760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 msgr2=0x5651333ccce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.565+0000 7f011fe00700 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333ccce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.565+0000 7f011fe00700 1 -- 172.21.15.28:0/2627666760 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5651332e3420 con 0x5651333d27d0 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.565+0000 7f0125a00700 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333ccce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.565+0000 7f011fe00700 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d27d0 0x5651333cc260 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f0108010650 tx=0x7f010801a000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f6e4e2f6fe51cfc9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.566+0000 7f0116000700 1 -- 172.21.15.28:0/2627666760 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0108023070 con 0x5651333d27d0 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.566+0000 7f0116000700 1 -- 172.21.15.28:0/2627666760 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0108013e90 con 0x5651333d27d0 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.566+0000 7f011fe00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01040b2b80 0x7f01040b5040 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f0108005fd0 tx=0x7f010800d4b0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:59.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.566+0000 7f0116000700 1 -- 172.21.15.28:0/2627666760 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0108003c70 con 0x5651333d27d0 2024-05-20T20:12:59.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.567+0000 7f013243fb80 1 -- 172.21.15.28:0/2627666760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d27d0 msgr2=0x5651333cc260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.568+0000 7f013243fb80 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d27d0 0x5651333cc260 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f0108010650 tx=0x7f010801a000 comp rx=0 tx=0).stop 2024-05-20T20:12:59.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.569+0000 7f013243fb80 1 -- 172.21.15.28:0/2627666760 shutdown_connections 2024-05-20T20:12:59.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.569+0000 7f013243fb80 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333ccce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.569+0000 7f013243fb80 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d1eb0 0x5651333d2290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.570+0000 7f013243fb80 1 --2- 172.21.15.28:0/2627666760 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d27d0 0x5651333cc260 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.570+0000 7f013243fb80 1 -- 172.21.15.28:0/2627666760 >> 172.21.15.28:0/2627666760 conn(0x5651332fabe0 msgr2=0x5651333c7010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:59.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.570+0000 7f013243fb80 1 -- 172.21.15.28:0/2627666760 shutdown_connections 2024-05-20T20:12:59.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.570+0000 7f013243fb80 1 -- 172.21.15.28:0/2627666760 wait complete. 2024-05-20T20:12:59.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.571+0000 7f013243fb80 1 -- start start 2024-05-20T20:12:59.573 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.571+0000 7f013243fb80 1 Processor -- start 2024-05-20T20:12:59.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.574+0000 7f013243fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333d8510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.577+0000 7f013243fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333d58b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.578 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.577+0000 7f013243fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333d5df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.577+0000 7f0125000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333d8510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.577+0000 7f0125000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333d8510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43138/0 (socket says 172.21.15.28:43138) 2024-05-20T20:12:59.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.578+0000 7f013243fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x565133220fa0 con 0x5651333cc880 2024-05-20T20:12:59.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.578+0000 7f013243fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5651332bb760 con 0x5651333d1eb0 2024-05-20T20:12:59.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.578+0000 7f013243fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5651333ea1f0 con 0x5651333d27d0 2024-05-20T20:12:59.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.579+0000 7f0125a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333d5df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.579+0000 7f0125a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333d5df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38088/0 (socket says 172.21.15.28:38088) 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.579+0000 7f0125a00700 1 -- 172.21.15.28:0/2511853624 learned_addr learned my addr 172.21.15.28:0/2511853624 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.580+0000 7f0125a00700 1 -- 172.21.15.28:0/2511853624 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 msgr2=0x5651333d58b0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.580+0000 7f0125a00700 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333d58b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.580+0000 7f0125a00700 1 -- 172.21.15.28:0/2511853624 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 msgr2=0x5651333d8510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.580+0000 7f0125a00700 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333d8510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.580+0000 7f0125a00700 1 -- 172.21.15.28:0/2511853624 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56513326a8d0 con 0x5651333d27d0 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.582+0000 7f0125000700 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333d8510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello state changed while learned_addr, mark_down or replacing must be happened just now 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.582+0000 7f013243fb80 10 client.0 _ll_register_callbacks cb 0x5651332c4c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:59.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.582+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0108013e90 con 0x5651333d27d0 2024-05-20T20:12:59.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.584+0000 7f0125a00700 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333d5df0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f011000b070 tx=0x7f0110004d30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a710948a46d04672 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:59.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.584+0000 7f010ea00700 10 client.26875 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:12:59.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.585+0000 7f010ea00700 1 -- 172.21.15.28:0/2511853624 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f011002c050 con 0x5651333d27d0 2024-05-20T20:12:59.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.585+0000 7f010ea00700 1 -- 172.21.15.28:0/2511853624 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f01100078f0 con 0x5651333d27d0 2024-05-20T20:12:59.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.585+0000 7f010ea00700 1 -- 172.21.15.28:0/2511853624 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0110022070 con 0x5651333d27d0 2024-05-20T20:12:59.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.586+0000 7f013243fb80 10 client.26875 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:59.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.586+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x5651332eb060 con 0x5651333d27d0 2024-05-20T20:12:59.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.586+0000 7f010ea00700 1 -- 172.21.15.28:0/2511853624 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f011002f020 con 0x5651333d27d0 2024-05-20T20:12:59.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.587+0000 7f013243fb80 10 client.26875 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:59.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.587+0000 7f013243fb80 10 client.26875 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:12:59.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.587+0000 7f013243fb80 20 client.26875 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:59.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.587+0000 7f013243fb80 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651333f6a30 0x5651333f8e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.587+0000 7f013243fb80 4 client.26875 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:12:59.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.587+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56513332cd90 con 0x5651333f6a30 2024-05-20T20:12:59.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.588+0000 7f0125000700 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651333f6a30 0x5651333f8e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.589+0000 7f0125000700 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651333f6a30 0x5651333f8e60 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=65b55dd1357310e server_cookie=a177d1b6fcf8874 in_seq=0 out_seq=0 2024-05-20T20:12:59.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.589+0000 7f010ea00700 10 client.26875 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:12:59.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.590+0000 7f010ea00700 1 -- 172.21.15.28:0/2511853624 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56513332cd90 con 0x5651333f6a30 2024-05-20T20:12:59.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.590+0000 7f010ea00700 10 client.26875 handle_command_reply: tid=0 2024-05-20T20:12:59.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.590+0000 7f013243fb80 1 client.26875 shutdown 2024-05-20T20:12:59.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.592+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 msgr2=0x5651333d5df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.592+0000 7f013243fb80 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333d5df0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f011000b070 tx=0x7f0110004d30 comp rx=0 tx=0).stop 2024-05-20T20:12:59.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.593+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 shutdown_connections 2024-05-20T20:12:59.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.593+0000 7f013243fb80 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651333f6a30 0x5651333f8e60 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.593+0000 7f010ea00700 0 client.26875 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:12:59.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.593+0000 7f013243fb80 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333d5df0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.593+0000 7f013243fb80 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333d8510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.593+0000 7f013243fb80 1 --2- 172.21.15.28:0/2511853624 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333d58b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.593+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 >> 172.21.15.28:0/2511853624 conn(0x5651332fabe0 msgr2=0x5651333cb250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:59.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.595+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 shutdown_connections 2024-05-20T20:12:59.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.595+0000 7f013243fb80 1 -- 172.21.15.28:0/2511853624 wait complete. 2024-05-20T20:12:59.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.595+0000 7f013243fb80 20 client.26875 trim_cache size 0 max 16384 2024-05-20T20:12:59.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.643+0000 7f013243fb80 1 Processor -- start 2024-05-20T20:12:59.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.643+0000 7f013243fb80 1 -- start start 2024-05-20T20:12:59.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.644+0000 7f013243fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333cc260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.644+0000 7f013243fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333cf950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.644+0000 7f013243fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333cfe90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.644+0000 7f013243fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f01201a5db0 con 0x5651333cc880 2024-05-20T20:12:59.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.644+0000 7f013243fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f01201ad920 con 0x5651333d1eb0 2024-05-20T20:12:59.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.644+0000 7f013243fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f01201a6480 con 0x5651333d27d0 2024-05-20T20:12:59.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.646+0000 7f0125000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333cc260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.647+0000 7f0125a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333cfe90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.647+0000 7f011fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333cf950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.649+0000 7f0125a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333cfe90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38092/0 (socket says 172.21.15.28:38092) 2024-05-20T20:12:59.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.649+0000 7f0125a00700 1 -- 172.21.15.28:0/1293785640 learned_addr learned my addr 172.21.15.28:0/1293785640 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:59.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.649+0000 7f0125000700 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333cc260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43146/0 (socket says 172.21.15.28:43146) 2024-05-20T20:12:59.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.655+0000 7f011fe00700 1 -- 172.21.15.28:0/1293785640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 msgr2=0x5651333cfe90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:59.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.655+0000 7f011fe00700 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333cfe90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.655+0000 7f011fe00700 1 -- 172.21.15.28:0/1293785640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 msgr2=0x5651333cc260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:59.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.655+0000 7f011fe00700 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333cc260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.655+0000 7f011fe00700 1 -- 172.21.15.28:0/1293785640 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5651333f94e0 con 0x5651333d1eb0 2024-05-20T20:12:59.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.655+0000 7f0125a00700 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333cfe90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:12:59.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.659+0000 7f011fe00700 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333cf950 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f0108000c00 tx=0x7f0108005ee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3216a4a490cc7bba server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:59.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.659+0000 7f0115600700 1 -- 172.21.15.28:0/1293785640 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0108023070 con 0x5651333d1eb0 2024-05-20T20:12:59.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.659+0000 7f0115600700 1 -- 172.21.15.28:0/1293785640 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f010800d780 con 0x5651333d1eb0 2024-05-20T20:12:59.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.659+0000 7f0115600700 1 -- 172.21.15.28:0/1293785640 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0108023070 con 0x5651333d1eb0 2024-05-20T20:12:59.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.661+0000 7f013243fb80 1 -- 172.21.15.28:0/1293785640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 msgr2=0x5651333cf950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.661+0000 7f013243fb80 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333cf950 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f0108000c00 tx=0x7f0108005ee0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.662+0000 7f013243fb80 1 -- 172.21.15.28:0/1293785640 shutdown_connections 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.662+0000 7f013243fb80 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333cfe90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.662+0000 7f013243fb80 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333cc880 0x5651333cc260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.662+0000 7f013243fb80 1 --2- 172.21.15.28:0/1293785640 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333d1eb0 0x5651333cf950 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.662+0000 7f013243fb80 1 -- 172.21.15.28:0/1293785640 >> 172.21.15.28:0/1293785640 conn(0x5651332fabe0 msgr2=0x5651333d41c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.662+0000 7f013243fb80 1 -- 172.21.15.28:0/1293785640 shutdown_connections 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.662+0000 7f013243fb80 1 -- 172.21.15.28:0/1293785640 wait complete. 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.663+0000 7f013243fb80 1 -- start start 2024-05-20T20:12:59.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.663+0000 7f013243fb80 1 Processor -- start 2024-05-20T20:12:59.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.664+0000 7f013243fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333cc880 0x5651333ef480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.664+0000 7f013243fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333d1eb0 0x5651333ef9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.664+0000 7f013243fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333ec450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.665+0000 7f0125a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333ec450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.665+0000 7f0125a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333ec450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38106/0 (socket says 172.21.15.28:38106) 2024-05-20T20:12:59.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.665+0000 7f0125a00700 1 -- 172.21.15.28:0/63194939 learned_addr learned my addr 172.21.15.28:0/63194939 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:12:59.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.665+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f01201a7220 con 0x5651333d1eb0 2024-05-20T20:12:59.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.665+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f01201a6b50 con 0x5651333cc880 2024-05-20T20:12:59.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.665+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f01201a3d90 con 0x5651333d27d0 2024-05-20T20:12:59.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.666+0000 7f0125000700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333cc880 0x5651333ef480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.666+0000 7f013243fb80 10 client.0 _ll_register_callbacks cb 0x5651332c4c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.667+0000 7f011fe00700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333d1eb0 0x5651333ef9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.667+0000 7f0125000700 1 -- 172.21.15.28:0/63194939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 msgr2=0x5651333ec450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.667+0000 7f0125000700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333ec450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.667+0000 7f0125000700 1 -- 172.21.15.28:0/63194939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333d1eb0 msgr2=0x5651333ef9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.667+0000 7f0125000700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333d1eb0 0x5651333ef9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.667+0000 7f0125000700 1 -- 172.21.15.28:0/63194939 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5651333ff270 con 0x5651333cc880 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.668+0000 7f011fe00700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333d1eb0 0x5651333ef9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.668+0000 7f0125000700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333cc880 0x5651333ef480 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f01180df4a0 tx=0x7f01180a5000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7d0f05f65666a547 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:12:59.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.668+0000 7f010ea00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:12:59.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.670+0000 7f010ea00700 1 -- 172.21.15.28:0/63194939 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01180ad3e0 con 0x5651333cc880 2024-05-20T20:12:59.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.670+0000 7f010ea00700 1 -- 172.21.15.28:0/63194939 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f01180a94a0 con 0x5651333cc880 2024-05-20T20:12:59.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.672+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f010800d780 con 0x5651333cc880 2024-05-20T20:12:59.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.673+0000 7f010ea00700 1 -- 172.21.15.28:0/63194939 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f01180c8050 con 0x5651333cc880 2024-05-20T20:12:59.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.673+0000 7f013243fb80 10 client.26987 fetch_fsmap learned FSMap version 29 2024-05-20T20:12:59.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.673+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x565133461c50 con 0x5651333cc880 2024-05-20T20:12:59.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.673+0000 7f010ea00700 1 -- 172.21.15.28:0/63194939 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f010800d780 con 0x5651333cc880 2024-05-20T20:12:59.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.674+0000 7f010ea00700 1 -- 172.21.15.28:0/63194939 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f01180a8290 con 0x5651333cc880 2024-05-20T20:12:59.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.674+0000 7f013243fb80 10 client.26987 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:12:59.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.674+0000 7f013243fb80 10 client.26987 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:12:59.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.674+0000 7f013243fb80 20 client.26987 populate_metadata read hostname 'smithi028' 2024-05-20T20:12:59.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.674+0000 7f013243fb80 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651334622b0 0x5651334646a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:12:59.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.674+0000 7f013243fb80 4 client.26987 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:12:59.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.674+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x565133326570 con 0x5651334622b0 2024-05-20T20:12:59.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.675+0000 7f011fe00700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651334622b0 0x5651334646a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:12:59.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.677+0000 7f011fe00700 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651334622b0 0x5651334646a0 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=13f81dc7bcf649f3 server_cookie=cc9bf5bff3d9abde in_seq=0 out_seq=0 2024-05-20T20:12:59.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.677+0000 7f010ea00700 10 client.26987 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:12:59.693 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f010ea00700 1 -- 172.21.15.28:0/63194939 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x565133326570 con 0x5651334622b0 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f010ea00700 10 client.26987 handle_command_reply: tid=0 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 client.26987 shutdown 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333cc880 msgr2=0x5651333ef480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333cc880 0x5651333ef480 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f01180df4a0 tx=0x7f01180a5000 comp rx=0 tx=0).stop 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 shutdown_connections 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x5651334622b0 0x5651334646a0 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5651333d27d0 0x5651333ec450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5651333cc880 0x5651333ef480 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 --2- 172.21.15.28:0/63194939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5651333d1eb0 0x5651333ef9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 >> 172.21.15.28:0/63194939 conn(0x5651332fabe0 msgr2=0x5651333d5690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.681+0000 7f010ea00700 0 client.26987 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.682+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 shutdown_connections 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.682+0000 7f013243fb80 1 -- 172.21.15.28:0/63194939 wait complete. 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.682+0000 7f013243fb80 20 client.26987 trim_cache size 0 max 16384 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.686+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01040b2b80 msgr2=0x7f01040b5040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.686+0000 7f0127a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01040b2b80 0x7f01040b5040 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f0108005fd0 tx=0x7f010800d4b0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.686+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 msgr2=0x7f0120198890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.686+0000 7f0127a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 0x7f0120198890 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f011800b530 tx=0x7f011800b8f0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.687+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 shutdown_connections 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.687+0000 7f0127a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0120108520 0x7f012019d8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.687+0000 7f0127a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01040b2b80 0x7f01040b5040 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.687+0000 7f0127a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0120102440 0x7f0120198890 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.687+0000 7f0127a00700 1 --2- 172.21.15.28:0/1809960350 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0120102e00 0x7f0120198dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:12:59.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.687+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 >> 172.21.15.28:0/1809960350 conn(0x7f01200793f0 msgr2=0x7f0120101040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:12:59.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.688+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 shutdown_connections 2024-05-20T20:12:59.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:12:59.690+0000 7f0127a00700 1 -- 172.21.15.28:0/1809960350 wait complete. 2024-05-20T20:13:00.644 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 143.3 --await-for 45.0' 2024-05-20T20:13:00.644 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 143.3 --await-for 45.0 2024-05-20T20:13:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:01 smithi154 ceph-mon[36314]: pgmap v226: 97 pgs: 97 active+clean; 44 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 328 MiB/s wr, 1.72k op/s 2024-05-20T20:13:01.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:01 smithi033 ceph-mon[36179]: pgmap v226: 97 pgs: 97 active+clean; 44 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 328 MiB/s wr, 1.72k op/s 2024-05-20T20:13:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:01 smithi028 ceph-mon[31178]: pgmap v226: 97 pgs: 97 active+clean; 44 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 328 MiB/s wr, 1.72k op/s 2024-05-20T20:13:02.185 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:13:02.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:02 smithi154 ceph-mon[36314]: pgmap v227: 97 pgs: 97 active+clean; 44 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 252 MiB/s wr, 1.22k op/s 2024-05-20T20:13:02.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:02 smithi033 ceph-mon[36179]: pgmap v227: 97 pgs: 97 active+clean; 44 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 252 MiB/s wr, 1.22k op/s 2024-05-20T20:13:02.417 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:02.456 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:02 smithi028 ceph-mon[31178]: pgmap v227: 97 pgs: 97 active+clean; 44 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 252 MiB/s wr, 1.22k op/s 2024-05-20T20:13:05.155 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:05.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:04 smithi028 ceph-mon[31178]: pgmap v228: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 344 MiB/s wr, 1.68k op/s 2024-05-20T20:13:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:04 smithi154 ceph-mon[36314]: pgmap v228: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 344 MiB/s wr, 1.68k op/s 2024-05-20T20:13:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:04 smithi033 ceph-mon[36179]: pgmap v228: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 344 MiB/s wr, 1.68k op/s 2024-05-20T20:13:07.187 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:06 smithi028 ceph-mon[31178]: pgmap v229: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 252 MiB/s wr, 1.21k op/s 2024-05-20T20:13:07.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:06 smithi154 ceph-mon[36314]: pgmap v229: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 252 MiB/s wr, 1.21k op/s 2024-05-20T20:13:07.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:06 smithi033 ceph-mon[36179]: pgmap v229: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 252 MiB/s wr, 1.21k op/s 2024-05-20T20:13:07.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.818+0000 7f9c70200700 1 -- 172.21.15.28:0/335489679 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68106130 msgr2=0x7f9c68112640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.818+0000 7f9c70200700 1 --2- 172.21.15.28:0/335489679 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68106130 0x7f9c68112640 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f9c600098f0 tx=0x7f9c60009c00 comp rx=0 tx=0).stop 2024-05-20T20:13:07.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.819+0000 7f9c70200700 1 -- 172.21.15.28:0/335489679 shutdown_connections 2024-05-20T20:13:07.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.819+0000 7f9c70200700 1 --2- 172.21.15.28:0/335489679 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68106130 0x7f9c68112640 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.819+0000 7f9c70200700 1 --2- 172.21.15.28:0/335489679 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9c6810b770 0x7f9c6810bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.819+0000 7f9c70200700 1 --2- 172.21.15.28:0/335489679 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9c68105770 0x7f9c68105bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.819+0000 7f9c70200700 1 -- 172.21.15.28:0/335489679 >> 172.21.15.28:0/335489679 conn(0x7f9c68101260 msgr2=0x7f9c68103680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.819+0000 7f9c70200700 1 -- 172.21.15.28:0/335489679 shutdown_connections 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.819+0000 7f9c70200700 1 -- 172.21.15.28:0/335489679 wait complete. 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.822+0000 7f9c70200700 1 Processor -- start 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.822+0000 7f9c70200700 1 -- start start 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c70200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68105770 0x7f9c681a1a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c70200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9c68106130 0x7f9c681a1fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c70200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9c6810b770 0x7f9c6819bb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c70200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9c68115a00 con 0x7f9c6810b770 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c70200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9c68115880 con 0x7f9c68106130 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c70200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9c68115b80 con 0x7f9c68105770 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c6e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9c6810b770 0x7f9c6819bb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c6e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9c6810b770 0x7f9c6819bb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43192/0 (socket says 172.21.15.28:43192) 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c6e200700 1 -- 172.21.15.28:0/4001113876 learned_addr learned my addr 172.21.15.28:0/4001113876 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c6d800700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68105770 0x7f9c681a1a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.824+0000 7f9c6ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9c68106130 0x7f9c681a1fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c6d800700 1 -- 172.21.15.28:0/4001113876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9c68106130 msgr2=0x7f9c681a1fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c6d800700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9c68106130 0x7f9c681a1fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c6d800700 1 -- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9c6810b770 msgr2=0x7f9c6819bb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c6d800700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9c6810b770 0x7f9c6819bb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c6d800700 1 -- 172.21.15.28:0/4001113876 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9c6819c460 con 0x7f9c68105770 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c6d800700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68105770 0x7f9c681a1a60 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f9c58007fd0 tx=0x7f9c5800d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c66000700 1 -- 172.21.15.28:0/4001113876 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c5801a070 con 0x7f9c68105770 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.825+0000 7f9c66000700 1 -- 172.21.15.28:0/4001113876 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9c5800ddd0 con 0x7f9c68105770 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.826+0000 7f9c66000700 1 -- 172.21.15.28:0/4001113876 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c58014600 con 0x7f9c68105770 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.826+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9c6819c750 con 0x7f9c68105770 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.826+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9c681a89e0 con 0x7f9c68105770 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.832+0000 7f9c66000700 1 -- 172.21.15.28:0/4001113876 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f9c5801d070 con 0x7f9c68105770 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.833+0000 7f9c7ab6cb80 1 Processor -- start 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.833+0000 7f9c7ab6cb80 1 -- start start 2024-05-20T20:13:07.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.833+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b529080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.833+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b5295c0 0x558d1b523060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.833+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b523ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c7ab6cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558d1b480fd0 con 0x558d1b528ca0 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c7ab6cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558d1b470b70 con 0x558d1b523680 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c7ab6cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558d1b474bb0 con 0x558d1b5295c0 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c6e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b529080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c6e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b529080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43208/0 (socket says 172.21.15.28:43208) 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c6e200700 1 -- 172.21.15.28:0/3408531075 learned_addr learned my addr 172.21.15.28:0/3408531075 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c66000700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9c4c0b2ba0 0x7f9c4c0b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b523ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.834+0000 7f9c66000700 1 -- 172.21.15.28:0/4001113876 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f9c580e1e80 con 0x7f9c68105770 2024-05-20T20:13:07.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.835+0000 7f9c6d800700 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b5295c0 0x558d1b523060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.835+0000 7f9c6e200700 1 -- 172.21.15.28:0/3408531075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b5295c0 msgr2=0x558d1b523060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.835+0000 7f9c6e200700 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b5295c0 0x558d1b523060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.835+0000 7f9c6e200700 1 -- 172.21.15.28:0/3408531075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 msgr2=0x558d1b523ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.835+0000 7f9c6e200700 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b523ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.835+0000 7f9c6e200700 1 -- 172.21.15.28:0/3408531075 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558d1b4395b0 con 0x558d1b528ca0 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.835+0000 7f9c6e200700 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b529080 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f9c6000fc10 tx=0x7f9c6000fc40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c1d2b46159391e57 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.836+0000 7f9c6e200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9c4c0b2ba0 0x7f9c4c0b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.836+0000 7f9c5ea00700 1 -- 172.21.15.28:0/3408531075 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c60029070 con 0x558d1b528ca0 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.836+0000 7f9c5ea00700 1 -- 172.21.15.28:0/3408531075 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9c600054f0 con 0x558d1b528ca0 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.836+0000 7f9c5ea00700 1 -- 172.21.15.28:0/3408531075 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c60004d80 con 0x558d1b528ca0 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.837+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/3408531075 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 msgr2=0x558d1b529080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.837+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b529080 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f9c6000fc10 tx=0x7f9c6000fc40 comp rx=0 tx=0).stop 2024-05-20T20:13:07.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.837+0000 7f9c6e200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9c4c0b2ba0 0x7f9c4c0b5060 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9c60026000 tx=0x7f9c6000b560 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.838+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/3408531075 shutdown_connections 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.838+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b523ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.838+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b529080 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.838+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/3408531075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b5295c0 0x558d1b523060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.838+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/3408531075 >> 172.21.15.28:0/3408531075 conn(0x558d1b450d70 msgr2=0x558d1b51df90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.838+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/3408531075 shutdown_connections 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.838+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/3408531075 wait complete. 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.840+0000 7f9c7ab6cb80 1 -- start start 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.841+0000 7f9c7ab6cb80 1 Processor -- start 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b523680 0x558d1b52f230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b528ca0 0x558d1b52c5d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b52cb10 0x558d1b52cf90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.482 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c7ab6cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558d1b377130 con 0x558d1b523680 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c7ab6cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558d1b4118f0 con 0x558d1b528ca0 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c7ab6cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558d1b5387e0 con 0x558d1b52cb10 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c7ab6cb80 10 client.0 _ll_register_callbacks cb 0x558d1b41ae00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c6e200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b52cb10 0x558d1b52cf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c6e200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b52cb10 0x558d1b52cf90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38140/0 (socket says 172.21.15.28:38140) 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.842+0000 7f9c6e200700 1 -- 172.21.15.28:0/1199763596 learned_addr learned my addr 172.21.15.28:0/1199763596 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.843+0000 7f9c6e200700 1 -- 172.21.15.28:0/1199763596 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b528ca0 msgr2=0x558d1b52c5d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.843+0000 7f9c6e200700 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b528ca0 0x558d1b52c5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.843+0000 7f9c6e200700 1 -- 172.21.15.28:0/1199763596 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b523680 msgr2=0x558d1b52f230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.843+0000 7f9c6e200700 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b523680 0x558d1b52f230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.843+0000 7f9c6e200700 1 -- 172.21.15.28:0/1199763596 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558d1b3c0a60 con 0x558d1b52cb10 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.843+0000 7f9c6e200700 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b52cb10 0x558d1b52cf90 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f9c6000b960 tx=0x7f9c60024000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9e38f67ce29ba4f9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.844+0000 7f9c53400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.844+0000 7f9c53400700 1 -- 172.21.15.28:0/1199763596 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c60029050 con 0x558d1b52cb10 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.844+0000 7f9c53400700 1 -- 172.21.15.28:0/1199763596 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f9c6000f460 con 0x558d1b52cb10 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.844+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9c600054f0 con 0x558d1b52cb10 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.845+0000 7f9c53400700 1 -- 172.21.15.28:0/1199763596 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f9c6000f660 con 0x558d1b52cb10 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.845+0000 7f9c7ab6cb80 10 client.26890 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.845+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558d1b4411f0 con 0x558d1b52cb10 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.846+0000 7f9c53400700 1 -- 172.21.15.28:0/1199763596 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9c600054f0 con 0x558d1b52cb10 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.847+0000 7f9c53400700 1 -- 172.21.15.28:0/1199763596 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9c6002c040 con 0x558d1b52cb10 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.847+0000 7f9c7ab6cb80 10 client.26890 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.847+0000 7f9c7ab6cb80 10 client.26890 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.847+0000 7f9c7ab6cb80 20 client.26890 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.847+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b545540 0x558d1b547930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.847+0000 7f9c7ab6cb80 4 client.26890 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.847+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558d1b482f20 con 0x558d1b545540 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.848+0000 7f9c6d800700 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b545540 0x558d1b547930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.849+0000 7f9c6d800700 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b545540 0x558d1b547930 crc :-1 s=READY pgs=171 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2b98861ba6b78677 server_cookie=66d9bd6e26e1bd8a in_seq=0 out_seq=0 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.849+0000 7f9c53400700 10 client.26890 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c53400700 1 -- 172.21.15.28:0/1199763596 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558d1b482f20 con 0x558d1b545540 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c53400700 10 client.26890 handle_command_reply: tid=0 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 client.26890 shutdown 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b52cb10 msgr2=0x558d1b52cf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b52cb10 0x558d1b52cf90 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f9c6000b960 tx=0x7f9c60024000 comp rx=0 tx=0).stop 2024-05-20T20:13:07.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 shutdown_connections 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b545540 0x558d1b547930 crc :-1 s=READY pgs=171 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b52cb10 0x558d1b52cf90 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b523680 0x558d1b52f230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.850+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1199763596 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b528ca0 0x558d1b52c5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.851+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 >> 172.21.15.28:0/1199763596 conn(0x558d1b450d70 msgr2=0x558d1b51d4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.851+0000 7f9c53400700 0 client.26890 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.851+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 shutdown_connections 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.851+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1199763596 wait complete. 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.851+0000 7f9c7ab6cb80 20 client.26890 trim_cache size 0 max 16384 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.914+0000 7f9c7ab6cb80 1 Processor -- start 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.915+0000 7f9c7ab6cb80 1 -- start start 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.915+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b523680 0x558d1b523060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.915+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b541880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.916+0000 7f9c6d800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b523680 0x558d1b523060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.916+0000 7f9c6d800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b523680 0x558d1b523060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59572/0 (socket says 172.21.15.28:59572) 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.916+0000 7f9c6d800700 1 -- 172.21.15.28:0/1483014614 learned_addr learned my addr 172.21.15.28:0/1483014614 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.917+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b52cb10 0x558d1b54ced0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.917+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9c681abcc0 con 0x558d1b528ca0 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.917+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9c681ad800 con 0x558d1b52cb10 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.917+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9c681ac390 con 0x558d1b523680 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.917+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b541880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.917+0000 7f9c6e200700 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b52cb10 0x558d1b54ced0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.919+0000 7f9c6ce00700 1 -- 172.21.15.28:0/1483014614 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b523680 msgr2=0x558d1b523060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.919+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b523680 0x558d1b523060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.919+0000 7f9c6ce00700 1 -- 172.21.15.28:0/1483014614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b52cb10 msgr2=0x558d1b54ced0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.919+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b52cb10 0x558d1b54ced0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.920+0000 7f9c6ce00700 1 -- 172.21.15.28:0/1483014614 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558d1b51e4d0 con 0x558d1b528ca0 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.921+0000 7f9c6e200700 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b52cb10 0x558d1b54ced0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.921+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b541880 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x558d1b52dba0 tx=0x7f9c54009ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4711b36819ed638f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.922+0000 7f9c5e000700 1 -- 172.21.15.28:0/1483014614 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c54004220 con 0x558d1b528ca0 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.922+0000 7f9c5e000700 1 -- 172.21.15.28:0/1483014614 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9c54012d40 con 0x558d1b528ca0 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.922+0000 7f9c5e000700 1 -- 172.21.15.28:0/1483014614 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c540038e0 con 0x558d1b528ca0 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.923+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 msgr2=0x558d1b541880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.923+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b541880 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x558d1b52dba0 tx=0x7f9c54009ee0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.923+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 shutdown_connections 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.923+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b52cb10 0x558d1b54ced0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.923+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b523680 0x558d1b523060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.923+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/1483014614 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b528ca0 0x558d1b541880 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.924+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 >> 172.21.15.28:0/1483014614 conn(0x558d1b450d70 msgr2=0x558d1b5266c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.924+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 shutdown_connections 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.924+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/1483014614 wait complete. 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.925+0000 7f9c7ab6cb80 1 -- start start 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.926+0000 7f9c7ab6cb80 1 Processor -- start 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.927+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b53dcc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.927+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b528ca0 0x558d1b53e200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.927+0000 7f9c7ab6cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b52cb10 0x558d1b53aac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.928+0000 7f9c6d800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b53dcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.928+0000 7f9c6ce00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b528ca0 0x558d1b53e200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c6ce00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b528ca0 0x558d1b53e200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59584/0 (socket says 172.21.15.28:59584) 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c6ce00700 1 -- 172.21.15.28:0/2324038769 learned_addr learned my addr 172.21.15.28:0/2324038769 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9c681ad130 con 0x558d1b52cb10 2024-05-20T20:13:07.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9c681aca60 con 0x558d1b523680 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9c681a9dc0 con 0x558d1b528ca0 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c7ab6cb80 10 client.0 _ll_register_callbacks cb 0x558d1b41ae00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.928+0000 7f9c6e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b52cb10 0x558d1b53aac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c6ce00700 1 -- 172.21.15.28:0/2324038769 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 msgr2=0x558d1b53dcc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b53dcc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c6ce00700 1 -- 172.21.15.28:0/2324038769 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b52cb10 msgr2=0x558d1b53aac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b52cb10 0x558d1b53aac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.929+0000 7f9c6ce00700 1 -- 172.21.15.28:0/2324038769 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558d1b53b300 con 0x558d1b528ca0 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.930+0000 7f9c6ce00700 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b528ca0 0x558d1b53e200 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f9c54004cb0 tx=0x7f9c540099d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b4a81c573f1bbe4b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.930+0000 7f9c53400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.931+0000 7f9c53400700 1 -- 172.21.15.28:0/2324038769 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9c54027070 con 0x558d1b528ca0 2024-05-20T20:13:07.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.931+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9c54012d40 con 0x558d1b528ca0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.931+0000 7f9c53400700 1 -- 172.21.15.28:0/2324038769 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f9c54004020 con 0x558d1b528ca0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.931+0000 7f9c53400700 1 -- 172.21.15.28:0/2324038769 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f9c54033050 con 0x558d1b528ca0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.932+0000 7f9c7ab6cb80 10 client.26896 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.932+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558d1b5c1940 con 0x558d1b528ca0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.932+0000 7f9c53400700 1 -- 172.21.15.28:0/2324038769 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9c54012d40 con 0x558d1b528ca0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.933+0000 7f9c53400700 1 -- 172.21.15.28:0/2324038769 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9c54019030 con 0x558d1b528ca0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.933+0000 7f9c7ab6cb80 10 client.26896 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.933+0000 7f9c7ab6cb80 10 client.26896 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.933+0000 7f9c7ab6cb80 20 client.26896 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.933+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b5c1fc0 0x558d1b5c43e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.933+0000 7f9c7ab6cb80 4 client.26896 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 143.3, "await_for": 45.0}] 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.933+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 143.3, "await_for": 45.0}) -- 0x558d1b47c700 con 0x558d1b5c1fc0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.934+0000 7f9c6d800700 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b5c1fc0 0x558d1b5c43e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.934+0000 7f9c6d800700 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b5c1fc0 0x558d1b5c43e0 crc :-1 s=READY pgs=172 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3d93be42e7120ad4 server_cookie=62fb8e9e56b3c172 in_seq=0 out_seq=0 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:06.934+0000 7f9c53400700 10 client.26896 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.185+0000 7f9c53400700 1 -- 172.21.15.28:0/2324038769 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x558d1b47c700 con 0x558d1b5c1fc0 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.185+0000 7f9c53400700 10 client.26896 handle_command_reply: tid=0 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.185+0000 7f9c7ab6cb80 1 client.26896 shutdown 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.185+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b528ca0 msgr2=0x558d1b53e200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.185+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b528ca0 0x558d1b53e200 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f9c54004cb0 tx=0x7f9c540099d0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 shutdown_connections 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558d1b5c1fc0 0x558d1b5c43e0 crc :-1 s=READY pgs=172 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c53400700 0 client.26896 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558d1b52cb10 0x558d1b53aac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558d1b523680 0x558d1b53dcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c7ab6cb80 1 --2- 172.21.15.28:0/2324038769 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558d1b528ca0 0x558d1b53e200 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 >> 172.21.15.28:0/2324038769 conn(0x558d1b450d70 msgr2=0x558d1b52adb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.186+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 shutdown_connections 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.187+0000 7f9c7ab6cb80 1 -- 172.21.15.28:0/2324038769 wait complete. 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 18, 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:13:07.493 INFO:teuthology.orchestra.run.smithi028.stdout: "4491be01": { 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 18, 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 143.3, 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:13:07.494 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:13:07.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.189+0000 7f9c7ab6cb80 20 client.26896 trim_cache size 0 max 16384 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9c4c0b2ba0 msgr2=0x7f9c4c0b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9c4c0b2ba0 0x7f9c4c0b5060 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9c60026000 tx=0x7f9c6000b560 comp rx=0 tx=0).stop 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68105770 msgr2=0x7f9c681a1a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68105770 0x7f9c681a1a60 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f9c58007fd0 tx=0x7f9c5800d3b0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 shutdown_connections 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9c6810b770 0x7f9c6819bb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9c68105770 0x7f9c681a1a60 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f9c4c0b2ba0 0x7f9c4c0b5060 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 --2- 172.21.15.28:0/4001113876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9c68106130 0x7f9c681a1fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 >> 172.21.15.28:0/4001113876 conn(0x7f9c68101260 msgr2=0x7f9c681017c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 shutdown_connections 2024-05-20T20:13:07.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:07.193+0000 7f9c70200700 1 -- 172.21.15.28:0/4001113876 wait complete. 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.710+0000 7f09af200640 1 -- 172.21.15.28:0/1993380043 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a8075670 msgr2=0x7f09a810ee00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.710+0000 7f09af200640 1 --2- 172.21.15.28:0/1993380043 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a8075670 0x7f09a810ee00 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f0998009900 tx=0x7f099802a490 comp rx=0 tx=0).stop 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.710+0000 7f09af200640 1 -- 172.21.15.28:0/1993380043 shutdown_connections 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.711+0000 7f09af200640 1 --2- 172.21.15.28:0/1993380043 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a8075670 0x7f09a810ee00 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.711+0000 7f09af200640 1 --2- 172.21.15.28:0/1993380043 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f09a810f340 0x7f09a8111730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.711+0000 7f09af200640 1 --2- 172.21.15.28:0/1993380043 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f09a8074d50 0x7f09a8075130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.711+0000 7f09af200640 1 -- 172.21.15.28:0/1993380043 >> 172.21.15.28:0/1993380043 conn(0x7f09a8079ee0 msgr2=0x7f09a807a2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.711+0000 7f09af200640 1 -- 172.21.15.28:0/1993380043 shutdown_connections 2024-05-20T20:13:07.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.711+0000 7f09af200640 1 -- 172.21.15.28:0/1993380043 wait complete. 2024-05-20T20:13:07.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 Processor -- start 2024-05-20T20:13:07.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 -- start start 2024-05-20T20:13:07.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f09a8074d50 0x7f09a819b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f09a8075670 0x7f09a819b8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a810f340 0x7f09a819fc00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f09a8114f10 con 0x7f09a8075670 2024-05-20T20:13:07.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f09a8114d90 con 0x7f09a810f340 2024-05-20T20:13:07.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.712+0000 7f09af200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f09a8115090 con 0x7f09a8074d50 2024-05-20T20:13:07.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.713+0000 7f09a7e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f09a8074d50 0x7f09a819b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.713+0000 7f09a7e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f09a8074d50 0x7f09a819b390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59594/0 (socket says 172.21.15.28:59594) 2024-05-20T20:13:07.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.713+0000 7f09a7e00640 1 -- 172.21.15.28:0/2297141105 learned_addr learned my addr 172.21.15.28:0/2297141105 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.713+0000 7f09a7400640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f09a8075670 0x7f09a819b8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.713+0000 7f09ad200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a810f340 0x7f09a819fc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.714+0000 7f09ad200640 1 -- 172.21.15.28:0/2297141105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f09a8074d50 msgr2=0x7f09a819b390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.714+0000 7f09ad200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f09a8074d50 0x7f09a819b390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.714+0000 7f09ad200640 1 -- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f09a8075670 msgr2=0x7f09a819b8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.714+0000 7f09ad200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f09a8075670 0x7f09a819b8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.714+0000 7f09ad200640 1 -- 172.21.15.28:0/2297141105 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f09a81a0380 con 0x7f09a810f340 2024-05-20T20:13:07.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.714+0000 7f09ad200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a810f340 0x7f09a819fc00 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f09a0005fd0 tx=0x7f09a000ee60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.715+0000 7f09a4c00640 1 -- 172.21.15.28:0/2297141105 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09a000cd80 con 0x7f09a810f340 2024-05-20T20:13:07.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.715+0000 7f09a4c00640 1 -- 172.21.15.28:0/2297141105 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f09a0005020 con 0x7f09a810f340 2024-05-20T20:13:07.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.715+0000 7f09a4c00640 1 -- 172.21.15.28:0/2297141105 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f09a0010690 con 0x7f09a810f340 2024-05-20T20:13:07.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.715+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f09a81a0670 con 0x7f09a810f340 2024-05-20T20:13:07.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.715+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f09a8102990 con 0x7f09a810f340 2024-05-20T20:13:07.720 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.720+0000 7f09a4c00640 1 -- 172.21.15.28:0/2297141105 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f09a0010830 con 0x7f09a810f340 2024-05-20T20:13:07.720 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.720+0000 7f09b2046740 1 Processor -- start 2024-05-20T20:13:07.721 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.721+0000 7f09b2046740 1 -- start start 2024-05-20T20:13:07.721 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.722+0000 7f09b2046740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688168c90 0x55d688169070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.721 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.722+0000 7f09b2046740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688173930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.721 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.722+0000 7f09b2046740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688173e70 0x55d688176260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.721 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.722+0000 7f09b2046740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d6880d9b90 con 0x55d688168c90 2024-05-20T20:13:07.722 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.722+0000 7f09b2046740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d6880c9730 con 0x55d6881695b0 2024-05-20T20:13:07.722 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.722+0000 7f09b2046740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d6880cd770 con 0x55d688173e70 2024-05-20T20:13:07.723 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.723+0000 7f09a4c00640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f098c0b2b60 0x7f098c0b5020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.723 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.723+0000 7f09a4c00640 1 -- 172.21.15.28:0/2297141105 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f09a00e2bc0 con 0x7f09a810f340 2024-05-20T20:13:07.724 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.724+0000 7f09a7400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688168c90 0x55d688169070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.724 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.724+0000 7f09a7400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688168c90 0x55d688169070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34022/0 (socket says 172.21.15.28:34022) 2024-05-20T20:13:07.724 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.724+0000 7f09a7400640 1 -- 172.21.15.28:0/2089730725 learned_addr learned my addr 172.21.15.28:0/2089730725 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.724 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.725+0000 7f09a7e00640 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688173930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.725 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.725+0000 7f09ad200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f098c0b2b60 0x7f098c0b5020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.725 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.726+0000 7f09a7400640 1 -- 172.21.15.28:0/2089730725 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688173e70 msgr2=0x55d688176260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.725 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.726+0000 7f09a7400640 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688173e70 0x55d688176260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.725 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.726+0000 7f09a7400640 1 -- 172.21.15.28:0/2089730725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 msgr2=0x55d688173930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.726 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.726+0000 7f09a7400640 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688173930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.726 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.726+0000 7f09a7400640 1 -- 172.21.15.28:0/2089730725 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d6880628b0 con 0x55d688168c90 2024-05-20T20:13:07.726 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.726+0000 7f09ad200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f098c0b2b60 0x7f098c0b5020 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f09a00e2ea0 tx=0x7f09a00e2e10 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.727+0000 7f09a7400640 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688168c90 0x55d688169070 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f0998000c00 tx=0x7f09980095e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2bd7f99ab128e66e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.728+0000 7f099d600640 1 -- 172.21.15.28:0/2089730725 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0998041070 con 0x55d688168c90 2024-05-20T20:13:07.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.728+0000 7f099d600640 1 -- 172.21.15.28:0/2089730725 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0998004490 con 0x55d688168c90 2024-05-20T20:13:07.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.728+0000 7f099d600640 1 -- 172.21.15.28:0/2089730725 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0998035040 con 0x55d688168c90 2024-05-20T20:13:07.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.729+0000 7f09b2046740 1 -- 172.21.15.28:0/2089730725 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688168c90 msgr2=0x55d688169070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.729+0000 7f09b2046740 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688168c90 0x55d688169070 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f0998000c00 tx=0x7f09980095e0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.730+0000 7f09b2046740 1 -- 172.21.15.28:0/2089730725 shutdown_connections 2024-05-20T20:13:07.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.730+0000 7f09b2046740 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688173930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.730+0000 7f09b2046740 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688173e70 0x55d688176260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.730+0000 7f09b2046740 1 --2- 172.21.15.28:0/2089730725 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688168c90 0x55d688169070 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.730+0000 7f09b2046740 1 -- 172.21.15.28:0/2089730725 >> 172.21.15.28:0/2089730725 conn(0x55d6881651a0 msgr2=0x55d688165580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.730+0000 7f09b2046740 1 -- 172.21.15.28:0/2089730725 shutdown_connections 2024-05-20T20:13:07.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.731+0000 7f09b2046740 1 -- 172.21.15.28:0/2089730725 wait complete. 2024-05-20T20:13:07.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.731+0000 7f09b2046740 1 -- start start 2024-05-20T20:13:07.731 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.731+0000 7f09b2046740 1 Processor -- start 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09b2046740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688173870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09b2046740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688170b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09b2046740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d688171170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09b2046740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d687daf790 con 0x55d688173e70 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09b2046740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d688071bc0 con 0x55d6881695b0 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09b2046740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d688176a40 con 0x55d688168c90 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09b2046740 10 client.0 _ll_register_callbacks cb 0x55d68804f400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:07.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09a7e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688170b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09a7e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688170b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58638/0 (socket says 172.21.15.28:58638) 2024-05-20T20:13:07.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09a7400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688173870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.732+0000 7f09a7e00640 1 -- 172.21.15.28:0/3077644500 learned_addr learned my addr 172.21.15.28:0/3077644500 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.733+0000 7f09a7400640 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d688171170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.733+0000 7f09a7e00640 1 -- 172.21.15.28:0/3077644500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 msgr2=0x55d688173870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.733+0000 7f09a7e00640 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688173870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.734 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.733+0000 7f09a7e00640 1 -- 172.21.15.28:0/3077644500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 msgr2=0x55d688171170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.733+0000 7f09a7e00640 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d688171170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.733+0000 7f09a7e00640 1 -- 172.21.15.28:0/3077644500 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d687dde210 con 0x55d6881695b0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f09a7e00640 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688170b60 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f099400bc20 tx=0x7f09940039c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=358c2d6fb5c62c9a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f0992000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f0992000640 1 -- 172.21.15.28:0/3077644500 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0994011650 con 0x55d6881695b0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0998004490 con 0x55d6881695b0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f0992000640 1 -- 172.21.15.28:0/3077644500 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f09940117f0 con 0x55d6881695b0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f0992000640 1 -- 172.21.15.28:0/3077644500 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f09940119f0 con 0x55d6881695b0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f09b2046740 10 client.27014 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d687eaa620 con 0x55d6881695b0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.734+0000 7f0992000640 1 -- 172.21.15.28:0/3077644500 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0998004490 con 0x55d6881695b0 2024-05-20T20:13:07.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.735+0000 7f0992000640 1 -- 172.21.15.28:0/3077644500 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f099401bb90 con 0x55d6881695b0 2024-05-20T20:13:07.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.735+0000 7f09b2046740 10 client.27014 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:07.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.735+0000 7f09b2046740 10 client.27014 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:07.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.735+0000 7f09b2046740 20 client.27014 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:07.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.735+0000 7f09b2046740 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d688183910 0x55d688185d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.735+0000 7f09b2046740 4 client.27014 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:07.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.735+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d6880dbae0 con 0x55d688183910 2024-05-20T20:13:07.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.738+0000 7f09a7400640 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d688183910 0x55d688185d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.739+0000 7f0992000640 10 client.27014 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.739+0000 7f09a7400640 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d688183910 0x55d688185d00 crc :-1 s=READY pgs=173 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cc1df33e5424ad6e server_cookie=fa27bef3a637e144 in_seq=0 out_seq=0 2024-05-20T20:13:07.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.740+0000 7f0992000640 1 -- 172.21.15.28:0/3077644500 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55d6880dbae0 con 0x55d688183910 2024-05-20T20:13:07.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.740+0000 7f0992000640 10 client.27014 handle_command_reply: tid=0 2024-05-20T20:13:07.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.740+0000 7f09b2046740 1 client.27014 shutdown 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 msgr2=0x55d688170b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688170b60 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f099400bc20 tx=0x7f09940039c0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 shutdown_connections 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d688183910 0x55d688185d00 crc :-1 s=READY pgs=173 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d688170b60 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d688171170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 --2- 172.21.15.28:0/3077644500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688173870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 >> 172.21.15.28:0/3077644500 conn(0x55d6881651a0 msgr2=0x55d68816af20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f0992000640 0 client.27014 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 shutdown_connections 2024-05-20T20:13:07.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 1 -- 172.21.15.28:0/3077644500 wait complete. 2024-05-20T20:13:07.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.741+0000 7f09b2046740 20 client.27014 trim_cache size 0 max 16384 2024-05-20T20:13:07.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.762+0000 7f09b2046740 1 Processor -- start 2024-05-20T20:13:07.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.762+0000 7f09b2046740 1 -- start start 2024-05-20T20:13:07.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09b2046740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688169c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09b2046740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d68816faa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09b2046740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d68817faa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09b2046740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f09a81b4c20 con 0x55d688173e70 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09b2046740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f09a81b6760 con 0x55d6881695b0 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09b2046740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f09a81b52f0 con 0x55d688168c90 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09a7e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d68816faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09a7e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d68816faa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58642/0 (socket says 172.21.15.28:58642) 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09a7e00640 1 -- 172.21.15.28:0/3201125052 learned_addr learned my addr 172.21.15.28:0/3201125052 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.763+0000 7f09a7400640 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688169c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.764+0000 7f09a7400640 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d68817faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.764+0000 7f09a7400640 1 -- 172.21.15.28:0/3201125052 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 msgr2=0x55d68816faa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.764+0000 7f09a7400640 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d68816faa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.764+0000 7f09a7400640 1 -- 172.21.15.28:0/3201125052 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 msgr2=0x55d68817faa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.764+0000 7f09a7400640 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d68817faa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.768 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.764+0000 7f09a7400640 1 -- 172.21.15.28:0/3201125052 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d6881636b0 con 0x55d688168c90 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.764+0000 7f09a7400640 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688169c20 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f0998005ab0 tx=0x7f0998004260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f79e80adafae0c6e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.765+0000 7f099cc00640 1 -- 172.21.15.28:0/3201125052 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0998035040 con 0x55d688168c90 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.765+0000 7f099cc00640 1 -- 172.21.15.28:0/3201125052 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f09980054f0 con 0x55d688168c90 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.765+0000 7f099cc00640 1 -- 172.21.15.28:0/3201125052 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0998041040 con 0x55d688168c90 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.765+0000 7f09b2046740 1 -- 172.21.15.28:0/3201125052 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 msgr2=0x55d688169c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.765+0000 7f09b2046740 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688169c20 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f0998005ab0 tx=0x7f0998004260 comp rx=0 tx=0).stop 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.766+0000 7f09b2046740 1 -- 172.21.15.28:0/3201125052 shutdown_connections 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.766+0000 7f09b2046740 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d6881695b0 0x55d68816faa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.766+0000 7f09b2046740 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d688173e70 0x55d68817faa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.766+0000 7f09b2046740 1 --2- 172.21.15.28:0/3201125052 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d688169c20 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.766+0000 7f09b2046740 1 -- 172.21.15.28:0/3201125052 >> 172.21.15.28:0/3201125052 conn(0x55d6881651a0 msgr2=0x55d68816c160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.769 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.767+0000 7f09b2046740 1 -- 172.21.15.28:0/3201125052 shutdown_connections 2024-05-20T20:13:07.770 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.767+0000 7f09b2046740 1 -- 172.21.15.28:0/3201125052 wait complete. 2024-05-20T20:13:07.770 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.767+0000 7f09b2046740 1 -- start start 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 1 Processor -- start 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d68817c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d6881695b0 0x55d688177450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d688173e70 0x55d688177990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f09a81b6090 con 0x55d6881695b0 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f09a81b59c0 con 0x55d688173e70 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f09a81b2c40 con 0x55d688168c90 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.768+0000 7f09b2046740 10 client.0 _ll_register_callbacks cb 0x7f0994004120 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.769+0000 7f09a7400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d68817c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.769+0000 7f09a7400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d68817c0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59646/0 (socket says 172.21.15.28:59646) 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.769+0000 7f09a7400640 1 -- 172.21.15.28:0/2605686531 learned_addr learned my addr 172.21.15.28:0/2605686531 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:07.771 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.769+0000 7f09a7400640 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d688173e70 0x55d688177990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.769+0000 7f09a7e00640 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d6881695b0 0x55d688177450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.770+0000 7f09a7400640 1 -- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d688173e70 msgr2=0x55d688177990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.770+0000 7f09a7400640 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d688173e70 0x55d688177990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.770+0000 7f09a7400640 1 -- 172.21.15.28:0/2605686531 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d6881695b0 msgr2=0x55d688177450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:07.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.770+0000 7f09a7400640 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d6881695b0 0x55d688177450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.770+0000 7f09a7400640 1 -- 172.21.15.28:0/2605686531 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d688177f00 con 0x55d688168c90 2024-05-20T20:13:07.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.772+0000 7f09a7400640 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d68817c0e0 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f0998009230 tx=0x7f0998005850 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7f66193c126f0bea server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:07.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.772+0000 7f0992000640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:07.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.773+0000 7f0992000640 1 -- 172.21.15.28:0/2605686531 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0998067060 con 0x55d688168c90 2024-05-20T20:13:07.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.773+0000 7f0992000640 1 -- 172.21.15.28:0/2605686531 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(83..83 src has 1..83) ==== 8542+0+0 (secure 0 0 0) 0x7f0998005040 con 0x55d688168c90 2024-05-20T20:13:07.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.773+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f09980054f0 con 0x55d688168c90 2024-05-20T20:13:07.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.773+0000 7f0992000640 1 -- 172.21.15.28:0/2605686531 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0998041030 con 0x55d688168c90 2024-05-20T20:13:07.773 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.773+0000 7f09b2046740 10 client.26911 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:07.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.773+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d6881ff220 con 0x55d688168c90 2024-05-20T20:13:07.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.774+0000 7f0992000640 1 -- 172.21.15.28:0/2605686531 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f09980054f0 con 0x55d688168c90 2024-05-20T20:13:07.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.774+0000 7f0992000640 1 -- 172.21.15.28:0/2605686531 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0998031630 con 0x55d688168c90 2024-05-20T20:13:07.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.775+0000 7f09b2046740 10 client.26911 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:07.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.775+0000 7f09b2046740 10 client.26911 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:07.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.775+0000 7f09b2046740 20 client.26911 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:07.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.775+0000 7f09b2046740 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d6881ff450 0x55d688201840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:07.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.775+0000 7f09b2046740 4 client.26911 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:13:07.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.775+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55d6880d52c0 con 0x55d6881ff450 2024-05-20T20:13:07.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.775+0000 7f09a7e00640 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d6881ff450 0x55d688201840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:07.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.776+0000 7f09a7e00640 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d6881ff450 0x55d688201840 crc :-1 s=READY pgs=174 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9969368eaf575b3e server_cookie=915fc95609b99585 in_seq=0 out_seq=0 2024-05-20T20:13:07.776 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.776+0000 7f0992000640 10 client.26911 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.801+0000 7f0992000640 1 -- 172.21.15.28:0/2605686531 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55d6880d52c0 con 0x55d6881ff450 2024-05-20T20:13:07.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.801+0000 7f0992000640 10 client.26911 handle_command_reply: tid=0 2024-05-20T20:13:07.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.801+0000 7f09b2046740 1 client.26911 shutdown 2024-05-20T20:13:07.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.802+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 msgr2=0x55d68817c0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.802+0000 7f09b2046740 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d68817c0e0 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f0998009230 tx=0x7f0998005850 comp rx=0 tx=0).stop 2024-05-20T20:13:07.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.803+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 shutdown_connections 2024-05-20T20:13:07.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.803+0000 7f09b2046740 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d6881ff450 0x55d688201840 crc :-1 s=READY pgs=174 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.803+0000 7f09b2046740 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d688173e70 0x55d688177990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.803+0000 7f09b2046740 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d6881695b0 0x55d688177450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.803+0000 7f09b2046740 1 --2- 172.21.15.28:0/2605686531 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d688168c90 0x55d68817c0e0 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.803+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 >> 172.21.15.28:0/2605686531 conn(0x55d6881651a0 msgr2=0x55d68816a640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.803+0000 7f0992000640 0 client.26911 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:07.804 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.804+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 shutdown_connections 2024-05-20T20:13:07.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.805+0000 7f09b2046740 1 -- 172.21.15.28:0/2605686531 wait complete. 2024-05-20T20:13:07.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.805+0000 7f09b2046740 20 client.26911 trim_cache size 0 max 16384 2024-05-20T20:13:07.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:13:07.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:13:07.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:13:07.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.806+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f098c0b2b60 msgr2=0x7f098c0b5020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.806+0000 7f09af200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f098c0b2b60 0x7f098c0b5020 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f09a00e2ea0 tx=0x7f09a00e2e10 comp rx=0 tx=0).stop 2024-05-20T20:13:07.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.806+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a810f340 msgr2=0x7f09a819fc00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:07.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.806+0000 7f09af200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a810f340 0x7f09a819fc00 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f09a0005fd0 tx=0x7f09a000ee60 comp rx=0 tx=0).stop 2024-05-20T20:13:07.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.807+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 shutdown_connections 2024-05-20T20:13:07.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.807+0000 7f09af200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f09a810f340 0x7f09a819fc00 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.807+0000 7f09af200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f098c0b2b60 0x7f098c0b5020 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.807+0000 7f09af200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f09a8075670 0x7f09a819b8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.807+0000 7f09af200640 1 --2- 172.21.15.28:0/2297141105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f09a8074d50 0x7f09a819b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:07.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.807+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 >> 172.21.15.28:0/2297141105 conn(0x7f09a8079ee0 msgr2=0x7f09a8100b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:07.808 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.808+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 shutdown_connections 2024-05-20T20:13:07.808 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:07.808+0000 7f09af200640 1 -- 172.21.15.28:0/2297141105 wait complete. 2024-05-20T20:13:08.310 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set '4491be01', quiesce took 17.5 seconds. Will release after: 5.800000000000001 2024-05-20T20:13:09.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:08 smithi028 ceph-mon[31178]: pgmap v230: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 273 MiB/s wr, 1.34k op/s 2024-05-20T20:13:09.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:08 smithi154 ceph-mon[36314]: pgmap v230: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 273 MiB/s wr, 1.34k op/s 2024-05-20T20:13:09.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:08 smithi033 ceph-mon[36179]: pgmap v230: 97 pgs: 97 active+clean; 45 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 273 MiB/s wr, 1.34k op/s 2024-05-20T20:13:09.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 -- 172.21.15.28:0/2653415180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 msgr2=0x7f1678075b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 --2- 172.21.15.28:0/2653415180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f1678075b50 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f1668009990 tx=0x7f1668009ca0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 -- 172.21.15.28:0/2653415180 shutdown_connections 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 --2- 172.21.15.28:0/2653415180 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f167810ff40 0x7f1678112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 --2- 172.21.15.28:0/2653415180 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f1678075b50 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 --2- 172.21.15.28:0/2653415180 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 0x7f167810fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 -- 172.21.15.28:0/2653415180 >> 172.21.15.28:0/2653415180 conn(0x7f167807a6a0 msgr2=0x7f167807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.627+0000 7f167f200700 1 -- 172.21.15.28:0/2653415180 shutdown_connections 2024-05-20T20:13:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.628+0000 7f167f200700 1 -- 172.21.15.28:0/2653415180 wait complete. 2024-05-20T20:13:09.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 Processor -- start 2024-05-20T20:13:09.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 -- start start 2024-05-20T20:13:09.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f167819be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 0x7f167819c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f167810ff40 0x7f16781a0700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1678115960 con 0x7f167810ff40 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f16781157e0 con 0x7f1678075770 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.629+0000 7f167f200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1678115ae0 con 0x7f1678076090 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 0x7f167819c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f167819be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f167819be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58680/0 (socket says 172.21.15.28:58680) 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f167d200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f167810ff40 0x7f16781a0700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 0x7f167819c370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59658/0 (socket says 172.21.15.28:59658) 2024-05-20T20:13:09.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677e00700 1 -- 172.21.15.28:0/2500970551 learned_addr learned my addr 172.21.15.28:0/2500970551 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:09.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 -- 172.21.15.28:0/2500970551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 msgr2=0x7f167819be30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f167819be30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 -- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f167810ff40 msgr2=0x7f16781a0700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f167810ff40 0x7f16781a0700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 -- 172.21.15.28:0/2500970551 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f16781a0e40 con 0x7f1678076090 2024-05-20T20:13:09.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677e00700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f167819be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:09.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.630+0000 7f1677400700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 0x7f167819c370 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f166400e9c0 tx=0x7f166400ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:09.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.631+0000 7f1674c00700 1 -- 172.21.15.28:0/2500970551 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1664004230 con 0x7f1678076090 2024-05-20T20:13:09.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.631+0000 7f1674c00700 1 -- 172.21.15.28:0/2500970551 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f16640043d0 con 0x7f1678076090 2024-05-20T20:13:09.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.631+0000 7f1674c00700 1 -- 172.21.15.28:0/2500970551 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1664003dc0 con 0x7f1678076090 2024-05-20T20:13:09.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.631+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f16781a1130 con 0x7f1678076090 2024-05-20T20:13:09.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.631+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f16781a89b0 con 0x7f1678076090 2024-05-20T20:13:09.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.635+0000 7f1674c00700 1 -- 172.21.15.28:0/2500970551 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f166400c7c0 con 0x7f1678076090 2024-05-20T20:13:09.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.636+0000 7f1689c83b80 1 Processor -- start 2024-05-20T20:13:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.636+0000 7f1689c83b80 1 -- start start 2024-05-20T20:13:09.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c669f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c66f30 0x55d2d7c60b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c61140 0x55d2d7c615c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1689c83b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d2d7bbee40 con 0x55d2d7c61140 2024-05-20T20:13:09.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1689c83b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d2d7bae9e0 con 0x55d2d7c66610 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1689c83b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d2d7bb2a20 con 0x55d2d7c66f30 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c66f30 0x55d2d7c60b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f167d200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c669f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c66f30 0x55d2d7c60b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59668/0 (socket says 172.21.15.28:59668) 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 -- 172.21.15.28:0/3645178467 learned_addr learned my addr 172.21.15.28:0/3645178467 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f167d200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c669f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58694/0 (socket says 172.21.15.28:58694) 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677400700 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c61140 0x55d2d7c615c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 -- 172.21.15.28:0/3645178467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 msgr2=0x55d2d7c669f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c669f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 -- 172.21.15.28:0/3645178467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c61140 msgr2=0x55d2d7c615c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c61140 0x55d2d7c615c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.637+0000 7f1677e00700 1 -- 172.21.15.28:0/3645178467 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d2d7b77420 con 0x55d2d7c66f30 2024-05-20T20:13:09.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.638+0000 7f1677e00700 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c66f30 0x55d2d7c60b20 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f16680096d0 tx=0x7f166800b260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f62e08bab7ca05b6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:09.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.638+0000 7f1674c00700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f165c0b2a70 0x7f165c0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.638+0000 7f166d600700 1 -- 172.21.15.28:0/3645178467 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1668027070 con 0x55d2d7c66f30 2024-05-20T20:13:09.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.638+0000 7f166d600700 1 -- 172.21.15.28:0/3645178467 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1668003680 con 0x55d2d7c66f30 2024-05-20T20:13:09.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.638+0000 7f1674c00700 1 -- 172.21.15.28:0/2500970551 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(84..84 src has 1..84) ==== 8586+0+0 (secure 0 0 0) 0x7f16640e2850 con 0x7f1678076090 2024-05-20T20:13:09.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.638+0000 7f166d600700 1 -- 172.21.15.28:0/3645178467 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f166800f6c0 con 0x55d2d7c66f30 2024-05-20T20:13:09.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.638+0000 7f167d200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f165c0b2a70 0x7f165c0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f1689c83b80 1 -- 172.21.15.28:0/3645178467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c66f30 msgr2=0x55d2d7c60b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f167d200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f165c0b2a70 0x7f165c0b4f30 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f167806bdd0 tx=0x7f167000b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:09.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c66f30 0x55d2d7c60b20 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f16680096d0 tx=0x7f166800b260 comp rx=0 tx=0).stop 2024-05-20T20:13:09.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f1689c83b80 1 -- 172.21.15.28:0/3645178467 shutdown_connections 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c61140 0x55d2d7c615c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c669f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3645178467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c66f30 0x55d2d7c60b20 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.639+0000 7f1689c83b80 1 -- 172.21.15.28:0/3645178467 >> 172.21.15.28:0/3645178467 conn(0x55d2d7b8ebe0 msgr2=0x55d2d7c5b690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.640+0000 7f1689c83b80 1 -- 172.21.15.28:0/3645178467 shutdown_connections 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.640+0000 7f1689c83b80 1 -- 172.21.15.28:0/3645178467 wait complete. 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.640+0000 7f1689c83b80 1 -- start start 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.640+0000 7f1689c83b80 1 Processor -- start 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c69fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1689c83b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d2d7ab4fa0 con 0x55d2d7c66f30 2024-05-20T20:13:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1689c83b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d2d7b4f760 con 0x55d2d7c66610 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1689c83b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d2d7c6d980 con 0x55d2d7c61140 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1689c83b80 10 client.0 _ll_register_callbacks cb 0x55d2d7b58c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1677400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c69fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1677400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c69fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58710/0 (socket says 172.21.15.28:58710) 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59672/0 (socket says 172.21.15.28:59672) 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f1677400700 1 -- 172.21.15.28:0/3756198714 learned_addr learned my addr 172.21.15.28:0/3756198714 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.641+0000 7f167d200700 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1677400700 1 -- 172.21.15.28:0/3756198714 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 msgr2=0x55d2d7c6cc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1677400700 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1677400700 1 -- 172.21.15.28:0/3756198714 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66f30 msgr2=0x55d2d7c6a510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1677400700 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1677400700 1 -- 172.21.15.28:0/3756198714 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d2d7afe8d0 con 0x55d2d7c66610 2024-05-20T20:13:09.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1677400700 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c69fd0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f166400f010 tx=0x7f16640a5fd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f6f281d2fa8c4f6f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:09.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1662000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:09.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1662000700 1 -- 172.21.15.28:0/3756198714 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1664003680 con 0x55d2d7c66610 2024-05-20T20:13:09.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1662000700 1 -- 172.21.15.28:0/3756198714 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(84..84 src has 1..84) ==== 8586+0+0 (secure 0 0 0) 0x7f16640a9f00 con 0x55d2d7c66610 2024-05-20T20:13:09.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.642+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1668003680 con 0x55d2d7c66610 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1662000700 1 -- 172.21.15.28:0/3756198714 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f16640a95a0 con 0x55d2d7c66610 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 10 client.27032 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d2d7b7f060 con 0x55d2d7c66610 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1662000700 1 -- 172.21.15.28:0/3756198714 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1668003680 con 0x55d2d7c66610 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1662000700 1 -- 172.21.15.28:0/3756198714 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f16640b2460 con 0x55d2d7c66610 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 10 client.27032 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 10 client.27032 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 20 client.27032 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7c7a3e0 0x55d2d7c7c7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 4 client.27032 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.643+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d2d7bc0d90 con 0x55d2d7c7a3e0 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.644+0000 7f1677e00700 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7c7a3e0 0x55d2d7c7c7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.644+0000 7f1677e00700 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7c7a3e0 0x55d2d7c7c7d0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=e7e34bec5798e7 server_cookie=874c0c32063b72c7 in_seq=0 out_seq=0 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.644+0000 7f1662000700 10 client.27032 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.645+0000 7f1662000700 1 -- 172.21.15.28:0/3756198714 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55d2d7bc0d90 con 0x55d2d7c7a3e0 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.645+0000 7f1662000700 10 client.27032 handle_command_reply: tid=0 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.645+0000 7f1689c83b80 1 client.27032 shutdown 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 msgr2=0x55d2d7c69fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c69fd0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f166400f010 tx=0x7f16640a5fd0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 shutdown_connections 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7c7a3e0 0x55d2d7c7c7d0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3756198714 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66610 0x55d2d7c69fd0 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1662000700 0 client.27032 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:13:09.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 >> 172.21.15.28:0/3756198714 conn(0x55d2d7b8ebe0 msgr2=0x55d2d7c5ad70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:09.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 shutdown_connections 2024-05-20T20:13:09.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 1 -- 172.21.15.28:0/3756198714 wait complete. 2024-05-20T20:13:09.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.646+0000 7f1689c83b80 20 client.27032 trim_cache size 0 max 16384 2024-05-20T20:13:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.663+0000 7f1689c83b80 1 Processor -- start 2024-05-20T20:13:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.663+0000 7f1689c83b80 1 -- start start 2024-05-20T20:13:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c67ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1689c83b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f16781b01a0 con 0x55d2d7c66610 2024-05-20T20:13:09.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1689c83b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f16781b1ce0 con 0x55d2d7c66f30 2024-05-20T20:13:09.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1689c83b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f16781b0870 con 0x55d2d7c61140 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1677400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c67ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1677400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c67ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34098/0 (socket says 172.21.15.28:34098) 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59688/0 (socket says 172.21.15.28:59688) 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1677400700 1 -- 172.21.15.28:0/3217184300 learned_addr learned my addr 172.21.15.28:0/3217184300 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f167d200700 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.664+0000 7f1677e00700 1 -- 172.21.15.28:0/3217184300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 msgr2=0x55d2d7c6a420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f1677e00700 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f1677e00700 1 -- 172.21.15.28:0/3217184300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 msgr2=0x55d2d7c67ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f1677e00700 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c67ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f1677e00700 1 -- 172.21.15.28:0/3217184300 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d2d7c6c870 con 0x55d2d7c61140 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f1677400700 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c67ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f1677e00700 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f16680096a0 tx=0x7f166800b650 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d0b3117cf5b96bb2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f166cc00700 1 -- 172.21.15.28:0/3217184300 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1668032070 con 0x55d2d7c61140 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f166cc00700 1 -- 172.21.15.28:0/3217184300 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f166800be70 con 0x55d2d7c61140 2024-05-20T20:13:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.665+0000 7f166cc00700 1 -- 172.21.15.28:0/3217184300 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1668027040 con 0x55d2d7c61140 2024-05-20T20:13:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 -- 172.21.15.28:0/3217184300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 msgr2=0x55d2d7c6cc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f16680096a0 tx=0x7f166800b650 comp rx=0 tx=0).stop 2024-05-20T20:13:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 -- 172.21.15.28:0/3217184300 shutdown_connections 2024-05-20T20:13:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c6a420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c6cc30 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 --2- 172.21.15.28:0/3217184300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c67ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 -- 172.21.15.28:0/3217184300 >> 172.21.15.28:0/3217184300 conn(0x55d2d7b8ebe0 msgr2=0x55d2d7c68920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:09.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 -- 172.21.15.28:0/3217184300 shutdown_connections 2024-05-20T20:13:09.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.666+0000 7f1689c83b80 1 -- 172.21.15.28:0/3217184300 wait complete. 2024-05-20T20:13:09.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.667+0000 7f1689c83b80 1 -- start start 2024-05-20T20:13:09.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.667+0000 7f1689c83b80 1 Processor -- start 2024-05-20T20:13:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c73150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c816f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1689c83b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c81c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1689c83b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f16781b1610 con 0x55d2d7c66610 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1689c83b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f16781b0f40 con 0x55d2d7c66f30 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1689c83b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f16781ae160 con 0x55d2d7c61140 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1689c83b80 10 client.0 _ll_register_callbacks cb 0x55d2d7b58c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c73150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1677e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c73150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59704/0 (socket says 172.21.15.28:59704) 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1677e00700 1 -- 172.21.15.28:0/490104318 learned_addr learned my addr 172.21.15.28:0/490104318 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f1677400700 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c816f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.668+0000 7f167d200700 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c81c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1677e00700 1 -- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 msgr2=0x55d2d7c81c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1677e00700 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c81c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1677e00700 1 -- 172.21.15.28:0/490104318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 msgr2=0x55d2d7c816f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1677e00700 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c816f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1677e00700 1 -- 172.21.15.28:0/490104318 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d2d7c82220 con 0x55d2d7c61140 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1677e00700 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c73150 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f1668003ab0 tx=0x7f1668003ae0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=578cba7dd21dc2d3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1662000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1662000700 1 -- 172.21.15.28:0/490104318 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f166803d070 con 0x55d2d7c61140 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1662000700 1 -- 172.21.15.28:0/490104318 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(84..84 src has 1..84) ==== 8586+0+0 (secure 0 0 0) 0x7f16680052b0 con 0x55d2d7c61140 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.669+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f166800be70 con 0x55d2d7c61140 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1662000700 1 -- 172.21.15.28:0/490104318 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1668027030 con 0x55d2d7c61140 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 10 client.26929 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d2d7ce4d70 con 0x55d2d7c61140 2024-05-20T20:13:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1662000700 1 -- 172.21.15.28:0/490104318 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f166800be70 con 0x55d2d7c61140 2024-05-20T20:13:09.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1662000700 1 -- 172.21.15.28:0/490104318 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1668040020 con 0x55d2d7c61140 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 10 client.26929 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 10 client.26929 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 20 client.26929 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7ce4ff0 0x55d2d7ce73e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 4 client.26929 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.670+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55d2d7bba570 con 0x55d2d7ce4ff0 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.671+0000 7f1677400700 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7ce4ff0 0x55d2d7ce73e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.671+0000 7f1677400700 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7ce4ff0 0x55d2d7ce73e0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=4a9055f669b74fd6 server_cookie=25aae53bd6b45fa2 in_seq=0 out_seq=0 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.671+0000 7f1662000700 10 client.26929 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.671+0000 7f1662000700 1 -- 172.21.15.28:0/490104318 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55d2d7bba570 con 0x55d2d7ce4ff0 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.671+0000 7f1662000700 10 client.26929 handle_command_reply: tid=0 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.671+0000 7f1689c83b80 1 client.26929 shutdown 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 msgr2=0x55d2d7c73150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c73150 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f1668003ab0 tx=0x7f1668003ae0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 shutdown_connections 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55d2d7ce4ff0 0x55d2d7ce73e0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d7c66f30 0x55d2d7c81c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d7c61140 0x55d2d7c73150 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 --2- 172.21.15.28:0/490104318 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d7c66610 0x55d2d7c816f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1662000700 0 client.26929 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 >> 172.21.15.28:0/490104318 conn(0x55d2d7b8ebe0 msgr2=0x55d2d7c65180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 shutdown_connections 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 1 -- 172.21.15.28:0/490104318 wait complete. 2024-05-20T20:13:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.672+0000 7f1689c83b80 20 client.26929 trim_cache size 0 max 16384 2024-05-20T20:13:09.675 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:13:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f165c0b2a70 msgr2=0x7f165c0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f165c0b2a70 0x7f165c0b4f30 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f167806bdd0 tx=0x7f167000b040 comp rx=0 tx=0).stop 2024-05-20T20:13:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 msgr2=0x7f167819c370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 0x7f167819c370 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f166400e9c0 tx=0x7f166400ecd0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 shutdown_connections 2024-05-20T20:13:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f167810ff40 0x7f16781a0700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f165c0b2a70 0x7f165c0b4f30 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1678075770 0x7f167819be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 --2- 172.21.15.28:0/2500970551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1678076090 0x7f167819c370 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.673+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 >> 172.21.15.28:0/2500970551 conn(0x7f167807a6a0 msgr2=0x7f1678100710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.674+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 shutdown_connections 2024-05-20T20:13:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:09.674+0000 7f167f200700 1 -- 172.21.15.28:0/2500970551 wait complete. 2024-05-20T20:13:10.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:09 smithi028 ceph-mon[31178]: osdmap e84: 12 total, 12 up, 12 in 2024-05-20T20:13:10.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:09 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:13:10.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:09 smithi154 ceph-mon[36314]: osdmap e84: 12 total, 12 up, 12 in 2024-05-20T20:13:10.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:09 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:13:10.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:09 smithi033 ceph-mon[36179]: osdmap e84: 12 total, 12 up, 12 in 2024-05-20T20:13:10.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:09 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:13:10.645 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:13:11.106 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:11.259 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:10 smithi028 ceph-mon[31178]: pgmap v232: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 299 MiB/s wr, 1.60k op/s 2024-05-20T20:13:11.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:10 smithi154 ceph-mon[36314]: pgmap v232: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 299 MiB/s wr, 1.60k op/s 2024-05-20T20:13:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:10 smithi033 ceph-mon[36179]: pgmap v232: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 299 MiB/s wr, 1.60k op/s 2024-05-20T20:13:12.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:11 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:13:12.298 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:11 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:13:12.299 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:11 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:13:12.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:11 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:13:12.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:11 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:13:12.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:11 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:13:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:11 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:13:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:11 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:13:12.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:11 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:13:13.261 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:12 smithi028 ceph-mon[31178]: pgmap v233: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 284 MiB/s wr, 1.54k op/s 2024-05-20T20:13:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:12 smithi154 ceph-mon[36314]: pgmap v233: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 284 MiB/s wr, 1.54k op/s 2024-05-20T20:13:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:12 smithi033 ceph-mon[36179]: pgmap v233: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 284 MiB/s wr, 1.54k op/s 2024-05-20T20:13:13.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.783+0000 7f5df2600700 1 -- 172.21.15.28:0/828548893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10abf0 msgr2=0x7f5dec10afd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.783+0000 7f5df2600700 1 --2- 172.21.15.28:0/828548893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10abf0 0x7f5dec10afd0 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f5dd40098f0 tx=0x7f5dd4009c00 comp rx=0 tx=0).stop 2024-05-20T20:13:13.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.784+0000 7f5df2600700 1 -- 172.21.15.28:0/828548893 shutdown_connections 2024-05-20T20:13:13.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.784+0000 7f5df2600700 1 --2- 172.21.15.28:0/828548893 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5dec10f190 0x7f5dec07c960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.784+0000 7f5df2600700 1 --2- 172.21.15.28:0/828548893 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10abf0 0x7f5dec10afd0 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.784+0000 7f5df2600700 1 --2- 172.21.15.28:0/828548893 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5dec10e7d0 0x7f5dec10ec50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.785+0000 7f5df2600700 1 -- 172.21.15.28:0/828548893 >> 172.21.15.28:0/828548893 conn(0x7f5dec06e6b0 msgr2=0x7f5dec06eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:13.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.785+0000 7f5df2600700 1 -- 172.21.15.28:0/828548893 shutdown_connections 2024-05-20T20:13:13.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.785+0000 7f5df2600700 1 -- 172.21.15.28:0/828548893 wait complete. 2024-05-20T20:13:13.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.787+0000 7f5df2600700 1 Processor -- start 2024-05-20T20:13:13.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.787+0000 7f5df2600700 1 -- start start 2024-05-20T20:13:13.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.788+0000 7f5df2600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5dec10abf0 0x7f5dec07c820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.788+0000 7f5df2600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5dec10e7d0 0x7f5dec076930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.788+0000 7f5df2600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10f190 0x7f5dec076f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.788+0000 7f5df2600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec07e330 con 0x7f5dec10f190 2024-05-20T20:13:13.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.788+0000 7f5df2600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec07e1b0 con 0x7f5dec10e7d0 2024-05-20T20:13:13.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.788+0000 7f5df2600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec07e4b0 con 0x7f5dec10abf0 2024-05-20T20:13:13.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.789+0000 7f5deb400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5dec10abf0 0x7f5dec07c820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.789+0000 7f5deb400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5dec10abf0 0x7f5dec07c820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59720/0 (socket says 172.21.15.28:59720) 2024-05-20T20:13:13.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.789+0000 7f5deb400700 1 -- 172.21.15.28:0/1966071278 learned_addr learned my addr 172.21.15.28:0/1966071278 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:13.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.789+0000 7f5de2a00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5dec10e7d0 0x7f5dec076930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.789+0000 7f5debe00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10f190 0x7f5dec076f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.789+0000 7f5de2a00700 1 -- 172.21.15.28:0/1966071278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5dec10abf0 msgr2=0x7f5dec07c820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.789+0000 7f5de2a00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5dec10abf0 0x7f5dec07c820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.790+0000 7f5de2a00700 1 -- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10f190 msgr2=0x7f5dec076f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.790+0000 7f5de2a00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10f190 0x7f5dec076f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.790+0000 7f5de2a00700 1 -- 172.21.15.28:0/1966071278 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5dec138a40 con 0x7f5dec10e7d0 2024-05-20T20:13:13.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.790+0000 7f5de2a00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5dec10e7d0 0x7f5dec076930 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc00c970 tx=0x7f5ddc00cc80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:13.791 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.791+0000 7f5de8c00700 1 -- 172.21.15.28:0/1966071278 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc004a30 con 0x7f5dec10e7d0 2024-05-20T20:13:13.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.791+0000 7f5de8c00700 1 -- 172.21.15.28:0/1966071278 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5ddc0054d0 con 0x7f5dec10e7d0 2024-05-20T20:13:13.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.791+0000 7f5de8c00700 1 -- 172.21.15.28:0/1966071278 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc00eb40 con 0x7f5dec10e7d0 2024-05-20T20:13:13.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.793+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5dec138d30 con 0x7f5dec10e7d0 2024-05-20T20:13:13.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.793+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5dec1391f0 con 0x7f5dec10e7d0 2024-05-20T20:13:13.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.806+0000 7f5dfd044b80 1 Processor -- start 2024-05-20T20:13:13.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.806+0000 7f5dfd044b80 1 -- start start 2024-05-20T20:13:13.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.807+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f7972d290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.807+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79727260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.807+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79727ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.807+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556f79685e40 con 0x556f7972ceb0 2024-05-20T20:13:13.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.807+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556f796759e0 con 0x556f7972d7d0 2024-05-20T20:13:13.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.807+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556f79679a20 con 0x556f79727880 2024-05-20T20:13:13.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.808+0000 7f5debe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f7972d290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.808+0000 7f5debe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f7972d290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:34124/0 (socket says 172.21.15.28:34124) 2024-05-20T20:13:13.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.808+0000 7f5debe00700 1 -- 172.21.15.28:0/2501161098 learned_addr learned my addr 172.21.15.28:0/2501161098 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:13.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.808+0000 7f5debe00700 1 -- 172.21.15.28:0/2501161098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 msgr2=0x556f79727ce0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:13:13.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.808+0000 7f5debe00700 1 --2- 172.21.15.28:0/2501161098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79727ce0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.808+0000 7f5debe00700 1 -- 172.21.15.28:0/2501161098 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 msgr2=0x556f79727260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:13.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.808+0000 7f5debe00700 1 --2- 172.21.15.28:0/2501161098 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79727260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.809+0000 7f5debe00700 1 -- 172.21.15.28:0/2501161098 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556f7963e420 con 0x556f7972ceb0 2024-05-20T20:13:13.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.809+0000 7f5debe00700 1 --2- 172.21.15.28:0/2501161098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f7972d290 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f5de4003e90 tx=0x7f5de4009c00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aece759eade0dbd0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:13.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.810+0000 7f5de0c00700 1 -- 172.21.15.28:0/2501161098 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de4007bd0 con 0x556f7972ceb0 2024-05-20T20:13:13.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.810+0000 7f5de0c00700 1 -- 172.21.15.28:0/2501161098 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5de401a070 con 0x556f7972ceb0 2024-05-20T20:13:13.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.810+0000 7f5de0c00700 1 -- 172.21.15.28:0/2501161098 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de40156e0 con 0x556f7972ceb0 2024-05-20T20:13:13.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.811+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2501161098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 msgr2=0x556f7972d290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.811+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2501161098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f7972d290 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f5de4003e90 tx=0x7f5de4009c00 comp rx=0 tx=0).stop 2024-05-20T20:13:13.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.812+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2501161098 shutdown_connections 2024-05-20T20:13:13.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.812+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2501161098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79727ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.812+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2501161098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f7972d290 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.812+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2501161098 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79727260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.812+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2501161098 >> 172.21.15.28:0/2501161098 conn(0x556f79655be0 msgr2=0x556f79722010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:13.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.813+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2501161098 shutdown_connections 2024-05-20T20:13:13.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.813+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2501161098 wait complete. 2024-05-20T20:13:13.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.814+0000 7f5dfd044b80 1 -- start start 2024-05-20T20:13:13.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.815+0000 7f5de8c00700 1 -- 172.21.15.28:0/1966071278 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f5ddc004bd0 con 0x7f5dec10e7d0 2024-05-20T20:13:13.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.817+0000 7f5dfd044b80 1 Processor -- start 2024-05-20T20:13:13.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.818+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79733540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.819+0000 7f5debe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79733540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.819+0000 7f5debe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79733540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59750/0 (socket says 172.21.15.28:59750) 2024-05-20T20:13:13.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.819+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f79730860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.819+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79730da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.819+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556f7957bfa0 con 0x556f7972ceb0 2024-05-20T20:13:13.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.819+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556f79616760 con 0x556f7972d7d0 2024-05-20T20:13:13.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.819+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556f79734240 con 0x556f79727880 2024-05-20T20:13:13.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.820+0000 7f5dfd044b80 10 client.0 _ll_register_callbacks cb 0x556f7961fc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:13.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.820+0000 7f5de8c00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5dd00b2b00 0x7f5dd00b4fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.820+0000 7f5debe00700 1 -- 172.21.15.28:0/1728214178 learned_addr learned my addr 172.21.15.28:0/1728214178 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:13.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.821+0000 7f5deb400700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f79730860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.821+0000 7f5de8c00700 1 -- 172.21.15.28:0/1966071278 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(84..84 src has 1..84) ==== 8586+0+0 (secure 0 0 0) 0x7f5ddc0e22c0 con 0x7f5dec10e7d0 2024-05-20T20:13:13.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.821+0000 7f5debe00700 1 -- 172.21.15.28:0/1728214178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 msgr2=0x556f79730da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:13.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.822+0000 7f5de2a00700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79730da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.822+0000 7f5debe00700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79730da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.822+0000 7f5debe00700 1 -- 172.21.15.28:0/1728214178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 msgr2=0x556f79730860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:13.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.822+0000 7f5debe00700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f79730860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.822+0000 7f5debe00700 1 -- 172.21.15.28:0/1728214178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556f795c58d0 con 0x556f79727880 2024-05-20T20:13:13.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.823+0000 7f5debe00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5dd00b2b00 0x7f5dd00b4fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.823+0000 7f5deb400700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f79730860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:13.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.823+0000 7f5de2a00700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79730da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:13.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.823+0000 7f5debe00700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79733540 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f5de4005bd0 tx=0x7f5de40043b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8f8dce218f7b18fa server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:13.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.824+0000 7f5dd9600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:13.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.825+0000 7f5dd9600700 1 -- 172.21.15.28:0/1728214178 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5de4015e60 con 0x556f79727880 2024-05-20T20:13:13.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.825+0000 7f5dd9600700 1 -- 172.21.15.28:0/1728214178 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(84..84 src has 1..84) ==== 8586+0+0 (secure 0 0 0) 0x7f5de4021620 con 0x556f79727880 2024-05-20T20:13:13.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.825+0000 7f5debe00700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5dd00b2b00 0x7f5dd00b4fc0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f5de4004800 tx=0x7f5de4026000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:13.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.827+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5de401a070 con 0x556f79727880 2024-05-20T20:13:13.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.831+0000 7f5dd9600700 1 -- 172.21.15.28:0/1728214178 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5de4021820 con 0x556f79727880 2024-05-20T20:13:13.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.831+0000 7f5dfd044b80 10 client.26938 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:13.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.831+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556f79646060 con 0x556f79727880 2024-05-20T20:13:13.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.832+0000 7f5dd9600700 1 -- 172.21.15.28:0/1728214178 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5de401a070 con 0x556f79727880 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.833+0000 7f5dd9600700 1 -- 172.21.15.28:0/1728214178 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5de4028040 con 0x556f79727880 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.833+0000 7f5dfd044b80 10 client.26938 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.833+0000 7f5dfd044b80 10 client.26938 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.833+0000 7f5dfd044b80 20 client.26938 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.833+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f79740d90 0x556f79743180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.834+0000 7f5dfd044b80 4 client.26938 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.834+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556f79687d90 con 0x556f79740d90 2024-05-20T20:13:13.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.834+0000 7f5deb400700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f79740d90 0x556f79743180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.836+0000 7f5deb400700 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f79740d90 0x556f79743180 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=b4f11a31ae1bef3f server_cookie=51b2bdca94cad66f in_seq=0 out_seq=0 2024-05-20T20:13:13.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.836+0000 7f5dd9600700 10 client.26938 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:13:13.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.838+0000 7f5dd9600700 1 -- 172.21.15.28:0/1728214178 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x556f79687d90 con 0x556f79740d90 2024-05-20T20:13:13.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.839+0000 7f5dd9600700 10 client.26938 handle_command_reply: tid=0 2024-05-20T20:13:13.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.839+0000 7f5dfd044b80 1 client.26938 shutdown 2024-05-20T20:13:13.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.850+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 msgr2=0x556f79733540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.850+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79733540 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f5de4005bd0 tx=0x7f5de40043b0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.851+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 shutdown_connections 2024-05-20T20:13:13.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.851+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972d7d0 0x556f79730da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.851+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f79733540 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.851+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f79740d90 0x556f79743180 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.851+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/1728214178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972ceb0 0x556f79730860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.851+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 >> 172.21.15.28:0/1728214178 conn(0x556f79655be0 msgr2=0x556f79726250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:13.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.851+0000 7f5dd9600700 0 client.26938 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:13:13.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.856+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 shutdown_connections 2024-05-20T20:13:13.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.858+0000 7f5dfd044b80 1 -- 172.21.15.28:0/1728214178 wait complete. 2024-05-20T20:13:13.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.858+0000 7f5dfd044b80 20 client.26938 trim_cache size 0 max 16384 2024-05-20T20:13:13.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.896+0000 7f5dfd044b80 1 Processor -- start 2024-05-20T20:13:13.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.897+0000 7f5dfd044b80 1 -- start start 2024-05-20T20:13:13.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.897+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f797220f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.897+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972ceb0 0x556f79748990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.897+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f79748ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.897+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1d64e0 con 0x556f7972d7d0 2024-05-20T20:13:13.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.897+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1d8020 con 0x556f7972ceb0 2024-05-20T20:13:13.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.897+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1d6bb0 con 0x556f79727880 2024-05-20T20:13:13.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.898+0000 7f5de2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972ceb0 0x556f79748990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5de2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972ceb0 0x556f79748990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58758/0 (socket says 172.21.15.28:58758) 2024-05-20T20:13:13.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5de2a00700 1 -- 172.21.15.28:0/2744686644 learned_addr learned my addr 172.21.15.28:0/2744686644 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:13.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5debe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f79748ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5de2a00700 1 -- 172.21.15.28:0/2744686644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 msgr2=0x556f797220f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:13.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5deb400700 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f797220f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5de2a00700 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f797220f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5de2a00700 1 -- 172.21.15.28:0/2744686644 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 msgr2=0x556f79748ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5de2a00700 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f79748ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5de2a00700 1 -- 172.21.15.28:0/2744686644 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556f7971a190 con 0x556f7972ceb0 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5deb400700 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f797220f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.899+0000 7f5debe00700 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f79748ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.900+0000 7f5de2a00700 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972ceb0 0x556f79748990 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc0a9280 tx=0x7f5ddc0a5570 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7e05a311d01522b7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.900+0000 7f5ddbe00700 1 -- 172.21.15.28:0/2744686644 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc0a9580 con 0x556f7972ceb0 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.900+0000 7f5ddbe00700 1 -- 172.21.15.28:0/2744686644 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5ddc0ad930 con 0x556f7972ceb0 2024-05-20T20:13:13.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.900+0000 7f5ddbe00700 1 -- 172.21.15.28:0/2744686644 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc0a97e0 con 0x556f7972ceb0 2024-05-20T20:13:13.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.901+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2744686644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972ceb0 msgr2=0x556f79748990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.901+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972ceb0 0x556f79748990 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc0a9280 tx=0x7f5ddc0a5570 comp rx=0 tx=0).stop 2024-05-20T20:13:13.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.901+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2744686644 shutdown_connections 2024-05-20T20:13:13.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.901+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f79748ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.901+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f79727880 0x556f797220f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.901+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/2744686644 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f7972ceb0 0x556f79748990 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.901+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2744686644 >> 172.21.15.28:0/2744686644 conn(0x556f79655be0 msgr2=0x556f797219e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:13.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.902+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2744686644 shutdown_connections 2024-05-20T20:13:13.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.902+0000 7f5dfd044b80 1 -- 172.21.15.28:0/2744686644 wait complete. 2024-05-20T20:13:13.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.902+0000 7f5dfd044b80 1 -- start start 2024-05-20T20:13:13.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.902+0000 7f5dfd044b80 1 Processor -- start 2024-05-20T20:13:13.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f79727880 0x556f797399b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f7972ceb0 0x556f79747f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5dfd044b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f797484a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1d7950 con 0x556f7972d7d0 2024-05-20T20:13:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1d7280 con 0x556f79727880 2024-05-20T20:13:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5dfd044b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5dec1d4550 con 0x556f7972ceb0 2024-05-20T20:13:13.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5dfd044b80 10 client.0 _ll_register_callbacks cb 0x556f7961fc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f79727880 0x556f797399b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5de2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f7972ceb0 0x556f79747f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f79727880 0x556f797399b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58766/0 (socket says 172.21.15.28:58766) 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 -- 172.21.15.28:0/972410270 learned_addr learned my addr 172.21.15.28:0/972410270 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5debe00700 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f797484a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 -- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f7972ceb0 msgr2=0x556f79747f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f7972ceb0 0x556f79747f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 -- 172.21.15.28:0/972410270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 msgr2=0x556f797484a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f797484a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.903+0000 7f5deb400700 1 -- 172.21.15.28:0/972410270 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556f79748a60 con 0x556f79727880 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5de2a00700 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f7972ceb0 0x556f79747f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:13:13.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5deb400700 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f79727880 0x556f797399b0 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f5dd4009440 tx=0x7f5dd40051a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bea5bb95eae8cc1b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5dd9600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5dd9600700 1 -- 172.21.15.28:0/972410270 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5dd4003880 con 0x556f79727880 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5dd9600700 1 -- 172.21.15.28:0/972410270 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(84..84 src has 1..84) ==== 8586+0+0 (secure 0 0 0) 0x7f5dd4003a20 con 0x556f79727880 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5ddc0ad930 con 0x556f79727880 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5dd9600700 1 -- 172.21.15.28:0/972410270 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5dd4005690 con 0x556f79727880 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5dfd044b80 10 client.27047 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.904+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556f797ab580 con 0x556f79727880 2024-05-20T20:13:13.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dd9600700 1 -- 172.21.15.28:0/972410270 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5dd4003c40 con 0x556f79727880 2024-05-20T20:13:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dd9600700 1 -- 172.21.15.28:0/972410270 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5dd4034030 con 0x556f79727880 2024-05-20T20:13:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dfd044b80 10 client.27047 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dfd044b80 10 client.27047 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:13:13.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dfd044b80 20 client.27047 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f797abbe0 0x556f797ae000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dfd044b80 4 client.27047 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.905+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x7f5ddc0ad930 con 0x556f797abbe0 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.907+0000 7f5de2a00700 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f797abbe0 0x556f797ae000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.907+0000 7f5de2a00700 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f797abbe0 0x556f797ae000 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=7f3992e93cd3bebe server_cookie=b2f3128b1cc341ed in_seq=0 out_seq=0 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.907+0000 7f5dd9600700 10 client.27047 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dd9600700 1 -- 172.21.15.28:0/972410270 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x7f5ddc0ad930 con 0x556f797abbe0 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dd9600700 10 client.27047 handle_command_reply: tid=0 2024-05-20T20:13:13.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 client.27047 shutdown 2024-05-20T20:13:13.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f79727880 msgr2=0x556f797399b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f79727880 0x556f797399b0 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f5dd4009440 tx=0x7f5dd40051a0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 shutdown_connections 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x556f797abbe0 0x556f797ae000 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f7972d7d0 0x556f797484a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f79727880 0x556f797399b0 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 --2- 172.21.15.28:0/972410270 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f7972ceb0 0x556f79747f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 >> 172.21.15.28:0/972410270 conn(0x556f79655be0 msgr2=0x556f79730640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.908+0000 7f5dd9600700 0 client.27047 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.909+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 shutdown_connections 2024-05-20T20:13:13.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.909+0000 7f5dfd044b80 1 -- 172.21.15.28:0/972410270 wait complete. 2024-05-20T20:13:13.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.910+0000 7f5dfd044b80 20 client.27047 trim_cache size 0 max 16384 2024-05-20T20:13:13.917 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:13:13.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.919+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5dd00b2b00 msgr2=0x7f5dd00b4fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.919+0000 7f5df2600700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5dd00b2b00 0x7f5dd00b4fc0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f5de4004800 tx=0x7f5de4026000 comp rx=0 tx=0).stop 2024-05-20T20:13:13.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.919+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5dec10e7d0 msgr2=0x7f5dec076930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:13.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.919+0000 7f5df2600700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5dec10e7d0 0x7f5dec076930 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc00c970 tx=0x7f5ddc00cc80 comp rx=0 tx=0).stop 2024-05-20T20:13:13.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.920+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 shutdown_connections 2024-05-20T20:13:13.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.920+0000 7f5df2600700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5dec10f190 0x7f5dec076f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.920+0000 7f5df2600700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5dec10abf0 0x7f5dec07c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.920+0000 7f5df2600700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5dd00b2b00 0x7f5dd00b4fc0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.920+0000 7f5df2600700 1 --2- 172.21.15.28:0/1966071278 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5dec10e7d0 0x7f5dec076930 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:13.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.920+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 >> 172.21.15.28:0/1966071278 conn(0x7f5dec06e6b0 msgr2=0x7f5dec10c8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:13.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.922+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 shutdown_connections 2024-05-20T20:13:13.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:13.922+0000 7f5df2600700 1 -- 172.21.15.28:0/1966071278 wait complete. 2024-05-20T20:13:14.111 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:13:15.009 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:13:15.166 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:15.199 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:15 smithi028 ceph-mon[31178]: pgmap v234: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 174 MiB/s wr, 988 op/s 2024-05-20T20:13:15.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:15 smithi154 ceph-mon[36314]: pgmap v234: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 174 MiB/s wr, 988 op/s 2024-05-20T20:13:15.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:15 smithi033 ceph-mon[36179]: pgmap v234: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 174 MiB/s wr, 988 op/s 2024-05-20T20:13:16.812 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:16 smithi028 ceph-mon[31178]: osdmap e85: 12 total, 12 up, 12 in 2024-05-20T20:13:16.812 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:16 smithi028 ceph-mon[31178]: pgmap v236: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 172 MiB/s wr, 952 op/s 2024-05-20T20:13:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:16 smithi154 ceph-mon[36314]: osdmap e85: 12 total, 12 up, 12 in 2024-05-20T20:13:16.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:16 smithi154 ceph-mon[36314]: pgmap v236: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 172 MiB/s wr, 952 op/s 2024-05-20T20:13:16.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:16 smithi033 ceph-mon[36179]: osdmap e85: 12 total, 12 up, 12 in 2024-05-20T20:13:16.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:16 smithi033 ceph-mon[36179]: pgmap v236: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 172 MiB/s wr, 952 op/s 2024-05-20T20:13:17.267 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:18.965 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:18 smithi028 ceph-mon[31178]: pgmap v237: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 70 MiB/s wr, 420 op/s 2024-05-20T20:13:19.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:18 smithi154 ceph-mon[36314]: pgmap v237: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 70 MiB/s wr, 420 op/s 2024-05-20T20:13:19.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:18 smithi033 ceph-mon[36179]: pgmap v237: 97 pgs: 97 active+clean; 45 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 70 MiB/s wr, 420 op/s 2024-05-20T20:13:20.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.867+0000 7f24e4000700 1 -- 172.21.15.28:0/2666664871 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc106140 msgr2=0x7f24dc112650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.867+0000 7f24e4000700 1 --2- 172.21.15.28:0/2666664871 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc106140 0x7f24dc112650 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f24d4009990 tx=0x7f24d4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.869+0000 7f24e4000700 1 -- 172.21.15.28:0/2666664871 shutdown_connections 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.869+0000 7f24e4000700 1 --2- 172.21.15.28:0/2666664871 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc106140 0x7f24dc112650 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.869+0000 7f24e4000700 1 --2- 172.21.15.28:0/2666664871 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24dc10b780 0x7f24dc10bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.869+0000 7f24e4000700 1 --2- 172.21.15.28:0/2666664871 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24dc105780 0x7f24dc105c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.869+0000 7f24e4000700 1 -- 172.21.15.28:0/2666664871 >> 172.21.15.28:0/2666664871 conn(0x7f24dc101270 msgr2=0x7f24dc103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.872+0000 7f24e4000700 1 -- 172.21.15.28:0/2666664871 shutdown_connections 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.872+0000 7f24e4000700 1 -- 172.21.15.28:0/2666664871 wait complete. 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.874+0000 7f24e4000700 1 Processor -- start 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.874+0000 7f24e4000700 1 -- start start 2024-05-20T20:13:20.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.875+0000 7f24e4000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24dc105780 0x7f24dc111020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e4000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24dc106140 0x7f24dc111560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e4000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc10b780 0x7f24dc1abe00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e4000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f24dc078b20 con 0x7f24dc106140 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e4000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f24dc0789a0 con 0x7f24dc10b780 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e4000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f24dc078ca0 con 0x7f24dc105780 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e1600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24dc105780 0x7f24dc111020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e1600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24dc105780 0x7f24dc111020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58046/0 (socket says 172.21.15.28:58046) 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e1600700 1 -- 172.21.15.28:0/3803871613 learned_addr learned my addr 172.21.15.28:0/3803871613 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.876+0000 7f24e2000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc10b780 0x7f24dc1abe00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.877+0000 7f24e0c00700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24dc106140 0x7f24dc111560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.877+0000 7f24e2000700 1 -- 172.21.15.28:0/3803871613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24dc105780 msgr2=0x7f24dc111020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.877+0000 7f24e2000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24dc105780 0x7f24dc111020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.877+0000 7f24e2000700 1 -- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24dc106140 msgr2=0x7f24dc111560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.877+0000 7f24e2000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24dc106140 0x7f24dc111560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.877+0000 7f24e2000700 1 -- 172.21.15.28:0/3803871613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f24dc1ac500 con 0x7f24dc10b780 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.877+0000 7f24e2000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc10b780 0x7f24dc1abe00 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7f24d4009990 tx=0x7f24d400b670 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.878+0000 7f24da000700 1 -- 172.21.15.28:0/3803871613 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f24d4027070 con 0x7f24dc10b780 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.878+0000 7f24da000700 1 -- 172.21.15.28:0/3803871613 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f24d4022470 con 0x7f24dc10b780 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.878+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f24dc1ac790 con 0x7f24dc10b780 2024-05-20T20:13:20.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.878+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f24dc1acbe0 con 0x7f24dc10b780 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.878+0000 7f24da000700 1 -- 172.21.15.28:0/3803871613 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f24d4017650 con 0x7f24dc10b780 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.880+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f24dc06a230 con 0x7f24dc10b780 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.884+0000 7f24da000700 1 -- 172.21.15.28:0/3803871613 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f24d402a070 con 0x7f24dc10b780 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.886+0000 7f24da000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f24c40b2b40 0x7f24c40b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.887+0000 7f24da000700 1 -- 172.21.15.28:0/3803871613 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f24d40a6db0 con 0x7f24dc10b780 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.887+0000 7f24e1600700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f24c40b2b40 0x7f24c40b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.889+0000 7f24e1600700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f24c40b2b40 0x7f24c40b5000 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f24d00077c0 tx=0x7f24d0007750 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:20.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:19.893+0000 7f24da000700 1 -- 172.21.15.28:0/3803871613 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f24d4013070 con 0x7f24dc10b780 2024-05-20T20:13:20.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.195+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f24dc02e560 con 0x7f24dc10b780 2024-05-20T20:13:20.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.196+0000 7f24da000700 1 -- 172.21.15.28:0/3803871613 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f24d40aaec0 con 0x7f24dc10b780 2024-05-20T20:13:20.196 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:13:20.196 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:13:20.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f24c40b2b40 msgr2=0x7f24c40b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f24c40b2b40 0x7f24c40b5000 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f24d00077c0 tx=0x7f24d0007750 comp rx=0 tx=0).stop 2024-05-20T20:13:20.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc10b780 msgr2=0x7f24dc1abe00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc10b780 0x7f24dc1abe00 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7f24d4009990 tx=0x7f24d400b670 comp rx=0 tx=0).stop 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 shutdown_connections 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f24c40b2b40 0x7f24c40b5000 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f24dc10b780 0x7f24dc1abe00 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f24dc105780 0x7f24dc111020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 --2- 172.21.15.28:0/3803871613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f24dc106140 0x7f24dc111560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.199+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 >> 172.21.15.28:0/3803871613 conn(0x7f24dc101270 msgr2=0x7f24dc1033e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.200+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 shutdown_connections 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:20.200+0000 7f24e4000700 1 -- 172.21.15.28:0/3803871613 wait complete. 2024-05-20T20:13:20.200 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:13:20.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 -- 172.21.15.28:0/2272939937 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628105860 msgr2=0x7f96281098a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 --2- 172.21.15.28:0/2272939937 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628105860 0x7f96281098a0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f9610009900 tx=0x7f961002a410 comp rx=0 tx=0).stop 2024-05-20T20:13:20.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 -- 172.21.15.28:0/2272939937 shutdown_connections 2024-05-20T20:13:20.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 --2- 172.21.15.28:0/2272939937 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628109f80 0x7f9628111b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 --2- 172.21.15.28:0/2272939937 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628105860 0x7f96281098a0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 --2- 172.21.15.28:0/2272939937 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9628104eb0 0x7f9628105290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 -- 172.21.15.28:0/2272939937 >> 172.21.15.28:0/2272939937 conn(0x7f9628100750 msgr2=0x7f9628102b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:20.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 -- 172.21.15.28:0/2272939937 shutdown_connections 2024-05-20T20:13:20.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.985+0000 7f962ee00640 1 -- 172.21.15.28:0/2272939937 wait complete. 2024-05-20T20:13:20.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.986+0000 7f962ee00640 1 Processor -- start 2024-05-20T20:13:20.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.986+0000 7f962ee00640 1 -- start start 2024-05-20T20:13:20.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f962ee00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628104eb0 0x7f962819b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f962ee00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628109f80 0x7f962819b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f962ee00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f962819fc90 0x7f96281a0140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f962ee00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f9628114f30 con 0x7f9628104eb0 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f962ee00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9628114db0 con 0x7f962819fc90 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f962ee00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f96281150b0 con 0x7f9628109f80 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628104eb0 0x7f962819b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628104eb0 0x7f962819b3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53646/0 (socket says 172.21.15.28:53646) 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627e00640 1 -- 172.21.15.28:0/3969342235 learned_addr learned my addr 172.21.15.28:0/3969342235 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627400640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628109f80 0x7f962819b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f962ce00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f962819fc90 0x7f96281a0140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627e00640 1 -- 172.21.15.28:0/3969342235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628109f80 msgr2=0x7f962819b910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627e00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628109f80 0x7f962819b910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.990 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627e00640 1 -- 172.21.15.28:0/3969342235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f962819fc90 msgr2=0x7f96281a0140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.987+0000 7f9627e00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f962819fc90 0x7f96281a0140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f9627e00640 1 -- 172.21.15.28:0/3969342235 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f96281a0780 con 0x7f9628104eb0 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f9627400640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628109f80 0x7f962819b910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f9627e00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628104eb0 0x7f962819b3d0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f961800db80 tx=0x7f961800b630 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f9624c00640 1 -- 172.21.15.28:0/3969342235 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9618017070 con 0x7f9628104eb0 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f96281a0a70 con 0x7f9628104eb0 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f9624c00640 1 -- 172.21.15.28:0/3969342235 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9618003b70 con 0x7f9628104eb0 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f9624c00640 1 -- 172.21.15.28:0/3969342235 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f96180106c0 con 0x7f9628104eb0 2024-05-20T20:13:20.991 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.988+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f96281a81d0 con 0x7f9628104eb0 2024-05-20T20:13:20.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.992+0000 7f9631c0b740 1 Processor -- start 2024-05-20T20:13:20.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9631c0b740 1 -- start start 2024-05-20T20:13:20.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c538572c90 0x55c538573070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c5385735b0 0x55c53857d930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c53857de70 0x55c538580260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9631c0b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c5384e3b90 con 0x55c5385735b0 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9631c0b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c5384d3730 con 0x55c538572c90 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9631c0b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c5384d7770 con 0x55c53857de70 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.993+0000 7f9624c00640 1 -- 172.21.15.28:0/3969342235 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f9618004020 con 0x7f9628104eb0 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c538572c90 0x55c538573070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c538572c90 0x55c538573070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:33632/0 (socket says 172.21.15.28:33632) 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 -- 172.21.15.28:0/3111332905 learned_addr learned my addr 172.21.15.28:0/3111332905 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f9627400640 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c53857de70 0x55c538580260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f9627e00640 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c5385735b0 0x55c53857d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.995 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 -- 172.21.15.28:0/3111332905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c53857de70 msgr2=0x55c538580260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:20.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c53857de70 0x55c538580260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 -- 172.21.15.28:0/3111332905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c5385735b0 msgr2=0x55c53857d930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:20.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c5385735b0 0x55c53857d930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.996 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 -- 172.21.15.28:0/3111332905 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c53846c8b0 con 0x55c538572c90 2024-05-20T20:13:20.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.994+0000 7f962ce00640 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c538572c90 0x55c538573070 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f9620009c20 tx=0x7f9620011810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c111b4195e4a30c1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:20.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.995+0000 7f961d600640 1 -- 172.21.15.28:0/3111332905 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f962001e070 con 0x55c538572c90 2024-05-20T20:13:20.997 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.995+0000 7f961d600640 1 -- 172.21.15.28:0/3111332905 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9620019ae0 con 0x55c538572c90 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.995+0000 7f961d600640 1 -- 172.21.15.28:0/3111332905 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f962000e8b0 con 0x55c538572c90 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.995+0000 7f9631c0b740 1 -- 172.21.15.28:0/3111332905 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c538572c90 msgr2=0x55c538573070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.995+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c538572c90 0x55c538573070 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f9620009c20 tx=0x7f9620011810 comp rx=0 tx=0).stop 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.995+0000 7f9624c00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f960c0b2a10 0x7f960c0b4ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 -- 172.21.15.28:0/3111332905 shutdown_connections 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c53857de70 0x55c538580260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c5385735b0 0x55c53857d930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3111332905 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c538572c90 0x55c538573070 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 -- 172.21.15.28:0/3111332905 >> 172.21.15.28:0/3111332905 conn(0x55c53856f1a0 msgr2=0x55c53856f580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9627e00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f960c0b2a10 0x7f960c0b4ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 -- 172.21.15.28:0/3111332905 shutdown_connections 2024-05-20T20:13:20.998 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9624c00640 1 -- 172.21.15.28:0/3969342235 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f96180e2620 con 0x7f9628104eb0 2024-05-20T20:13:20.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 -- 172.21.15.28:0/3111332905 wait complete. 2024-05-20T20:13:20.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.996+0000 7f9631c0b740 1 -- start start 2024-05-20T20:13:20.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9627e00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f960c0b2a10 0x7f960c0b4ed0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x55c538574230 tx=0x7f9618000f40 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:20.999 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 1 Processor -- start 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c53857d860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857abc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c53857b240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55c5381b9790 con 0x55c53857de70 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55c53847bbc0 con 0x55c5385735b0 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55c538580aa0 con 0x55c538572c90 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9631c0b740 10 client.0 _ll_register_callbacks cb 0x55c538459400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f9627400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.997+0000 7f962ce00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c53857d860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857abc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:33636/0 (socket says 172.21.15.28:33636) 2024-05-20T20:13:21.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 -- 172.21.15.28:0/3806635092 learned_addr learned my addr 172.21.15.28:0/3806635092 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f962ce00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c53857d860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58092/0 (socket says 172.21.15.28:58092) 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f962ce00640 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c53857b240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 -- 172.21.15.28:0/3806635092 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 msgr2=0x55c53857d860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c53857d860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 -- 172.21.15.28:0/3806635092 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 msgr2=0x55c53857b240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c53857b240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 -- 172.21.15.28:0/3806635092 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c5381e8210 con 0x55c5385735b0 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9627400640 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857abc0 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f9610009900 tx=0x7f9610005270 comp rx=0 tx=0).ready entity=mon.2 client_cookie=64d81f94bea3ce1c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.998+0000 7f9616000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9616000640 1 -- 172.21.15.28:0/3806635092 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9610035040 con 0x55c5385735b0 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9616000640 1 -- 172.21.15.28:0/3806635092 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f9610004970 con 0x55c5385735b0 2024-05-20T20:13:21.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9620019ae0 con 0x55c5385735b0 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9616000640 1 -- 172.21.15.28:0/3806635092 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f9610004b70 con 0x55c5385735b0 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9631c0b740 10 client.27065 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c5382b4620 con 0x55c5385735b0 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9616000640 1 -- 172.21.15.28:0/3806635092 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9620019ae0 con 0x55c5385735b0 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:20.999+0000 7f9616000640 1 -- 172.21.15.28:0/3806635092 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9610044020 con 0x55c5385735b0 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.000+0000 7f9631c0b740 10 client.27065 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.000+0000 7f9631c0b740 10 client.27065 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.000+0000 7f9631c0b740 20 client.27065 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.000+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c53858d8f0 0x55c53858fce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.000+0000 7f9631c0b740 4 client.27065 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.000+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55c5384e5ae0 con 0x55c53858d8f0 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.000+0000 7f962ce00640 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c53858d8f0 0x55c53858fce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.001+0000 7f962ce00640 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c53858d8f0 0x55c53858fce0 crc :-1 s=READY pgs=175 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=975459bfe8519845 server_cookie=24cc12271047dd6 in_seq=0 out_seq=0 2024-05-20T20:13:21.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.001+0000 7f9616000640 10 client.27065 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.001+0000 7f9616000640 1 -- 172.21.15.28:0/3806635092 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55c5384e5ae0 con 0x55c53858d8f0 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.001+0000 7f9616000640 10 client.27065 handle_command_reply: tid=0 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 client.27065 shutdown 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 msgr2=0x55c53857abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857abc0 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f9610009900 tx=0x7f9610005270 comp rx=0 tx=0).stop 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 shutdown_connections 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c53858d8f0 0x55c53858fce0 crc :-1 s=READY pgs=175 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857abc0 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c53857b240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9616000640 0 client.27065 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 --2- 172.21.15.28:0/3806635092 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c53857d860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 >> 172.21.15.28:0/3806635092 conn(0x55c53856f1a0 msgr2=0x55c538574f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 shutdown_connections 2024-05-20T20:13:21.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 1 -- 172.21.15.28:0/3806635092 wait complete. 2024-05-20T20:13:21.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.002+0000 7f9631c0b740 20 client.27065 trim_cache size 0 max 16384 2024-05-20T20:13:21.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.019+0000 7f9631c0b740 1 Processor -- start 2024-05-20T20:13:21.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.019+0000 7f9631c0b740 1 -- start start 2024-05-20T20:13:21.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c538589b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857cb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c538595580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f9631c0b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f96281b08c0 con 0x55c53857de70 2024-05-20T20:13:21.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f9631c0b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f96281b2400 con 0x55c5385735b0 2024-05-20T20:13:21.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f9631c0b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f96281b0f90 con 0x55c538572c90 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f962ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c538595580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f962ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c538595580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53684/0 (socket says 172.21.15.28:53684) 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f962ce00640 1 -- 172.21.15.28:0/2415355126 learned_addr learned my addr 172.21.15.28:0/2415355126 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f9627400640 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857cb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.020+0000 7f962ce00640 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c538589b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f962ce00640 1 -- 172.21.15.28:0/2415355126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 msgr2=0x55c538589b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f962ce00640 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c538589b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f962ce00640 1 -- 172.21.15.28:0/2415355126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 msgr2=0x55c53857cb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f962ce00640 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857cb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f962ce00640 1 -- 172.21.15.28:0/2415355126 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55c5385805d0 con 0x55c53857de70 2024-05-20T20:13:21.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f9627400640 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:21.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f962ce00640 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c538595580 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f962001b000 tx=0x7f9620005f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bb7615839d2791b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f961cc00640 1 -- 172.21.15.28:0/2415355126 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f962001e040 con 0x55c53857de70 2024-05-20T20:13:21.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f961cc00640 1 -- 172.21.15.28:0/2415355126 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f962001f040 con 0x55c53857de70 2024-05-20T20:13:21.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.021+0000 7f961cc00640 1 -- 172.21.15.28:0/2415355126 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f962000e580 con 0x55c53857de70 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 -- 172.21.15.28:0/2415355126 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 msgr2=0x55c538595580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c538595580 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f962001b000 tx=0x7f9620005f90 comp rx=0 tx=0).stop 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 -- 172.21.15.28:0/2415355126 shutdown_connections 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c5385735b0 0x55c53857cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c53857de70 0x55c538595580 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 --2- 172.21.15.28:0/2415355126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c538572c90 0x55c538589b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 -- 172.21.15.28:0/2415355126 >> 172.21.15.28:0/2415355126 conn(0x55c53856f1a0 msgr2=0x55c53857c3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 -- 172.21.15.28:0/2415355126 shutdown_connections 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 -- 172.21.15.28:0/2415355126 wait complete. 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.022+0000 7f9631c0b740 1 -- start start 2024-05-20T20:13:21.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 1 Processor -- start 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c538572c90 0x55c538580cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c5385735b0 0x55c5385811f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c53857de70 0x55c538581730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f96281b1d30 con 0x55c538572c90 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f96281b1660 con 0x55c53857de70 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f96281ae8c0 con 0x55c5385735b0 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f9631c0b740 10 client.0 _ll_register_callbacks cb 0x7f9610005b50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f962ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c538572c90 0x55c538580cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f962ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c538572c90 0x55c538580cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53698/0 (socket says 172.21.15.28:53698) 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.023+0000 7f962ce00640 1 -- 172.21.15.28:0/1375682014 learned_addr learned my addr 172.21.15.28:0/1375682014 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f9627400640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c5385735b0 0x55c5385811f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f962ce00640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c53857de70 0x55c538581730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f962ce00640 1 -- 172.21.15.28:0/1375682014 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c5385735b0 msgr2=0x55c5385811f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f962ce00640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c5385735b0 0x55c5385811f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f962ce00640 1 -- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c53857de70 msgr2=0x55c538581730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f962ce00640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c53857de70 0x55c538581730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f962ce00640 1 -- 172.21.15.28:0/1375682014 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55c5385f6660 con 0x55c538572c90 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f9627400640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c5385735b0 0x55c5385811f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f962ce00640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c538572c90 0x55c538580cb0 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f9620009f70 tx=0x7f962001ae10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=697d5eb1a5b3e842 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f9616000640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:13:21.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f9616000640 1 -- 172.21.15.28:0/1375682014 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f962001e070 con 0x55c538572c90 2024-05-20T20:13:21.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f9616000640 1 -- 172.21.15.28:0/1375682014 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f9620005280 con 0x55c538572c90 2024-05-20T20:13:21.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.024+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f962001f040 con 0x55c538572c90 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9616000640 1 -- 172.21.15.28:0/1375682014 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f9620043050 con 0x55c538572c90 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 10 client.16704 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55c5385f7970 con 0x55c538572c90 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9616000640 1 -- 172.21.15.28:0/1375682014 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f962001f040 con 0x55c538572c90 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9616000640 1 -- 172.21.15.28:0/1375682014 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f962003a440 con 0x55c538572c90 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 10 client.16704 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 10 client.16704 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 20 client.16704 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c5385f7df0 0x55c5385fa1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 4 client.16704 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.025+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55c5384df2c0 con 0x55c5385f7df0 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.026+0000 7f9627400640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c5385f7df0 0x55c5385fa1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.026+0000 7f9627400640 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c5385f7df0 0x55c5385fa1e0 crc :-1 s=READY pgs=176 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cd32ca404e30db1e server_cookie=c63941e9194aa42b in_seq=0 out_seq=0 2024-05-20T20:13:21.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.026+0000 7f9616000640 10 client.16704 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:21.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.047+0000 7f9616000640 1 -- 172.21.15.28:0/1375682014 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55c5384df2c0 con 0x55c5385f7df0 2024-05-20T20:13:21.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.047+0000 7f9616000640 10 client.16704 handle_command_reply: tid=0 2024-05-20T20:13:21.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 client.16704 shutdown 2024-05-20T20:13:21.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c538572c90 msgr2=0x55c538580cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c538572c90 0x55c538580cb0 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f9620009f70 tx=0x7f962001ae10 comp rx=0 tx=0).stop 2024-05-20T20:13:21.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 shutdown_connections 2024-05-20T20:13:21.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55c5385f7df0 0x55c5385fa1e0 crc :-1 s=READY pgs=176 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55c53857de70 0x55c538581730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55c5385735b0 0x55c5385811f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 --2- 172.21.15.28:0/1375682014 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55c538572c90 0x55c538580cb0 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 >> 172.21.15.28:0/1375682014 conn(0x55c53856f1a0 msgr2=0x55c53857cc80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9616000640 0 client.16704 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 shutdown_connections 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 1 -- 172.21.15.28:0/1375682014 wait complete. 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.048+0000 7f9631c0b740 20 client.16704 trim_cache size 0 max 16384 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:13:21.050 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:13:21.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:13:21.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f960c0b2a10 msgr2=0x7f960c0b4ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f960c0b2a10 0x7f960c0b4ed0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x55c538574230 tx=0x7f9618000f40 comp rx=0 tx=0).stop 2024-05-20T20:13:21.051 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628104eb0 msgr2=0x7f962819b3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628104eb0 0x7f962819b3d0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f961800db80 tx=0x7f961800b630 comp rx=0 tx=0).stop 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 shutdown_connections 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f962819fc90 0x7f96281a0140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f960c0b2a10 0x7f960c0b4ed0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9628109f80 0x7f962819b910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 --2- 172.21.15.28:0/3969342235 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9628104eb0 0x7f962819b3d0 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.049+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 >> 172.21.15.28:0/3969342235 conn(0x7f9628100750 msgr2=0x7f9628102a10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.050+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 shutdown_connections 2024-05-20T20:13:21.052 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:21.050+0000 7f962ee00640 1 -- 172.21.15.28:0/3969342235 wait complete. 2024-05-20T20:13:21.053 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:20 smithi028 ceph-mon[31178]: pgmap v238: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 215 MiB/s wr, 1.55k op/s 2024-05-20T20:13:21.053 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:20 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3803871613' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:21.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 -- 172.21.15.28:0/61030124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff23806bbe0 msgr2=0x7ff2381103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 --2- 172.21.15.28:0/61030124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff23806bbe0 0x7ff2381103d0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7ff23000b0a0 tx=0x7ff23000b3b0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 -- 172.21.15.28:0/61030124 shutdown_connections 2024-05-20T20:13:21.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 --2- 172.21.15.28:0/61030124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff23806bbe0 0x7ff2381103d0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 --2- 172.21.15.28:0/61030124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff2381095b0 0x7ff238109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 --2- 172.21.15.28:0/61030124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff23806b220 0x7ff23806b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 -- 172.21.15.28:0/61030124 >> 172.21.15.28:0/61030124 conn(0x7ff2380793f0 msgr2=0x7ff238079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 -- 172.21.15.28:0/61030124 shutdown_connections 2024-05-20T20:13:21.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.187+0000 7ff23e000700 1 -- 172.21.15.28:0/61030124 wait complete. 2024-05-20T20:13:21.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.188+0000 7ff23e000700 1 Processor -- start 2024-05-20T20:13:21.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff23e000700 1 -- start start 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff23e000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff23806b220 0x7ff23819be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff23e000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff23806bbe0 0x7ff23819c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff23e000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff2381095b0 0x7ff2381a06f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff23e000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff238113730 con 0x7ff2381095b0 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff23e000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381135b0 con 0x7ff23806bbe0 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff23e000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381138b0 con 0x7ff23806b220 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff237e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff2381095b0 0x7ff2381a06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff237400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff23806b220 0x7ff23819be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff237e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff2381095b0 0x7ff2381a06f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53720/0 (socket says 172.21.15.28:53720) 2024-05-20T20:13:21.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff237e00700 1 -- 172.21.15.28:0/1472644732 learned_addr learned my addr 172.21.15.28:0/1472644732 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.189+0000 7ff236a00700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff23806bbe0 0x7ff23819c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff237e00700 1 -- 172.21.15.28:0/1472644732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff23806b220 msgr2=0x7ff23819be20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff237e00700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff23806b220 0x7ff23819be20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff237e00700 1 -- 172.21.15.28:0/1472644732 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff23806bbe0 msgr2=0x7ff23819c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff237e00700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff23806bbe0 0x7ff23819c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff237e00700 1 -- 172.21.15.28:0/1472644732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff2381a0e30 con 0x7ff2381095b0 2024-05-20T20:13:21.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff237400700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff23806b220 0x7ff23819be20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:21.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff237e00700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff2381095b0 0x7ff2381a06f0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7ff23000b070 tx=0x7ff230009c90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.190+0000 7ff22fe00700 1 -- 172.21.15.28:0/1472644732 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff230021070 con 0x7ff2381095b0 2024-05-20T20:13:21.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.191+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff2381a10c0 con 0x7ff2381095b0 2024-05-20T20:13:21.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.191+0000 7ff22fe00700 1 -- 172.21.15.28:0/1472644732 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff230004e40 con 0x7ff2381095b0 2024-05-20T20:13:21.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.191+0000 7ff22fe00700 1 -- 172.21.15.28:0/1472644732 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff230004630 con 0x7ff2381095b0 2024-05-20T20:13:21.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.191+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff2381a89a0 con 0x7ff2381095b0 2024-05-20T20:13:21.195 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.195+0000 7ff22fe00700 1 -- 172.21.15.28:0/1472644732 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ff230007980 con 0x7ff2381095b0 2024-05-20T20:13:21.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.195+0000 7ff24897cb80 1 Processor -- start 2024-05-20T20:13:21.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff24897cb80 1 -- start start 2024-05-20T20:13:21.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b858eb0 0x55d88b859290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b853260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b853880 0x55d88b853ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff24897cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d88b7b1e40 con 0x55d88b858eb0 2024-05-20T20:13:21.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff24897cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d88b7a19e0 con 0x55d88b8597d0 2024-05-20T20:13:21.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff24897cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d88b7a5a20 con 0x55d88b853880 2024-05-20T20:13:21.197 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff22fe00700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff21c0b29f0 0x7ff21c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff237400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b853260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff237400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b853260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:33690/0 (socket says 172.21.15.28:33690) 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff237400700 1 -- 172.21.15.28:0/308822845 learned_addr learned my addr 172.21.15.28:0/308822845 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.196+0000 7ff22fe00700 1 -- 172.21.15.28:0/1472644732 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7ff23000f020 con 0x7ff2381095b0 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff236a00700 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b858eb0 0x55d88b859290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff237e00700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff21c0b29f0 0x7ff21c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff236a00700 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b853880 0x55d88b853ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff237400700 1 -- 172.21.15.28:0/308822845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b853880 msgr2=0x55d88b853ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff237400700 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b853880 0x55d88b853ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff237400700 1 -- 172.21.15.28:0/308822845 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b858eb0 msgr2=0x55d88b859290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.199 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff237400700 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b858eb0 0x55d88b859290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff237400700 1 -- 172.21.15.28:0/308822845 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d88b76a420 con 0x55d88b8597d0 2024-05-20T20:13:21.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff236a00700 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b853880 0x55d88b853ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:21.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.197+0000 7ff237400700 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b853260 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7ff2280115c0 tx=0x7ff2280118d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=61640e57da4497ef server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.198+0000 7ff237e00700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff21c0b29f0 0x7ff21c0b4eb0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff23000b780 tx=0x7ff230009b80 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.198+0000 7ff22cc00700 1 -- 172.21.15.28:0/308822845 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff22801e070 con 0x55d88b8597d0 2024-05-20T20:13:21.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.198+0000 7ff22cc00700 1 -- 172.21.15.28:0/308822845 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff2280198c0 con 0x55d88b8597d0 2024-05-20T20:13:21.200 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.198+0000 7ff22cc00700 1 -- 172.21.15.28:0/308822845 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff22800e8f0 con 0x55d88b8597d0 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 -- 172.21.15.28:0/308822845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 msgr2=0x55d88b853260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b853260 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7ff2280115c0 tx=0x7ff2280118d0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 -- 172.21.15.28:0/308822845 shutdown_connections 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b853880 0x55d88b853ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b858eb0 0x55d88b859290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 --2- 172.21.15.28:0/308822845 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b853260 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 -- 172.21.15.28:0/308822845 >> 172.21.15.28:0/308822845 conn(0x55d88b781be0 msgr2=0x55d88b84e010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 -- 172.21.15.28:0/308822845 shutdown_connections 2024-05-20T20:13:21.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 -- 172.21.15.28:0/308822845 wait complete. 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.199+0000 7ff24897cb80 1 -- start start 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 1 Processor -- start 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b853880 0x55d88b85f470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85c810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b8597d0 0x55d88b85cd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d88b6a7fa0 con 0x55d88b8597d0 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d88b742760 con 0x55d88b853880 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d88b860240 con 0x55d88b858eb0 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.200+0000 7ff24897cb80 10 client.0 _ll_register_callbacks cb 0x55d88b74bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff236a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b8597d0 0x55d88b85cd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85c810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58132/0 (socket says 172.21.15.28:58132) 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 -- 172.21.15.28:0/1640204789 learned_addr learned my addr 172.21.15.28:0/1640204789 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff236a00700 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b853880 0x55d88b85f470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 -- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b853880 msgr2=0x55d88b85f470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b853880 0x55d88b85f470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 -- 172.21.15.28:0/1640204789 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b8597d0 msgr2=0x55d88b85cd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b8597d0 0x55d88b85cd50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 -- 172.21.15.28:0/1640204789 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d88b6f18d0 con 0x55d88b858eb0 2024-05-20T20:13:21.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff237400700 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85c810 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7ff228005920 tx=0x7ff228019ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4dd2f49872261402 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.201+0000 7ff221600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.202+0000 7ff221600700 1 -- 172.21.15.28:0/1640204789 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff22801e040 con 0x55d88b858eb0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.202+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff2280198c0 con 0x55d88b858eb0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.202+0000 7ff221600700 1 -- 172.21.15.28:0/1640204789 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7ff228005070 con 0x55d88b858eb0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.202+0000 7ff221600700 1 -- 172.21.15.28:0/1640204789 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff22802f050 con 0x55d88b858eb0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.202+0000 7ff24897cb80 10 client.26974 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.202+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d88b772060 con 0x55d88b858eb0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.202+0000 7ff221600700 1 -- 172.21.15.28:0/1640204789 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff2280198c0 con 0x55d88b858eb0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff221600700 1 -- 172.21.15.28:0/1640204789 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff22802a030 con 0x55d88b858eb0 2024-05-20T20:13:21.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff24897cb80 10 client.26974 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff24897cb80 10 client.26974 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff24897cb80 20 client.26974 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b86cb80 0x55d88b86ef70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff24897cb80 4 client.26974 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d88b7b3d90 con 0x55d88b86cb80 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.203+0000 7ff236a00700 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b86cb80 0x55d88b86ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.204+0000 7ff236a00700 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b86cb80 0x55d88b86ef70 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=d2054aac6ac14348 server_cookie=18d4fb112a0ae2af in_seq=0 out_seq=0 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.204+0000 7ff221600700 10 client.26974 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff221600700 1 -- 172.21.15.28:0/1640204789 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55d88b7b3d90 con 0x55d88b86cb80 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff221600700 10 client.26974 handle_command_reply: tid=0 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 client.26974 shutdown 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 msgr2=0x55d88b85c810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85c810 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7ff228005920 tx=0x7ff228019ca0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 shutdown_connections 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b86cb80 0x55d88b86ef70 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b8597d0 0x55d88b85cd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b853880 0x55d88b85f470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1640204789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85c810 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff221600700 0 client.26974 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 >> 172.21.15.28:0/1640204789 conn(0x55d88b781be0 msgr2=0x55d88b852250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 shutdown_connections 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 1 -- 172.21.15.28:0/1640204789 wait complete. 2024-05-20T20:13:21.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.205+0000 7ff24897cb80 20 client.26974 trim_cache size 0 max 16384 2024-05-20T20:13:21.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.222+0000 7ff24897cb80 1 Processor -- start 2024-05-20T20:13:21.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.222+0000 7ff24897cb80 1 -- start start 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b853260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85d3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b84e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff24897cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381b0000 con 0x55d88b853880 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff24897cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381b1b40 con 0x55d88b8597d0 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff24897cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381b06d0 con 0x55d88b858eb0 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff236a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b853260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff237400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85d3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff236a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b853260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53758/0 (socket says 172.21.15.28:53758) 2024-05-20T20:13:21.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff236a00700 1 -- 172.21.15.28:0/3322094600 learned_addr learned my addr 172.21.15.28:0/3322094600 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.223+0000 7ff236a00700 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b84e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff237400700 1 -- 172.21.15.28:0/3322094600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 msgr2=0x55d88b84e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff237400700 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b84e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff237400700 1 -- 172.21.15.28:0/3322094600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 msgr2=0x55d88b853260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff237400700 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b853260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff237400700 1 -- 172.21.15.28:0/3322094600 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d88b85f0b0 con 0x55d88b858eb0 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff237400700 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85d3b0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7ff228004eb0 tx=0x7ff228001e50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=817dd4c3524199bc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff223e00700 1 -- 172.21.15.28:0/3322094600 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff228015070 con 0x55d88b858eb0 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff223e00700 1 -- 172.21.15.28:0/3322094600 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff22800e5d0 con 0x55d88b858eb0 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.224+0000 7ff223e00700 1 -- 172.21.15.28:0/3322094600 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff22801e040 con 0x55d88b858eb0 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 -- 172.21.15.28:0/3322094600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 msgr2=0x55d88b85d3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85d3b0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7ff228004eb0 tx=0x7ff228001e50 comp rx=0 tx=0).stop 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 -- 172.21.15.28:0/3322094600 shutdown_connections 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b8597d0 0x55d88b84e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b853260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 --2- 172.21.15.28:0/3322094600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b858eb0 0x55d88b85d3b0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 -- 172.21.15.28:0/3322094600 >> 172.21.15.28:0/3322094600 conn(0x55d88b781be0 msgr2=0x55d88b84d9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 -- 172.21.15.28:0/3322094600 shutdown_connections 2024-05-20T20:13:21.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 -- 172.21.15.28:0/3322094600 wait complete. 2024-05-20T20:13:21.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.225+0000 7ff24897cb80 1 -- start start 2024-05-20T20:13:21.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 1 Processor -- start 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b8659b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b858eb0 0x55d88b873cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b8597d0 0x55d88b8741f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381b1470 con 0x55d88b853880 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381b0da0 con 0x55d88b858eb0 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff2381adfc0 con 0x55d88b8597d0 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.226+0000 7ff24897cb80 10 client.0 _ll_register_callbacks cb 0x55d88b74bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b8659b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff237400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b858eb0 0x55d88b873cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b8659b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53760/0 (socket says 172.21.15.28:53760) 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 -- 172.21.15.28:0/1230419530 learned_addr learned my addr 172.21.15.28:0/1230419530 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b8597d0 0x55d88b8741f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 -- 172.21.15.28:0/1230419530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b8597d0 msgr2=0x55d88b8741f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b8597d0 0x55d88b8741f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 -- 172.21.15.28:0/1230419530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b858eb0 msgr2=0x55d88b873cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b858eb0 0x55d88b873cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 -- 172.21.15.28:0/1230419530 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d88b8747b0 con 0x55d88b853880 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff236a00700 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b8659b0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x55d88b86f650 tx=0x7ff224022040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=464034ba298bba6c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.227+0000 7ff221600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff221600700 1 -- 172.21.15.28:0/1230419530 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff22401c040 con 0x55d88b853880 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff22800e5d0 con 0x55d88b853880 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff221600700 1 -- 172.21.15.28:0/1230419530 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7ff224004110 con 0x55d88b853880 2024-05-20T20:13:21.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff221600700 1 -- 172.21.15.28:0/1230419530 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff224031050 con 0x55d88b853880 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff24897cb80 10 client.16722 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d88b8d73b0 con 0x55d88b853880 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff221600700 1 -- 172.21.15.28:0/1230419530 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff22800e5d0 con 0x55d88b853880 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.228+0000 7ff221600700 1 -- 172.21.15.28:0/1230419530 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff224020300 con 0x55d88b853880 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.229+0000 7ff24897cb80 10 client.16722 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.229+0000 7ff24897cb80 10 client.16722 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.229+0000 7ff24897cb80 20 client.16722 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.229+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b8d7a40 0x55d88b8d9e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.229+0000 7ff24897cb80 4 client.16722 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.229+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55d88b7ad570 con 0x55d88b8d7a40 2024-05-20T20:13:21.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.229+0000 7ff237400700 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b8d7a40 0x55d88b8d9e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:21.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff237400700 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b8d7a40 0x55d88b8d9e60 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e080b9d344506869 server_cookie=b1da48919c3e322e in_seq=0 out_seq=0 2024-05-20T20:13:21.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff221600700 10 client.16722 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:13:21.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff221600700 1 -- 172.21.15.28:0/1230419530 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55d88b7ad570 con 0x55d88b8d7a40 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff221600700 10 client.16722 handle_command_reply: tid=0 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 client.16722 shutdown 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 msgr2=0x55d88b8659b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b8659b0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x55d88b86f650 tx=0x7ff224022040 comp rx=0 tx=0).stop 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 shutdown_connections 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55d88b8d7a40 0x55d88b8d9e60 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d88b8597d0 0x55d88b8741f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d88b853880 0x55d88b8659b0 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 --2- 172.21.15.28:0/1230419530 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d88b858eb0 0x55d88b873cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff221600700 0 client.16722 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 >> 172.21.15.28:0/1230419530 conn(0x55d88b781be0 msgr2=0x55d88b859bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 shutdown_connections 2024-05-20T20:13:21.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.230+0000 7ff24897cb80 1 -- 172.21.15.28:0/1230419530 wait complete. 2024-05-20T20:13:21.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.231+0000 7ff24897cb80 20 client.16722 trim_cache size 0 max 16384 2024-05-20T20:13:21.235 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:13:21.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.231+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff21c0b29f0 msgr2=0x7ff21c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.231+0000 7ff23e000700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff21c0b29f0 0x7ff21c0b4eb0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff23000b780 tx=0x7ff230009b80 comp rx=0 tx=0).stop 2024-05-20T20:13:21.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.231+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff2381095b0 msgr2=0x7ff2381a06f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.231+0000 7ff23e000700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff2381095b0 0x7ff2381a06f0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7ff23000b070 tx=0x7ff230009c90 comp rx=0 tx=0).stop 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 shutdown_connections 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff2381095b0 0x7ff2381a06f0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff23806b220 0x7ff23819be20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff21c0b29f0 0x7ff21c0b4eb0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 --2- 172.21.15.28:0/1472644732 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff23806bbe0 0x7ff23819c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 >> 172.21.15.28:0/1472644732 conn(0x7ff2380793f0 msgr2=0x7ff23810e720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 shutdown_connections 2024-05-20T20:13:21.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:21.232+0000 7ff23e000700 1 -- 172.21.15.28:0/1472644732 wait complete. 2024-05-20T20:13:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:20 smithi154 ceph-mon[36314]: pgmap v238: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 215 MiB/s wr, 1.55k op/s 2024-05-20T20:13:21.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:20 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3803871613' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:21.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:20 smithi033 ceph-mon[36179]: pgmap v238: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 215 MiB/s wr, 1.55k op/s 2024-05-20T20:13:21.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:20 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3803871613' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:21.635 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id 4491be01 --release --await' 2024-05-20T20:13:21.635 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id 4491be01 --release --await 2024-05-20T20:13:22.423 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:13:22.490 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:23.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:22 smithi028 ceph-mon[31178]: pgmap v239: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 225 MiB/s wr, 1.69k op/s 2024-05-20T20:13:23.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:22 smithi154 ceph-mon[36314]: pgmap v239: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 225 MiB/s wr, 1.69k op/s 2024-05-20T20:13:23.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:22 smithi033 ceph-mon[36179]: pgmap v239: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 7.7 KiB/s rd, 225 MiB/s wr, 1.69k op/s 2024-05-20T20:13:25.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:24 smithi028 ceph-mon[31178]: pgmap v240: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 235 MiB/s wr, 1.98k op/s 2024-05-20T20:13:25.350 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:25.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:24 smithi154 ceph-mon[36314]: pgmap v240: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 235 MiB/s wr, 1.98k op/s 2024-05-20T20:13:25.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:24 smithi033 ceph-mon[36179]: pgmap v240: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 235 MiB/s wr, 1.98k op/s 2024-05-20T20:13:26.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 -- 172.21.15.28:0/1668554567 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f978810ffa0 msgr2=0x7f9788112390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 --2- 172.21.15.28:0/1668554567 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f978810ffa0 0x7f9788112390 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f978000b0a0 tx=0x7f978000b3b0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 -- 172.21.15.28:0/1668554567 shutdown_connections 2024-05-20T20:13:26.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 --2- 172.21.15.28:0/1668554567 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f978810ffa0 0x7f9788112390 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.374 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 --2- 172.21.15.28:0/1668554567 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9788075770 0x7f9788075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 --2- 172.21.15.28:0/1668554567 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788076090 0x7f978810fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 -- 172.21.15.28:0/1668554567 >> 172.21.15.28:0/1668554567 conn(0x7f978807a6a0 msgr2=0x7f978807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:26.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 -- 172.21.15.28:0/1668554567 shutdown_connections 2024-05-20T20:13:26.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.373+0000 7f978d400700 1 -- 172.21.15.28:0/1668554567 wait complete. 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.374+0000 7f978d400700 1 Processor -- start 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.374+0000 7f978d400700 1 -- start start 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.374+0000 7f978d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788075770 0x7f978819bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.374+0000 7f978d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9788076090 0x7f978819c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f978d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f978810ffa0 0x7f97881a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f978d400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f97881159a0 con 0x7f9788076090 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f978d400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f9788115820 con 0x7f9788075770 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f978d400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f9788115b20 con 0x7f978810ffa0 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788075770 0x7f978819bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788075770 0x7f978819bdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:33738/0 (socket says 172.21.15.28:33738) 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9787200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f978810ffa0 0x7f97881a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 -- 172.21.15.28:0/1903466397 learned_addr learned my addr 172.21.15.28:0/1903466397 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:26.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f977fe00700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9788076090 0x7f978819c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 -- 172.21.15.28:0/1903466397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f978810ffa0 msgr2=0x7f97881a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f978810ffa0 0x7f97881a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 -- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9788076090 msgr2=0x7f978819c310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9788076090 0x7f978819c310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.375+0000 7f9786800700 1 -- 172.21.15.28:0/1903466397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f97881a0de0 con 0x7f9788075770 2024-05-20T20:13:26.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.376+0000 7f9786800700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788075770 0x7f978819bdd0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f977800e9c0 tx=0x7f977800ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:26.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.376+0000 7f977f400700 1 -- 172.21.15.28:0/1903466397 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f977800cbd0 con 0x7f9788075770 2024-05-20T20:13:26.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.376+0000 7f977f400700 1 -- 172.21.15.28:0/1903466397 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f977800ee60 con 0x7f9788075770 2024-05-20T20:13:26.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.376+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f97881a10d0 con 0x7f9788075770 2024-05-20T20:13:26.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.376+0000 7f977f400700 1 -- 172.21.15.28:0/1903466397 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f97780055c0 con 0x7f9788075770 2024-05-20T20:13:26.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.376+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f97881a8950 con 0x7f9788075770 2024-05-20T20:13:26.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.380+0000 7f977f400700 1 -- 172.21.15.28:0/1903466397 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f9778004750 con 0x7f9788075770 2024-05-20T20:13:26.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 Processor -- start 2024-05-20T20:13:26.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 -- start start 2024-05-20T20:13:26.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada37c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfad9d960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfad9e3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55fbfacfbf40 con 0x55fbfad9df80 2024-05-20T20:13:26.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55fbfacebae0 con 0x55fbfada33e0 2024-05-20T20:13:26.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.381+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55fbfacefb20 con 0x55fbfada3d00 2024-05-20T20:13:26.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada37c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada37c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:33748/0 (socket says 172.21.15.28:33748) 2024-05-20T20:13:26.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 -- 172.21.15.28:0/3899440206 learned_addr learned my addr 172.21.15.28:0/3899440206 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:26.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9786800700 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfad9d960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f977fe00700 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfad9e3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f977f400700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f976c0b29f0 0x7f976c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f977f400700 1 -- 172.21.15.28:0/1903466397 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f97780e2910 con 0x7f9788075770 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 -- 172.21.15.28:0/3899440206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 msgr2=0x55fbfad9d960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfad9d960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 -- 172.21.15.28:0/3899440206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 msgr2=0x55fbfad9e3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfad9e3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.382+0000 7f9787200700 1 -- 172.21.15.28:0/3899440206 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fbfacb4520 con 0x55fbfada33e0 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.383+0000 7f9787200700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f976c0b29f0 0x7f976c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.383+0000 7f9787200700 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada37c0 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f978000b0a0 tx=0x7f9780009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2f88d97444d62d3c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:26.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.383+0000 7f9777e00700 1 -- 172.21.15.28:0/3899440206 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f978002e070 con 0x55fbfada33e0 2024-05-20T20:13:26.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.383+0000 7f9777e00700 1 -- 172.21.15.28:0/3899440206 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9780024070 con 0x55fbfada33e0 2024-05-20T20:13:26.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.383+0000 7f9777e00700 1 -- 172.21.15.28:0/3899440206 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f97800052a0 con 0x55fbfada33e0 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 -- 172.21.15.28:0/3899440206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 msgr2=0x55fbfada37c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada37c0 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f978000b0a0 tx=0x7f9780009560 comp rx=0 tx=0).stop 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9787200700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f976c0b29f0 0x7f976c0b4eb0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f978002b000 tx=0x7f978002c000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 -- 172.21.15.28:0/3899440206 shutdown_connections 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfad9e3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada37c0 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/3899440206 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfad9d960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 -- 172.21.15.28:0/3899440206 >> 172.21.15.28:0/3899440206 conn(0x55fbfaccbce0 msgr2=0x55fbfad98420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 -- 172.21.15.28:0/3899440206 shutdown_connections 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.384+0000 7f9797cbfb80 1 -- 172.21.15.28:0/3899440206 wait complete. 2024-05-20T20:13:26.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.385+0000 7f9797cbfb80 1 -- start start 2024-05-20T20:13:26.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.385+0000 7f9797cbfb80 1 Processor -- start 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfad9df80 0x55fbfada99c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada33e0 0x55fbfada6d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada3d00 0x55fbfada72a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55fbfabf20a0 con 0x55fbfada3d00 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55fbfac8c860 con 0x55fbfad9df80 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55fbfadaa7e0 con 0x55fbfada33e0 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9797cbfb80 10 client.0 _ll_register_callbacks cb 0x55fbfac95d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9787200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada3d00 0x55fbfada72a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9787200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada3d00 0x55fbfada72a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53810/0 (socket says 172.21.15.28:53810) 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9787200700 1 -- 172.21.15.28:0/4178271940 learned_addr learned my addr 172.21.15.28:0/4178271940 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.386+0000 7f9786800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfad9df80 0x55fbfada99c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9786800700 1 -- 172.21.15.28:0/4178271940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada33e0 msgr2=0x55fbfada6d60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:13:26.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9786800700 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada33e0 0x55fbfada6d60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9786800700 1 -- 172.21.15.28:0/4178271940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada3d00 msgr2=0x55fbfada72a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:26.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9786800700 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada3d00 0x55fbfada72a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9786800700 1 -- 172.21.15.28:0/4178271940 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fbfac3b9d0 con 0x55fbfad9df80 2024-05-20T20:13:26.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9787200700 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada3d00 0x55fbfada72a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:26.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9786800700 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfad9df80 0x55fbfada99c0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f97780dfff0 tx=0x7f97780aa160 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e562637c275cf345 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:26.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9774c00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:26.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9774c00700 1 -- 172.21.15.28:0/4178271940 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f97780e8020 con 0x55fbfad9df80 2024-05-20T20:13:26.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9774c00700 1 -- 172.21.15.28:0/4178271940 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f97780a8de0 con 0x55fbfad9df80 2024-05-20T20:13:26.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.387+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9780024070 con 0x55fbfad9df80 2024-05-20T20:13:26.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9774c00700 1 -- 172.21.15.28:0/4178271940 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f97780a92f0 con 0x55fbfad9df80 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 10 client.27098 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55fbfacbc160 con 0x55fbfad9df80 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9774c00700 1 -- 172.21.15.28:0/4178271940 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9780024070 con 0x55fbfad9df80 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9774c00700 1 -- 172.21.15.28:0/4178271940 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9778004a40 con 0x55fbfad9df80 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 10 client.27098 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 10 client.27098 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 20 client.27098 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfadb97e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 4 client.27098 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.388+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55fbfacfde90 con 0x55fbfadb73b0 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.389+0000 7f977fe00700 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfadb97e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.389+0000 7f977fe00700 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfadb97e0 crc :-1 s=READY pgs=177 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9a7adabd05745ba2 server_cookie=f5be2db35a5fde98 in_seq=0 out_seq=0 2024-05-20T20:13:26.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.389+0000 7f9774c00700 10 client.27098 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:26.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.390+0000 7f9774c00700 1 -- 172.21.15.28:0/4178271940 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55fbfacfde90 con 0x55fbfadb73b0 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.390+0000 7f9774c00700 10 client.27098 handle_command_reply: tid=0 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.390+0000 7f9797cbfb80 1 client.27098 shutdown 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.390+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfad9df80 msgr2=0x55fbfada99c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.390+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfad9df80 0x55fbfada99c0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7f97780dfff0 tx=0x7f97780aa160 comp rx=0 tx=0).stop 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 shutdown_connections 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfadb97e0 crc :-1 s=READY pgs=177 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada3d00 0x55fbfada72a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfad9df80 0x55fbfada99c0 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9774c00700 0 client.27098 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/4178271940 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada33e0 0x55fbfada6d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 >> 172.21.15.28:0/4178271940 conn(0x55fbfaccbce0 msgr2=0x55fbfad9c950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 shutdown_connections 2024-05-20T20:13:26.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 1 -- 172.21.15.28:0/4178271940 wait complete. 2024-05-20T20:13:26.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.391+0000 7f9797cbfb80 20 client.27098 trim_cache size 0 max 16384 2024-05-20T20:13:26.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 Processor -- start 2024-05-20T20:13:26.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 -- start start 2024-05-20T20:13:26.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfada5cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada7250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfadbf0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f97881affd0 con 0x55fbfad9df80 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f97881b1b10 con 0x55fbfada33e0 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f97881b06a0 con 0x55fbfada3d00 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.414+0000 7f9786800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfada5cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfada5cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53814/0 (socket says 172.21.15.28:53814) 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 -- 172.21.15.28:0/232114443 learned_addr learned my addr 172.21.15.28:0/232114443 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f977fe00700 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada7250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9787200700 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfadbf0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 -- 172.21.15.28:0/232114443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 msgr2=0x55fbfadbf0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfadbf0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 -- 172.21.15.28:0/232114443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 msgr2=0x55fbfada7250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada7250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 -- 172.21.15.28:0/232114443 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fbfada52d0 con 0x55fbfad9df80 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.415+0000 7f9786800700 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfada5cc0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f977800c5b0 tx=0x7f9778005ec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=acae56c220cd1865 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:26.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9777400700 1 -- 172.21.15.28:0/232114443 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f97780a5310 con 0x55fbfad9df80 2024-05-20T20:13:26.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9777400700 1 -- 172.21.15.28:0/232114443 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f97780a9690 con 0x55fbfad9df80 2024-05-20T20:13:26.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9777400700 1 -- 172.21.15.28:0/232114443 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f97780a54f0 con 0x55fbfad9df80 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9797cbfb80 1 -- 172.21.15.28:0/232114443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 msgr2=0x55fbfada5cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfada5cc0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f977800c5b0 tx=0x7f9778005ec0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9797cbfb80 1 -- 172.21.15.28:0/232114443 shutdown_connections 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfada3d00 0x55fbfadbf0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfad9df80 0x55fbfada5cc0 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.416+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/232114443 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada33e0 0x55fbfada7250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.417+0000 7f9797cbfb80 1 -- 172.21.15.28:0/232114443 >> 172.21.15.28:0/232114443 conn(0x55fbfaccbce0 msgr2=0x55fbfada6b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:26.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.417+0000 7f9797cbfb80 1 -- 172.21.15.28:0/232114443 shutdown_connections 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.417+0000 7f9797cbfb80 1 -- 172.21.15.28:0/232114443 wait complete. 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.417+0000 7f9797cbfb80 1 -- start start 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.417+0000 7f9797cbfb80 1 Processor -- start 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfad9df80 0x55fbfadafc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada33e0 0x55fbfadb01c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9797cbfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada3d00 0x55fbfadaca70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f97881b1440 con 0x55fbfada33e0 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f97881b0d70 con 0x55fbfada3d00 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9797cbfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f97881adf90 con 0x55fbfad9df80 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9797cbfb80 10 client.0 _ll_register_callbacks cb 0x55fbfac95d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9787200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada3d00 0x55fbfadaca70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f977fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada33e0 0x55fbfadb01c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9787200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada3d00 0x55fbfadaca70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:33778/0 (socket says 172.21.15.28:33778) 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9787200700 1 -- 172.21.15.28:0/781621775 learned_addr learned my addr 172.21.15.28:0/781621775 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:26.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.418+0000 7f9786800700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfad9df80 0x55fbfadafc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9786800700 1 -- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada3d00 msgr2=0x55fbfadaca70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:26.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9786800700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada3d00 0x55fbfadaca70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9786800700 1 -- 172.21.15.28:0/781621775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada33e0 msgr2=0x55fbfadb01c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:26.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9786800700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada33e0 0x55fbfadb01c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9786800700 1 -- 172.21.15.28:0/781621775 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fbfadad0f0 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9787200700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada3d00 0x55fbfadaca70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f977fe00700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada33e0 0x55fbfadb01c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9786800700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfad9df80 0x55fbfadafc80 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f97780a5c00 tx=0x7f977800ce70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b48bc8f8b842495d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9774c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9774c00700 1 -- 172.21.15.28:0/781621775 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f97780adf30 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.419+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f97780a9690 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9774c00700 1 -- 172.21.15.28:0/781621775 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f97780c4060 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9774c00700 1 -- 172.21.15.28:0/781621775 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f97780aa280 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 10 client.26995 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55fbfadad0f0 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9774c00700 1 -- 172.21.15.28:0/781621775 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f97780a9690 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9774c00700 1 -- 172.21.15.28:0/781621775 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f97780a89e0 con 0x55fbfad9df80 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 10 client.26995 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:26.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 10 client.26995 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:13:26.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 20 client.26995 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:26.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfae24b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:26.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 4 client.26995 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "4491be01", "release": true, "await": true}] 2024-05-20T20:13:26.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.420+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "4491be01", "release": true, "await": true}) -- 0x55fbfacf7670 con 0x55fbfadb73b0 2024-05-20T20:13:26.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.421+0000 7f977fe00700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfae24b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:26.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.421+0000 7f977fe00700 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfae24b50 crc :-1 s=READY pgs=178 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4b1d7b4ecca023d6 server_cookie=5ddb51d7dc48974f in_seq=0 out_seq=0 2024-05-20T20:13:26.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.421+0000 7f9774c00700 10 client.26995 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:26.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.429+0000 7f9774c00700 1 -- 172.21.15.28:0/781621775 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x55fbfacf7670 con 0x55fbfadb73b0 2024-05-20T20:13:26.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.429+0000 7f9774c00700 10 client.26995 handle_command_reply: tid=0 2024-05-20T20:13:26.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.429+0000 7f9797cbfb80 1 client.26995 shutdown 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfad9df80 msgr2=0x55fbfadafc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfad9df80 0x55fbfadafc80 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f97780a5c00 tx=0x7f977800ce70 comp rx=0 tx=0).stop 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 shutdown_connections 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55fbfadb73b0 0x55fbfae24b50 crc :-1 s=READY pgs=178 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55fbfada3d00 0x55fbfadaca70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55fbfad9df80 0x55fbfadafc80 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 --2- 172.21.15.28:0/781621775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55fbfada33e0 0x55fbfadb01c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 >> 172.21.15.28:0/781621775 conn(0x55fbfaccbce0 msgr2=0x55fbfad91d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:26.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.430+0000 7f9774c00700 0 client.26995 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:26.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.431+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 shutdown_connections 2024-05-20T20:13:26.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.431+0000 7f9797cbfb80 1 -- 172.21.15.28:0/781621775 wait complete. 2024-05-20T20:13:26.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.431+0000 7f9797cbfb80 20 client.26995 trim_cache size 0 max 16384 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 20, 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "4491be01": { 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 20, 2024-05-20T20:13:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 143.3, 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:13:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:26.435 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:13:26.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f976c0b29f0 msgr2=0x7f976c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f976c0b29f0 0x7f976c0b4eb0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f978002b000 tx=0x7f978002c000 comp rx=0 tx=0).stop 2024-05-20T20:13:26.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788075770 msgr2=0x7f978819bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:26.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788075770 0x7f978819bdd0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f977800e9c0 tx=0x7f977800ecd0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 shutdown_connections 2024-05-20T20:13:26.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f978810ffa0 0x7f97881a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f976c0b29f0 0x7f976c0b4eb0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f9788075770 0x7f978819bdd0 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 --2- 172.21.15.28:0/1903466397 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9788076090 0x7f978819c310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:26.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 >> 172.21.15.28:0/1903466397 conn(0x7f978807a6a0 msgr2=0x7f9788100710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:26.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.435+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 shutdown_connections 2024-05-20T20:13:26.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:26.436+0000 7f978d400700 1 -- 172.21.15.28:0/1903466397 wait complete. 2024-05-20T20:13:27.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:26 smithi028 ceph-mon[31178]: pgmap v241: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 225 MiB/s wr, 1.90k op/s 2024-05-20T20:13:27.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:26 smithi154 ceph-mon[36314]: pgmap v241: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 225 MiB/s wr, 1.90k op/s 2024-05-20T20:13:27.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:26 smithi033 ceph-mon[36179]: pgmap v241: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 225 MiB/s wr, 1.90k op/s 2024-05-20T20:13:29.010 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set '4491be01', total seconds elapsed: 38.2 2024-05-20T20:13:29.010 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 23.3 2024-05-20T20:13:29.011 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:13:29.131 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:29 smithi028 ceph-mon[31178]: pgmap v242: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 194 MiB/s wr, 1.64k op/s 2024-05-20T20:13:29.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:29 smithi154 ceph-mon[36314]: pgmap v242: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 194 MiB/s wr, 1.64k op/s 2024-05-20T20:13:29.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:29 smithi033 ceph-mon[36179]: pgmap v242: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 194 MiB/s wr, 1.64k op/s 2024-05-20T20:13:30.596 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.469+0000 7fc26dc00700 1 -- 172.21.15.28:0/2763999592 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26806bb70 msgr2=0x7fc26810fab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.469+0000 7fc26dc00700 1 --2- 172.21.15.28:0/2763999592 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26806bb70 0x7fc26810fab0 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7fc258009990 tx=0x7fc258009ca0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.470+0000 7fc26dc00700 1 -- 172.21.15.28:0/2763999592 shutdown_connections 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.470+0000 7fc26dc00700 1 --2- 172.21.15.28:0/2763999592 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc26810fff0 0x7fc2681123e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.470+0000 7fc26dc00700 1 --2- 172.21.15.28:0/2763999592 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc26806b250 0x7fc26806b630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.470+0000 7fc26dc00700 1 --2- 172.21.15.28:0/2763999592 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26806bb70 0x7fc26810fab0 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.470+0000 7fc26dc00700 1 -- 172.21.15.28:0/2763999592 >> 172.21.15.28:0/2763999592 conn(0x7fc2680793f0 msgr2=0x7fc268079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.472+0000 7fc26dc00700 1 -- 172.21.15.28:0/2763999592 shutdown_connections 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.472+0000 7fc26dc00700 1 -- 172.21.15.28:0/2763999592 wait complete. 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.475+0000 7fc26dc00700 1 Processor -- start 2024-05-20T20:13:30.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.476+0000 7fc26dc00700 1 -- start start 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.476+0000 7fc26dc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc26806b250 0x7fc268109140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.476+0000 7fc26dc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc26806bb70 0x7fc268109680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.476+0000 7fc26dc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26810fff0 0x7fc268104650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.476+0000 7fc26dc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc2681159a0 con 0x7fc26806b250 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.476+0000 7fc26dc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc268115820 con 0x7fc26806bb70 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.476+0000 7fc26dc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc268115b20 con 0x7fc26810fff0 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.477+0000 7fc266000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc26806bb70 0x7fc268109680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.477+0000 7fc267400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26810fff0 0x7fc268104650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.477+0000 7fc266000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc26806bb70 0x7fc268109680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54882/0 (socket says 172.21.15.28:54882) 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.477+0000 7fc266000700 1 -- 172.21.15.28:0/365963920 learned_addr learned my addr 172.21.15.28:0/365963920 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:30.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.478+0000 7fc267400700 1 -- 172.21.15.28:0/365963920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc26806bb70 msgr2=0x7fc268109680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.478+0000 7fc267400700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc26806bb70 0x7fc268109680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.478+0000 7fc267400700 1 -- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc26806b250 msgr2=0x7fc268109140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.478+0000 7fc267400700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc26806b250 0x7fc268109140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.478+0000 7fc267400700 1 -- 172.21.15.28:0/365963920 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc268104f50 con 0x7fc26810fff0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.479+0000 7fc267400700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26810fff0 0x7fc268104650 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7fc260007ee0 tx=0x7fc26000d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.479+0000 7fc25f400700 1 -- 172.21.15.28:0/365963920 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc26000ddd0 con 0x7fc26810fff0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.480+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc268105240 con 0x7fc26810fff0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.480+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc26810f030 con 0x7fc26810fff0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.480+0000 7fc25f400700 1 -- 172.21.15.28:0/365963920 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc260004bb0 con 0x7fc26810fff0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.480+0000 7fc25f400700 1 -- 172.21.15.28:0/365963920 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc26000ddd0 con 0x7fc26810fff0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.486+0000 7fc25f400700 1 -- 172.21.15.28:0/365963920 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc26001c020 con 0x7fc26810fff0 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.488+0000 7fc278537b80 1 Processor -- start 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.488+0000 7fc278537b80 1 -- start start 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.489+0000 7fc278537b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9339f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.489+0000 7fc278537b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae92db20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.489+0000 7fc278537b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92e5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.489+0000 7fc278537b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5612ae88be40 con 0x5612ae92e140 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.489+0000 7fc278537b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5612ae87b9e0 con 0x5612ae933f30 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.489+0000 7fc278537b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5612ae87fa20 con 0x5612ae933610 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.490+0000 7fc266000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9339f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.490+0000 7fc266000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9339f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37356/0 (socket says 172.21.15.28:37356) 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.491+0000 7fc266000700 1 -- 172.21.15.28:0/2321756121 learned_addr learned my addr 172.21.15.28:0/2321756121 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.491+0000 7fc266000700 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92e5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.491+0000 7fc266a00700 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae92db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.492+0000 7fc25f400700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc24c0b2d20 0x7fc24c0b51e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.492+0000 7fc266000700 1 -- 172.21.15.28:0/2321756121 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 msgr2=0x5612ae92db20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.492+0000 7fc266000700 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae92db20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.492+0000 7fc266000700 1 -- 172.21.15.28:0/2321756121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 msgr2=0x5612ae92e5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.492+0000 7fc25f400700 1 -- 172.21.15.28:0/365963920 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fc2600e1920 con 0x7fc26810fff0 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.492+0000 7fc266000700 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92e5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.492+0000 7fc266000700 1 -- 172.21.15.28:0/2321756121 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5612ae844420 con 0x5612ae933610 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.493+0000 7fc266000700 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9339f0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7fc258006010 tx=0x7fc25801a040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b9c76ecc415b785e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.493+0000 7fc267400700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc24c0b2d20 0x7fc24c0b51e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.494+0000 7fc253e00700 1 -- 172.21.15.28:0/2321756121 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc25800b980 con 0x5612ae933610 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.494+0000 7fc267400700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc24c0b2d20 0x7fc24c0b51e0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fc2600a8890 tx=0x7fc26000b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.494+0000 7fc253e00700 1 -- 172.21.15.28:0/2321756121 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc258003860 con 0x5612ae933610 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.494+0000 7fc253e00700 1 -- 172.21.15.28:0/2321756121 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc258005300 con 0x5612ae933610 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 -- 172.21.15.28:0/2321756121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 msgr2=0x5612ae9339f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9339f0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7fc258006010 tx=0x7fc25801a040 comp rx=0 tx=0).stop 2024-05-20T20:13:30.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 -- 172.21.15.28:0/2321756121 shutdown_connections 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9339f0 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 --2- 172.21.15.28:0/2321756121 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae92db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 -- 172.21.15.28:0/2321756121 >> 172.21.15.28:0/2321756121 conn(0x5612ae85bbe0 msgr2=0x5612ae928690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 -- 172.21.15.28:0/2321756121 shutdown_connections 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.495+0000 7fc278537b80 1 -- 172.21.15.28:0/2321756121 wait complete. 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.496+0000 7fc278537b80 1 -- start start 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.496+0000 7fc278537b80 1 Processor -- start 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc278537b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae92e140 0x5612ae939c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc278537b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933610 0x5612ae936fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc278537b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae937520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc278537b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5612ae781fa0 con 0x5612ae933f30 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc278537b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5612ae81c760 con 0x5612ae933610 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc278537b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5612ae93a9d0 con 0x5612ae92e140 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc278537b80 10 client.0 _ll_register_callbacks cb 0x5612ae825c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.498+0000 7fc266000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae92e140 0x5612ae939c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.499+0000 7fc266000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae92e140 0x5612ae939c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37372/0 (socket says 172.21.15.28:37372) 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.499+0000 7fc266000700 1 -- 172.21.15.28:0/618520480 learned_addr learned my addr 172.21.15.28:0/618520480 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.499+0000 7fc266000700 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae937520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.499+0000 7fc266000700 1 -- 172.21.15.28:0/618520480 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933610 msgr2=0x5612ae936fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.604 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.499+0000 7fc266000700 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933610 0x5612ae936fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.500+0000 7fc266000700 1 -- 172.21.15.28:0/618520480 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 msgr2=0x5612ae937520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.500+0000 7fc266000700 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae937520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.500+0000 7fc266000700 1 -- 172.21.15.28:0/618520480 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5612ae7cb8d0 con 0x5612ae92e140 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.501+0000 7fc266000700 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae92e140 0x5612ae939c40 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7fc258005820 tx=0x7fc258005a90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=35a6bbb9d4432c19 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.501+0000 7fc250c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.501+0000 7fc250c00700 1 -- 172.21.15.28:0/618520480 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc25800b900 con 0x5612ae92e140 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.501+0000 7fc250c00700 1 -- 172.21.15.28:0/618520480 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fc25800baa0 con 0x5612ae92e140 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.501+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc258003860 con 0x5612ae92e140 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.502+0000 7fc250c00700 1 -- 172.21.15.28:0/618520480 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc25800bca0 con 0x5612ae92e140 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.502+0000 7fc278537b80 10 client.27007 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.502+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5612ae84c060 con 0x5612ae92e140 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.503+0000 7fc250c00700 1 -- 172.21.15.28:0/618520480 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc258003860 con 0x5612ae92e140 2024-05-20T20:13:30.605 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc250c00700 1 -- 172.21.15.28:0/618520480 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc2580173f0 con 0x5612ae92e140 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc278537b80 10 client.27007 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc278537b80 10 client.27007 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc278537b80 20 client.27007 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc278537b80 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae947400 0x5612ae9497f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc278537b80 4 client.27007 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5612ae88dd90 con 0x5612ae947400 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.504+0000 7fc266a00700 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae947400 0x5612ae9497f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.505+0000 7fc266a00700 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae947400 0x5612ae9497f0 crc :-1 s=READY pgs=179 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=eddba18b252a994c server_cookie=b3a80575695f79f0 in_seq=0 out_seq=0 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.505+0000 7fc250c00700 10 client.27007 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.506+0000 7fc250c00700 1 -- 172.21.15.28:0/618520480 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5612ae88dd90 con 0x5612ae947400 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.506+0000 7fc250c00700 10 client.27007 handle_command_reply: tid=0 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.506+0000 7fc278537b80 1 client.27007 shutdown 2024-05-20T20:13:30.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.506+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae92e140 msgr2=0x5612ae939c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.506+0000 7fc278537b80 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae92e140 0x5612ae939c40 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7fc258005820 tx=0x7fc258005a90 comp rx=0 tx=0).stop 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.506+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 shutdown_connections 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.506+0000 7fc278537b80 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae947400 0x5612ae9497f0 crc :-1 s=READY pgs=179 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc278537b80 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae937520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc250c00700 0 client.27007 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc278537b80 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae92e140 0x5612ae939c40 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc278537b80 1 --2- 172.21.15.28:0/618520480 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933610 0x5612ae936fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 >> 172.21.15.28:0/618520480 conn(0x5612ae85bbe0 msgr2=0x5612ae927d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 shutdown_connections 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc278537b80 1 -- 172.21.15.28:0/618520480 wait complete. 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.507+0000 7fc278537b80 20 client.27007 trim_cache size 0 max 16384 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.534+0000 7fc278537b80 1 Processor -- start 2024-05-20T20:13:30.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.535+0000 7fc278537b80 1 -- start start 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.535+0000 7fc278537b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92db20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.535+0000 7fc278537b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9374d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.535+0000 7fc278537b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae9284f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.535+0000 7fc278537b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc24800d6e0 con 0x5612ae92e140 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.535+0000 7fc278537b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc24800f220 con 0x5612ae933f30 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.535+0000 7fc278537b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc24800ddb0 con 0x5612ae933610 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.536+0000 7fc266a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9374d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.536+0000 7fc266a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9374d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37376/0 (socket says 172.21.15.28:37376) 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.536+0000 7fc266a00700 1 -- 172.21.15.28:0/88035389 learned_addr learned my addr 172.21.15.28:0/88035389 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.536+0000 7fc266000700 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae9284f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.536+0000 7fc266000700 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.537+0000 7fc266a00700 1 -- 172.21.15.28:0/88035389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 msgr2=0x5612ae9284f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.537+0000 7fc266a00700 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae9284f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.537+0000 7fc266a00700 1 -- 172.21.15.28:0/88035389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 msgr2=0x5612ae92db20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.538+0000 7fc266a00700 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92db20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.538+0000 7fc266a00700 1 -- 172.21.15.28:0/88035389 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5612ae920620 con 0x5612ae933610 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.538+0000 7fc266000700 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.538+0000 7fc266a00700 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9374d0 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7fc254007c40 tx=0x7fc254007f50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=66a30e121b56f8ba server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.539+0000 7fc253400700 1 -- 172.21.15.28:0/88035389 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc254012cd0 con 0x5612ae933610 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.539+0000 7fc253400700 1 -- 172.21.15.28:0/88035389 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc254004500 con 0x5612ae933610 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.539+0000 7fc253400700 1 -- 172.21.15.28:0/88035389 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc25401a3f0 con 0x5612ae933610 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.541+0000 7fc278537b80 1 -- 172.21.15.28:0/88035389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 msgr2=0x5612ae9374d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.541+0000 7fc278537b80 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9374d0 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7fc254007c40 tx=0x7fc254007f50 comp rx=0 tx=0).stop 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.541+0000 7fc278537b80 1 -- 172.21.15.28:0/88035389 shutdown_connections 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.541+0000 7fc278537b80 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae933f30 0x5612ae9284f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.541+0000 7fc278537b80 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae92e140 0x5612ae92db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.541+0000 7fc278537b80 1 --2- 172.21.15.28:0/88035389 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae933610 0x5612ae9374d0 secure :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7fc254007c40 tx=0x7fc254007f50 comp rx=0 tx=0).stop 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.541+0000 7fc278537b80 1 -- 172.21.15.28:0/88035389 >> 172.21.15.28:0/88035389 conn(0x5612ae85bbe0 msgr2=0x5612ae936dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.542+0000 7fc278537b80 1 -- 172.21.15.28:0/88035389 shutdown_connections 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.543+0000 7fc278537b80 1 -- 172.21.15.28:0/88035389 wait complete. 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.543+0000 7fc278537b80 1 -- start start 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 1 Processor -- start 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae92e140 0x5612ae94ce20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae94a170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae947400 0x5612ae94a6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc24800eb50 con 0x5612ae933f30 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc24800e480 con 0x5612ae92e140 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc24800b6a0 con 0x5612ae947400 2024-05-20T20:13:30.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.544+0000 7fc278537b80 10 client.0 _ll_register_callbacks cb 0x5612ae825c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.545+0000 7fc266a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae94a170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.545+0000 7fc266000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae947400 0x5612ae94a6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.546+0000 7fc266a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae94a170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37278/0 (socket says 172.21.15.28:37278) 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.546+0000 7fc266a00700 1 -- 172.21.15.28:0/3681441450 learned_addr learned my addr 172.21.15.28:0/3681441450 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.546+0000 7fc266000700 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae92e140 0x5612ae94ce20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc266000700 1 -- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae92e140 msgr2=0x5612ae94ce20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc266000700 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae92e140 0x5612ae94ce20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc266000700 1 -- 172.21.15.28:0/3681441450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 msgr2=0x5612ae94a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc266000700 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae94a170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc266000700 1 -- 172.21.15.28:0/3681441450 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5612ae94ae00 con 0x5612ae947400 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc266a00700 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae94a170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:30.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc266000700 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae947400 0x5612ae94a6b0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x5612ae92e850 tx=0x7fc25801b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9c1bcb7ed92d3b62 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc250c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.547+0000 7fc250c00700 1 -- 172.21.15.28:0/3681441450 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc258005620 con 0x5612ae947400 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.548+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc254004500 con 0x5612ae947400 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.548+0000 7fc250c00700 1 -- 172.21.15.28:0/3681441450 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fc25800b880 con 0x5612ae947400 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.548+0000 7fc250c00700 1 -- 172.21.15.28:0/3681441450 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc25800baa0 con 0x5612ae947400 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.548+0000 7fc278537b80 10 client.27013 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5612ae9c2b20 con 0x5612ae947400 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc250c00700 1 -- 172.21.15.28:0/3681441450 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc254004500 con 0x5612ae947400 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc250c00700 1 -- 172.21.15.28:0/3681441450 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc258021070 con 0x5612ae947400 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc278537b80 10 client.27013 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc278537b80 10 client.27013 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc278537b80 20 client.27013 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:30.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc278537b80 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae9c31a0 0x5612ae9c55c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc278537b80 4 client.27013 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.549+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5612ae887570 con 0x5612ae9c31a0 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.550+0000 7fc266a00700 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae9c31a0 0x5612ae9c55c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.551+0000 7fc266a00700 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae9c31a0 0x5612ae9c55c0 crc :-1 s=READY pgs=180 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=32549f29d9e93711 server_cookie=49352cfc5f65882e in_seq=0 out_seq=0 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.551+0000 7fc250c00700 10 client.27013 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.556+0000 7fc250c00700 1 -- 172.21.15.28:0/3681441450 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5612ae887570 con 0x5612ae9c31a0 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.556+0000 7fc250c00700 10 client.27013 handle_command_reply: tid=0 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.556+0000 7fc278537b80 1 client.27013 shutdown 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae947400 msgr2=0x5612ae94a6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae947400 0x5612ae94a6b0 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x5612ae92e850 tx=0x7fc25801b040 comp rx=0 tx=0).stop 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 shutdown_connections 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5612ae9c31a0 0x5612ae9c55c0 crc :-1 s=READY pgs=180 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5612ae947400 0x5612ae94a6b0 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc250c00700 0 client.27013 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5612ae92e140 0x5612ae94ce20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 --2- 172.21.15.28:0/3681441450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5612ae933f30 0x5612ae94a170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.557+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 >> 172.21.15.28:0/3681441450 conn(0x5612ae85bbe0 msgr2=0x5612ae934310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.558+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 shutdown_connections 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.558+0000 7fc278537b80 1 -- 172.21.15.28:0/3681441450 wait complete. 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.558+0000 7fc278537b80 20 client.27013 trim_cache size 0 max 16384 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc24c0b2d20 msgr2=0x7fc24c0b51e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc24c0b2d20 0x7fc24c0b51e0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fc2600a8890 tx=0x7fc26000b040 comp rx=0 tx=0).stop 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26810fff0 msgr2=0x7fc268104650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26810fff0 0x7fc268104650 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7fc260007ee0 tx=0x7fc26000d3b0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 shutdown_connections 2024-05-20T20:13:30.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc26810fff0 0x7fc268104650 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc26806b250 0x7fc268109140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc24c0b2d20 0x7fc24c0b51e0 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 --2- 172.21.15.28:0/365963920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc26806bb70 0x7fc268109680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:30.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.560+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 >> 172.21.15.28:0/365963920 conn(0x7fc2680793f0 msgr2=0x7fc268111440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:30.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.577+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 shutdown_connections 2024-05-20T20:13:30.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:30.577+0000 7fc26dc00700 1 -- 172.21.15.28:0/365963920 wait complete. 2024-05-20T20:13:31.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:31 smithi028 ceph-mon[31178]: pgmap v243: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 182 MiB/s wr, 1.64k op/s 2024-05-20T20:13:31.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:31 smithi154 ceph-mon[36314]: pgmap v243: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 182 MiB/s wr, 1.64k op/s 2024-05-20T20:13:31.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:31 smithi033 ceph-mon[36179]: pgmap v243: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 182 MiB/s wr, 1.64k op/s 2024-05-20T20:13:33.040 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:33.215 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:33 smithi028 ceph-mon[31178]: pgmap v244: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 63 MiB/s wr, 729 op/s 2024-05-20T20:13:33.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:33 smithi154 ceph-mon[36314]: pgmap v244: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 63 MiB/s wr, 729 op/s 2024-05-20T20:13:33.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:33 smithi033 ceph-mon[36179]: pgmap v244: 97 pgs: 97 active+clean; 46 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 63 MiB/s wr, 729 op/s 2024-05-20T20:13:33.946 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:13:33.946 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:13:34.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.293+0000 7f2929800640 1 -- 172.21.15.28:0/2273259540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f292410de30 msgr2=0x7f292411a2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.293+0000 7f2929800640 1 --2- 172.21.15.28:0/2273259540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f292410de30 0x7f292411a2f0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f291c009cd0 tx=0x7f291c02a910 comp rx=0 tx=0).stop 2024-05-20T20:13:34.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.294+0000 7f2929800640 1 -- 172.21.15.28:0/2273259540 shutdown_connections 2024-05-20T20:13:34.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.294+0000 7f2929800640 1 --2- 172.21.15.28:0/2273259540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f292410d470 0x7f292410d8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.294+0000 7f2929800640 1 --2- 172.21.15.28:0/2273259540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f292410de30 0x7f292411a2f0 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.294+0000 7f2929800640 1 --2- 172.21.15.28:0/2273259540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f29241134c0 0x7f29241138a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.294+0000 7f2929800640 1 -- 172.21.15.28:0/2273259540 >> 172.21.15.28:0/2273259540 conn(0x7f292406dcf0 msgr2=0x7f292410ba60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:34.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.294+0000 7f2929800640 1 -- 172.21.15.28:0/2273259540 shutdown_connections 2024-05-20T20:13:34.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.294+0000 7f2929800640 1 -- 172.21.15.28:0/2273259540 wait complete. 2024-05-20T20:13:34.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.296+0000 7f2929800640 1 Processor -- start 2024-05-20T20:13:34.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.296+0000 7f2929800640 1 -- start start 2024-05-20T20:13:34.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.297+0000 7f2929800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f292410d470 0x7f292411a1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.297+0000 7f2929800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f292410de30 0x7f29241172b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.298+0000 7f2922000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f292410de30 0x7f29241172b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.298+0000 7f2929800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29241134c0 0x7f2924117810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.298+0000 7f2929800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f292411d790 con 0x7f292410de30 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.298+0000 7f2929800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f292411d610 con 0x7f292410d470 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.298+0000 7f2929800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f292411d910 con 0x7f29241134c0 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.299+0000 7f2923400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29241134c0 0x7f2924117810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.299+0000 7f2923400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29241134c0 0x7f2924117810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37412/0 (socket says 172.21.15.28:37412) 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.299+0000 7f2923400640 1 -- 172.21.15.28:0/3909608214 learned_addr learned my addr 172.21.15.28:0/3909608214 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.299+0000 7f2923400640 1 -- 172.21.15.28:0/3909608214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f292410d470 msgr2=0x7f292411a1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.299+0000 7f2922a00640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f292410d470 0x7f292411a1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.300+0000 7f2923400640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f292410d470 0x7f292411a1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.300+0000 7f2923400640 1 -- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f292410de30 msgr2=0x7f29241172b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.300+0000 7f2923400640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f292410de30 0x7f29241172b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.300+0000 7f2923400640 1 -- 172.21.15.28:0/3909608214 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f29241180d0 con 0x7f29241134c0 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.300+0000 7f2922000640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f292410de30 0x7f29241172b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.300+0000 7f2923400640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29241134c0 0x7f2924117810 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f291c031a70 tx=0x7f291c003970 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.301+0000 7f2922a00640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f292410d470 0x7f292411a1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:34.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.301+0000 7f291b400640 1 -- 172.21.15.28:0/3909608214 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f291c003c60 con 0x7f29241134c0 2024-05-20T20:13:34.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.301+0000 7f291b400640 1 -- 172.21.15.28:0/3909608214 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f291c003e00 con 0x7f29241134c0 2024-05-20T20:13:34.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.302+0000 7f291b400640 1 -- 172.21.15.28:0/3909608214 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f291c02fcc0 con 0x7f29241134c0 2024-05-20T20:13:34.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.302+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f29241a9d70 con 0x7f29241134c0 2024-05-20T20:13:34.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.302+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f29241aa270 con 0x7f29241134c0 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.309+0000 7f291b400640 1 -- 172.21.15.28:0/3909608214 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f291c0305d0 con 0x7f29241134c0 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.311+0000 7f292c664740 1 Processor -- start 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.312+0000 7f292c664740 1 -- start start 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.312+0000 7f292c664740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1dd070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.312+0000 7f292c664740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e7930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.313+0000 7f292c664740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1ea260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.313+0000 7f292c664740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55786a14db90 con 0x55786a1dcc90 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.313+0000 7f292c664740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55786a13d730 con 0x55786a1dd5b0 2024-05-20T20:13:34.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.313+0000 7f292c664740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55786a141770 con 0x55786a1e7e70 2024-05-20T20:13:34.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.315+0000 7f2922a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e7930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.315+0000 7f291b400640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f29080b2ba0 0x7f29080b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.316+0000 7f291b400640 1 -- 172.21.15.28:0/3909608214 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f291c103a40 con 0x7f29241134c0 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.319+0000 7f2922a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e7930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54930/0 (socket says 172.21.15.28:54930) 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.319+0000 7f2922a00640 1 -- 172.21.15.28:0/2137112582 learned_addr learned my addr 172.21.15.28:0/2137112582 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.319+0000 7f2922000640 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1ea260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.320+0000 7f2922000640 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1dd070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.320+0000 7f2922000640 1 -- 172.21.15.28:0/2137112582 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 msgr2=0x55786a1e7930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.320+0000 7f2922000640 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e7930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.320+0000 7f2922000640 1 -- 172.21.15.28:0/2137112582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 msgr2=0x55786a1dd070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.320+0000 7f2922000640 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1dd070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.320+0000 7f2922000640 1 -- 172.21.15.28:0/2137112582 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55786a0d68b0 con 0x55786a1e7e70 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.321+0000 7f2922000640 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1ea260 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f2910006dc0 tx=0x7f291000a4d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bd394bb3044f4a07 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:34.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.322+0000 7f2923400640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f29080b2ba0 0x7f29080b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.323+0000 7f290fe00640 1 -- 172.21.15.28:0/2137112582 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f291001c070 con 0x55786a1e7e70 2024-05-20T20:13:34.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.323+0000 7f290fe00640 1 -- 172.21.15.28:0/2137112582 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f291000fd40 con 0x55786a1e7e70 2024-05-20T20:13:34.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.324+0000 7f292c664740 1 -- 172.21.15.28:0/2137112582 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 msgr2=0x55786a1ea260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.324+0000 7f292c664740 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1ea260 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7f2910006dc0 tx=0x7f291000a4d0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.326+0000 7f292c664740 1 -- 172.21.15.28:0/2137112582 shutdown_connections 2024-05-20T20:13:34.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.326+0000 7f292c664740 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.326 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.326+0000 7f292c664740 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1ea260 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.326+0000 7f292c664740 1 --2- 172.21.15.28:0/2137112582 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1dd070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.326+0000 7f292c664740 1 -- 172.21.15.28:0/2137112582 >> 172.21.15.28:0/2137112582 conn(0x55786a1d91a0 msgr2=0x55786a1d9580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:34.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.328+0000 7f2923400640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f29080b2ba0 0x7f29080b5060 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f291c0c6e40 tx=0x7f291c0098d0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:34.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.328+0000 7f292c664740 1 -- 172.21.15.28:0/2137112582 shutdown_connections 2024-05-20T20:13:34.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.328+0000 7f292c664740 1 -- 172.21.15.28:0/2137112582 wait complete. 2024-05-20T20:13:34.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.329+0000 7f292c664740 1 -- start start 2024-05-20T20:13:34.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.329+0000 7f292c664740 1 Processor -- start 2024-05-20T20:13:34.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.331+0000 7f292c664740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1e7780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.332 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.331+0000 7f292c664740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e4a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.332 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.331+0000 7f2922000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1e7780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.332 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.332+0000 7f2922000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1e7780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37316/0 (socket says 172.21.15.28:37316) 2024-05-20T20:13:34.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.332+0000 7f292c664740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1e4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.332+0000 7f292c664740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557869e23790 con 0x55786a1dcc90 2024-05-20T20:13:34.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.332+0000 7f292c664740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55786a0e5bc0 con 0x55786a1dd5b0 2024-05-20T20:13:34.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.332+0000 7f292c664740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55786a1eaaa0 con 0x55786a1e7e70 2024-05-20T20:13:34.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.332+0000 7f292c664740 10 client.0 _ll_register_callbacks cb 0x55786a0c3400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:34.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.333+0000 7f2922000640 1 -- 172.21.15.28:0/3292341856 learned_addr learned my addr 172.21.15.28:0/3292341856 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:34.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.333+0000 7f2922a00640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e4a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.333+0000 7f2922000640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1e4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.334+0000 7f2922a00640 1 -- 172.21.15.28:0/3292341856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 msgr2=0x55786a1e4fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.334+0000 7f2922a00640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1e4fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.334+0000 7f2922a00640 1 -- 172.21.15.28:0/3292341856 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 msgr2=0x55786a1e7780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.334+0000 7f2922a00640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1e7780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.334+0000 7f2922a00640 1 -- 172.21.15.28:0/3292341856 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557869e52210 con 0x55786a1dd5b0 2024-05-20T20:13:34.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.334+0000 7f2922000640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1e4fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:34.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.335+0000 7f2922a00640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e4a70 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x55786a1de230 tx=0x7f291400f950 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f7a6c3f76c801c35 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:34.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.335+0000 7f290cc00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:34.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.335+0000 7f290cc00640 1 -- 172.21.15.28:0/3292341856 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2914013850 con 0x55786a1dd5b0 2024-05-20T20:13:34.341 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.340+0000 7f290cc00640 1 -- 172.21.15.28:0/3292341856 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f29140139f0 con 0x55786a1dd5b0 2024-05-20T20:13:34.342 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.342+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f291000fd40 con 0x55786a1dd5b0 2024-05-20T20:13:34.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.342+0000 7f290cc00640 1 -- 172.21.15.28:0/3292341856 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2914013bf0 con 0x55786a1dd5b0 2024-05-20T20:13:34.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.342+0000 7f292c664740 10 client.27122 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:34.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.342+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557869f1e620 con 0x55786a1dd5b0 2024-05-20T20:13:34.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f290cc00640 1 -- 172.21.15.28:0/3292341856 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f291000fd40 con 0x55786a1dd5b0 2024-05-20T20:13:34.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f290cc00640 1 -- 172.21.15.28:0/3292341856 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f291401b400 con 0x55786a1dd5b0 2024-05-20T20:13:34.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f292c664740 10 client.27122 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:34.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f292c664740 10 client.27122 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:34.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f292c664740 20 client.27122 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:34.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f292c664740 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a1f7930 0x55786a1f9d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f292c664740 4 client.27122 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:34.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.347+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55786a14fae0 con 0x55786a1f7930 2024-05-20T20:13:34.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.348+0000 7f2922000640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a1f7930 0x55786a1f9d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.348+0000 7f290cc00640 10 client.27122 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:34.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.348+0000 7f2922000640 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a1f7930 0x55786a1f9d90 crc :-1 s=READY pgs=181 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f33ba251d350ad56 server_cookie=198c905ad080a72a in_seq=0 out_seq=0 2024-05-20T20:13:34.351 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f290cc00640 1 -- 172.21.15.28:0/3292341856 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55786a14fae0 con 0x55786a1f7930 2024-05-20T20:13:34.351 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f290cc00640 10 client.27122 handle_command_reply: tid=0 2024-05-20T20:13:34.351 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 client.27122 shutdown 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 msgr2=0x55786a1e4a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e4a70 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x55786a1de230 tx=0x7f291400f950 comp rx=0 tx=0).stop 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 shutdown_connections 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a1f7930 0x55786a1f9d90 crc :-1 s=READY pgs=181 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f290cc00640 0 client.27122 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1e4a70 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1e4fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 --2- 172.21.15.28:0/3292341856 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1e7780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.351+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 >> 172.21.15.28:0/3292341856 conn(0x55786a1d91a0 msgr2=0x55786a1def20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.352+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 shutdown_connections 2024-05-20T20:13:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.352+0000 7f292c664740 1 -- 172.21.15.28:0/3292341856 wait complete. 2024-05-20T20:13:34.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.352+0000 7f292c664740 20 client.27122 trim_cache size 0 max 16384 2024-05-20T20:13:34.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.379+0000 7f292c664740 1 Processor -- start 2024-05-20T20:13:34.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.379+0000 7f292c664740 1 -- start start 2024-05-20T20:13:34.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.380+0000 7f292c664740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1dcc90 0x55786a1f3be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.380+0000 7f292c664740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dd5b0 0x55786a1d8190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.381+0000 7f2922a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dd5b0 0x55786a1d8190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.381+0000 7f2922a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dd5b0 0x55786a1d8190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:37322/0 (socket says 172.21.15.28:37322) 2024-05-20T20:13:34.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.381+0000 7f2922a00640 1 -- 172.21.15.28:0/3987130671 learned_addr learned my addr 172.21.15.28:0/3987130671 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:34.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.381+0000 7f292c664740 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1e7e70 0x55786a1ff6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.382+0000 7f2922000640 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1e7e70 0x55786a1ff6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.382+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f290400e7e0 con 0x55786a1dd5b0 2024-05-20T20:13:34.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.382+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2904010320 con 0x55786a1e7e70 2024-05-20T20:13:34.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.382+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f290400eeb0 con 0x55786a1dcc90 2024-05-20T20:13:34.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.383+0000 7f2922000640 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1dcc90 0x55786a1f3be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.384+0000 7f2922a00640 1 -- 172.21.15.28:0/3987130671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1dcc90 msgr2=0x55786a1f3be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.384+0000 7f2922a00640 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1dcc90 0x55786a1f3be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.384+0000 7f2922a00640 1 -- 172.21.15.28:0/3987130671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1e7e70 msgr2=0x55786a1ff6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.384+0000 7f2922a00640 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1e7e70 0x55786a1ff6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.384+0000 7f2922a00640 1 -- 172.21.15.28:0/3987130671 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55786a1ea5d0 con 0x55786a1dd5b0 2024-05-20T20:13:34.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.385+0000 7f2922a00640 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dd5b0 0x55786a1d8190 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f2914000c00 tx=0x7f2914006ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bca89648cda724c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:34.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.385+0000 7f290f400640 1 -- 172.21.15.28:0/3987130671 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f291400fe50 con 0x55786a1dd5b0 2024-05-20T20:13:34.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.386+0000 7f290f400640 1 -- 172.21.15.28:0/3987130671 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f291401ba10 con 0x55786a1dd5b0 2024-05-20T20:13:34.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f290f400640 1 -- 172.21.15.28:0/3987130671 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f29140062a0 con 0x55786a1dd5b0 2024-05-20T20:13:34.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dd5b0 msgr2=0x55786a1d8190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f292c664740 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dd5b0 0x55786a1d8190 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f2914000c00 tx=0x7f2914006ba0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 shutdown_connections 2024-05-20T20:13:34.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f292c664740 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1e7e70 0x55786a1ff6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f292c664740 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dd5b0 0x55786a1d8190 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f292c664740 1 --2- 172.21.15.28:0/3987130671 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1dcc90 0x55786a1f3be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.387+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 >> 172.21.15.28:0/3987130671 conn(0x55786a1d91a0 msgr2=0x55786a1d7a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:34.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.391+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 shutdown_connections 2024-05-20T20:13:34.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.392+0000 7f292c664740 1 -- 172.21.15.28:0/3987130671 wait complete. 2024-05-20T20:13:34.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.393+0000 7f292c664740 1 -- start start 2024-05-20T20:13:34.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.393+0000 7f292c664740 1 Processor -- start 2024-05-20T20:13:34.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.399+0000 7f292c664740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1f0310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.399+0000 7f292c664740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1ed6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.399+0000 7f292c664740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1edc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.399+0000 7f292c664740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f290400fc50 con 0x55786a1dcc90 2024-05-20T20:13:34.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.399+0000 7f292c664740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f290400f580 con 0x55786a1dd5b0 2024-05-20T20:13:34.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.399+0000 7f292c664740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f290400c7e0 con 0x55786a1e7e70 2024-05-20T20:13:34.401 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.399+0000 7f292c664740 10 client.0 _ll_register_callbacks cb 0x7f291400a560 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:34.402 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1ed6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.402 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1ed6f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54958/0 (socket says 172.21.15.28:54958) 2024-05-20T20:13:34.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 -- 172.21.15.28:0/2207265434 learned_addr learned my addr 172.21.15.28:0/2207265434 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:34.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922000640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1edc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922000640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1f0310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 -- 172.21.15.28:0/2207265434 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 msgr2=0x55786a1edc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1edc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 -- 172.21.15.28:0/2207265434 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 msgr2=0x55786a1f0310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:34.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1f0310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.404 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.402+0000 7f2922a00640 1 -- 172.21.15.28:0/2207265434 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55786a1ee270 con 0x55786a1dd5b0 2024-05-20T20:13:34.404 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.403+0000 7f2922000640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1f0310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.403+0000 7f2922a00640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1ed6f0 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x55786a1f4620 tx=0x7f291400ff00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f62e48945888c715 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.403+0000 7f290cc00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.403+0000 7f290cc00640 1 -- 172.21.15.28:0/2207265434 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2914013850 con 0x55786a1dd5b0 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.404+0000 7f290cc00640 1 -- 172.21.15.28:0/2207265434 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f2914030430 con 0x55786a1dd5b0 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.404+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f291401ba10 con 0x55786a1dd5b0 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.404+0000 7f290cc00640 1 -- 172.21.15.28:0/2207265434 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f291403f050 con 0x55786a1dd5b0 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.404+0000 7f292c664740 10 client.27128 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:34.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.404+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55786a262410 con 0x55786a1dd5b0 2024-05-20T20:13:34.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f290cc00640 1 -- 172.21.15.28:0/2207265434 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f291401ba10 con 0x55786a1dd5b0 2024-05-20T20:13:34.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f290cc00640 1 -- 172.21.15.28:0/2207265434 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f29140254b0 con 0x55786a1dd5b0 2024-05-20T20:13:34.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f292c664740 10 client.27128 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:34.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f292c664740 10 client.27128 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:34.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f292c664740 20 client.27128 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:34.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f292c664740 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a262a70 0x55786a264e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:34.407 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f292c664740 4 client.27128 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:13:34.407 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.405+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55786a1492c0 con 0x55786a262a70 2024-05-20T20:13:34.407 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.406+0000 7f2922000640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a262a70 0x55786a264e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:34.407 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.407+0000 7f290cc00640 10 client.27128 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:34.407 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.407+0000 7f2922000640 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a262a70 0x55786a264e90 crc :-1 s=READY pgs=182 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7044f30f3c50aabd server_cookie=6dda6d478710e618 in_seq=0 out_seq=0 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.446+0000 7f290cc00640 1 -- 172.21.15.28:0/2207265434 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55786a1492c0 con 0x55786a262a70 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.446+0000 7f290cc00640 10 client.27128 handle_command_reply: tid=0 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.446+0000 7f292c664740 1 client.27128 shutdown 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 msgr2=0x55786a1ed6f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1ed6f0 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x55786a1f4620 tx=0x7f291400ff00 comp rx=0 tx=0).stop 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 shutdown_connections 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55786a262a70 0x55786a264e90 crc :-1 s=READY pgs=182 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55786a1dd5b0 0x55786a1ed6f0 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55786a1e7e70 0x55786a1edc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 --2- 172.21.15.28:0/2207265434 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55786a1dcc90 0x55786a1f0310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 >> 172.21.15.28:0/2207265434 conn(0x55786a1d91a0 msgr2=0x55786a1d8310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f290cc00640 0 client.27128 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 shutdown_connections 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 1 -- 172.21.15.28:0/2207265434 wait complete. 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.447+0000 7f292c664740 20 client.27128 trim_cache size 0 max 16384 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:13:34.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:13:34.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:13:34.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.448+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f29080b2ba0 msgr2=0x7f29080b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.448+0000 7f2929800640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f29080b2ba0 0x7f29080b5060 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f291c0c6e40 tx=0x7f291c0098d0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.448+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29241134c0 msgr2=0x7f2924117810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:34.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.448+0000 7f2929800640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29241134c0 0x7f2924117810 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f291c031a70 tx=0x7f291c003970 comp rx=0 tx=0).stop 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 shutdown_connections 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f29080b2ba0 0x7f29080b5060 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f29241134c0 0x7f2924117810 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f292410de30 0x7f29241172b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 --2- 172.21.15.28:0/3909608214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f292410d470 0x7f292411a1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 >> 172.21.15.28:0/3909608214 conn(0x7f292406dcf0 msgr2=0x7f29241103a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 shutdown_connections 2024-05-20T20:13:34.451 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:34.449+0000 7f2929800640 1 -- 172.21.15.28:0/3909608214 wait complete. 2024-05-20T20:13:35.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:35 smithi028 ceph-mon[31178]: pgmap v245: 97 pgs: 97 active+clean; 47 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 135 MiB/s wr, 1.28k op/s 2024-05-20T20:13:35.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:35 smithi154 ceph-mon[36314]: pgmap v245: 97 pgs: 97 active+clean; 47 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 135 MiB/s wr, 1.28k op/s 2024-05-20T20:13:35.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:35 smithi033 ceph-mon[36179]: pgmap v245: 97 pgs: 97 active+clean; 47 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 135 MiB/s wr, 1.28k op/s 2024-05-20T20:13:35.908 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:37.308 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:37 smithi028 ceph-mon[31178]: pgmap v246: 97 pgs: 97 active+clean; 47 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 KiB/s rd, 115 MiB/s wr, 988 op/s 2024-05-20T20:13:37.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:37 smithi154 ceph-mon[36314]: pgmap v246: 97 pgs: 97 active+clean; 47 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 KiB/s rd, 115 MiB/s wr, 988 op/s 2024-05-20T20:13:37.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:37 smithi033 ceph-mon[36179]: pgmap v246: 97 pgs: 97 active+clean; 47 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 KiB/s rd, 115 MiB/s wr, 988 op/s 2024-05-20T20:13:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:37.990+0000 7fe4e4000700 1 -- 172.21.15.28:0/2986817178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe4dc10eeb0 msgr2=0x7fe4dc116a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:37.990+0000 7fe4e4000700 1 --2- 172.21.15.28:0/2986817178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe4dc10eeb0 0x7fe4dc116a80 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7fe4d4009fc0 tx=0x7fe4d400b330 comp rx=0 tx=0).stop 2024-05-20T20:13:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:37.991+0000 7fe4e4000700 1 -- 172.21.15.28:0/2986817178 shutdown_connections 2024-05-20T20:13:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:37.991+0000 7fe4e4000700 1 --2- 172.21.15.28:0/2986817178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe4dc10eeb0 0x7fe4dc116a80 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:37.991+0000 7fe4e4000700 1 --2- 172.21.15.28:0/2986817178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe4dc109dd0 0x7fe4dc10a1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.111 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:37.991+0000 7fe4e4000700 1 --2- 172.21.15.28:0/2986817178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc10a780 0x7fe4dc10e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:37.991+0000 7fe4e4000700 1 -- 172.21.15.28:0/2986817178 >> 172.21.15.28:0/2986817178 conn(0x7fe4dc076d10 msgr2=0x7fe4dc108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.004+0000 7fe4e4000700 1 -- 172.21.15.28:0/2986817178 shutdown_connections 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.004+0000 7fe4e4000700 1 -- 172.21.15.28:0/2986817178 wait complete. 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.006+0000 7fe4e4000700 1 Processor -- start 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.007+0000 7fe4e4000700 1 -- start start 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.012+0000 7fe4e4000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc109dd0 0x7fe4dc083ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.012+0000 7fe4e4000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe4dc10a780 0x7fe4dc084020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.013+0000 7fe4e4000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe4dc10eeb0 0x7fe4dc07cdc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.015+0000 7fe4e1600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc109dd0 0x7fe4dc083ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.015+0000 7fe4e1600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc109dd0 0x7fe4dc083ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37680/0 (socket says 172.21.15.28:37680) 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.018+0000 7fe4e1600700 1 -- 172.21.15.28:0/1736651493 learned_addr learned my addr 172.21.15.28:0/1736651493 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.018+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe4dc119d90 con 0x7fe4dc10eeb0 2024-05-20T20:13:38.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.018+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe4dc119c10 con 0x7fe4dc109dd0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.018+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe4dc119f10 con 0x7fe4dc10a780 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.019+0000 7fe4e1600700 1 -- 172.21.15.28:0/1736651493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe4dc10a780 msgr2=0x7fe4dc084020 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.019+0000 7fe4e1600700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe4dc10a780 0x7fe4dc084020 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.019+0000 7fe4e1600700 1 -- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe4dc10eeb0 msgr2=0x7fe4dc07cdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.019+0000 7fe4e1600700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe4dc10eeb0 0x7fe4dc07cdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.019+0000 7fe4e1600700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe4dc07d680 con 0x7fe4dc109dd0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.020+0000 7fe4e1600700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc109dd0 0x7fe4dc083ae0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fe4d000ec10 tx=0x7fe4d000c2d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.020+0000 7fe4da000700 1 -- 172.21.15.28:0/1736651493 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe4d000cde0 con 0x7fe4dc109dd0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.020+0000 7fe4da000700 1 -- 172.21.15.28:0/1736651493 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe4d00044d0 con 0x7fe4dc109dd0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.020+0000 7fe4da000700 1 -- 172.21.15.28:0/1736651493 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe4d0010690 con 0x7fe4dc109dd0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.021+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe4dc07d970 con 0x7fe4dc109dd0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.021+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe4dc0818a0 con 0x7fe4dc109dd0 2024-05-20T20:13:38.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.024+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe4dc10b2b0 con 0x7fe4dc109dd0 2024-05-20T20:13:38.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.038+0000 7fe4da000700 1 -- 172.21.15.28:0/1736651493 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fe4d0003f00 con 0x7fe4dc109dd0 2024-05-20T20:13:38.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.041+0000 7fe4da000700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe4c40b2bc0 0x7fe4c40b5080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:38.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.041+0000 7fe4da000700 1 -- 172.21.15.28:0/1736651493 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fe4d00e2a00 con 0x7fe4dc109dd0 2024-05-20T20:13:38.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.042+0000 7fe4e0c00700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe4c40b2bc0 0x7fe4c40b5080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:38.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.044+0000 7fe4e0c00700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe4c40b2bc0 0x7fe4c40b5080 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fe4dc07e330 tx=0x7fe4c8009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:38.114 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.045+0000 7fe4da000700 1 -- 172.21.15.28:0/1736651493 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fe4d00e2e20 con 0x7fe4dc109dd0 2024-05-20T20:13:38.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.616+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fe4dc07e0a0 con 0x7fe4dc109dd0 2024-05-20T20:13:38.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.618+0000 7fe4da000700 1 -- 172.21.15.28:0/1736651493 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fe4dc07e0a0 con 0x7fe4dc109dd0 2024-05-20T20:13:38.620 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:13:38.620 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:13:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe4c40b2bc0 msgr2=0x7fe4c40b5080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe4c40b2bc0 0x7fe4c40b5080 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fe4dc07e330 tx=0x7fe4c8009260 comp rx=0 tx=0).stop 2024-05-20T20:13:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc109dd0 msgr2=0x7fe4dc083ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc109dd0 0x7fe4dc083ae0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fe4d000ec10 tx=0x7fe4d000c2d0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 shutdown_connections 2024-05-20T20:13:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe4dc10eeb0 0x7fe4dc07cdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe4c40b2bc0 0x7fe4c40b5080 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe4dc109dd0 0x7fe4dc083ae0 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 --2- 172.21.15.28:0/1736651493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe4dc10a780 0x7fe4dc084020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.623+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 >> 172.21.15.28:0/1736651493 conn(0x7fe4dc076d10 msgr2=0x7fe4dc0772d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.624+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 shutdown_connections 2024-05-20T20:13:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:38.624+0000 7fe4e4000700 1 -- 172.21.15.28:0/1736651493 wait complete. 2024-05-20T20:13:38.627 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:13:39.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:39 smithi028 ceph-mon[31178]: pgmap v247: 97 pgs: 97 active+clean; 48 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 KiB/s rd, 135 MiB/s wr, 1.12k op/s 2024-05-20T20:13:39.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:39 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1736651493' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:39.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:39 smithi154 ceph-mon[36314]: pgmap v247: 97 pgs: 97 active+clean; 48 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 KiB/s rd, 135 MiB/s wr, 1.12k op/s 2024-05-20T20:13:39.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:39 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1736651493' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:39.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:39 smithi033 ceph-mon[36179]: pgmap v247: 97 pgs: 97 active+clean; 48 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 6.8 KiB/s rd, 135 MiB/s wr, 1.12k op/s 2024-05-20T20:13:39.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:39 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1736651493' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:40.941 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 143.3 --await-for 45.0' 2024-05-20T20:13:40.941 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 143.3 --await-for 45.0 2024-05-20T20:13:41.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:41 smithi154 ceph-mon[36314]: pgmap v248: 97 pgs: 97 active+clean; 49 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 226 MiB/s wr, 1.70k op/s 2024-05-20T20:13:41.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:41 smithi033 ceph-mon[36179]: pgmap v248: 97 pgs: 97 active+clean; 49 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 226 MiB/s wr, 1.70k op/s 2024-05-20T20:13:41.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:41 smithi028 ceph-mon[31178]: pgmap v248: 97 pgs: 97 active+clean; 49 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 226 MiB/s wr, 1.70k op/s 2024-05-20T20:13:41.901 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:43.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.160+0000 7f8fee800700 1 -- 172.21.15.28:0/16579129 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 msgr2=0x7f8fe81028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.160+0000 7f8fee800700 1 --2- 172.21.15.28:0/16579129 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe81028c0 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7f8fd800fb60 tx=0x7f8fd800fe70 comp rx=0 tx=0).stop 2024-05-20T20:13:43.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.161+0000 7f8fee800700 1 -- 172.21.15.28:0/16579129 shutdown_connections 2024-05-20T20:13:43.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.161+0000 7f8fee800700 1 --2- 172.21.15.28:0/16579129 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fe8102e00 0x7f8fe810f2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.161+0000 7f8fee800700 1 --2- 172.21.15.28:0/16579129 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fe8108520 0x7f8fe8108900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.161+0000 7f8fee800700 1 --2- 172.21.15.28:0/16579129 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe81028c0 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.161+0000 7f8fee800700 1 -- 172.21.15.28:0/16579129 >> 172.21.15.28:0/16579129 conn(0x7f8fe80793f0 msgr2=0x7f8fe8079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:43.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.172+0000 7f8fee800700 1 -- 172.21.15.28:0/16579129 shutdown_connections 2024-05-20T20:13:43.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.172+0000 7f8fee800700 1 -- 172.21.15.28:0/16579129 wait complete. 2024-05-20T20:13:43.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.174+0000 7f8fee800700 1 Processor -- start 2024-05-20T20:13:43.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.175+0000 7f8fee800700 1 -- start start 2024-05-20T20:13:43.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.181+0000 7f8fee800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe819d620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.181+0000 7f8fee800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fe8102e00 0x7f8fe819db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.181+0000 7f8fee800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fe8108520 0x7f8fe81976f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.181+0000 7f8fee800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe81115d0 con 0x7f8fe8108520 2024-05-20T20:13:43.182 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.181+0000 7f8fee800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe8111450 con 0x7f8fe8102440 2024-05-20T20:13:43.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.181+0000 7f8fee800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe8111750 con 0x7f8fe8102e00 2024-05-20T20:13:43.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.181+0000 7f8fe6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fe8102e00 0x7f8fe819db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.182+0000 7f8fe7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe819d620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.182+0000 7f8fe7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe819d620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37692/0 (socket says 172.21.15.28:37692) 2024-05-20T20:13:43.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.182+0000 7f8fe7400700 1 -- 172.21.15.28:0/2763572734 learned_addr learned my addr 172.21.15.28:0/2763572734 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:43.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.190+0000 7f8fe7400700 1 -- 172.21.15.28:0/2763572734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fe8102e00 msgr2=0x7f8fe819db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.190+0000 7f8fe7400700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fe8102e00 0x7f8fe819db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.190+0000 7f8fe7400700 1 -- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fe8108520 msgr2=0x7f8fe81976f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.190+0000 7f8fe7400700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fe8108520 0x7f8fe81976f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.190+0000 7f8fe7400700 1 -- 172.21.15.28:0/2763572734 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8fe8197fd0 con 0x7f8fe8102440 2024-05-20T20:13:43.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.191+0000 7f8fe7400700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe819d620 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f8fd4007a00 tx=0x7f8fd4007dc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:43.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.191+0000 7f8fdfe00700 1 -- 172.21.15.28:0/2763572734 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd4004020 con 0x7f8fe8102440 2024-05-20T20:13:43.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.191+0000 7f8fdfe00700 1 -- 172.21.15.28:0/2763572734 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fd400a950 con 0x7f8fe8102440 2024-05-20T20:13:43.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.191+0000 7f8fdfe00700 1 -- 172.21.15.28:0/2763572734 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd4003c40 con 0x7f8fe8102440 2024-05-20T20:13:43.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.193+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8fe8198260 con 0x7f8fe8102440 2024-05-20T20:13:43.196 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.194+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8fe81a43d0 con 0x7f8fe8102440 2024-05-20T20:13:43.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.209+0000 7f8ff9167b80 1 Processor -- start 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.210+0000 7f8fdfe00700 1 -- 172.21.15.28:0/2763572734 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f8fd401b060 con 0x7f8fe8102440 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.212+0000 7f8fdfe00700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fcc0b2b20 0x7f8fcc0b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.213+0000 7f8fdfe00700 1 -- 172.21.15.28:0/2763572734 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f8fd40e1610 con 0x7f8fe8102440 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.214+0000 7f8ff9167b80 1 -- start start 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.214+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c115cd0 0x55d61c1160b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.214+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11a3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.214+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c11a900 0x55d61c11ad40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.214+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d61c06bec0 con 0x55d61c115cd0 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.214+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d61c05ba60 con 0x55d61c11a900 2024-05-20T20:13:43.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.214+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d61c05faa0 con 0x55d61c1165f0 2024-05-20T20:13:43.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.216+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fcc0b2b20 0x7f8fcc0b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.217+0000 7f8fe6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11a3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.217+0000 7f8fe6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11a3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43796/0 (socket says 172.21.15.28:43796) 2024-05-20T20:13:43.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.217+0000 7f8fe6a00700 1 -- 172.21.15.28:0/78757658 learned_addr learned my addr 172.21.15.28:0/78757658 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:43.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.218+0000 7f8fe6a00700 1 -- 172.21.15.28:0/78757658 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c11a900 msgr2=0x55d61c11ad40 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:13:43.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.218+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/78757658 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c11a900 0x55d61c11ad40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.218+0000 7f8fe6a00700 1 -- 172.21.15.28:0/78757658 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c115cd0 msgr2=0x55d61c1160b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:13:43.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.218+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/78757658 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c115cd0 0x55d61c1160b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.218+0000 7f8fe6a00700 1 -- 172.21.15.28:0/78757658 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d61c0244a0 con 0x55d61c1165f0 2024-05-20T20:13:43.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.219+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/78757658 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11a3c0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f8fd800fa80 tx=0x7f8fd8009d20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4996fe7f09e1dfcb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:43.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.220+0000 7f8fdcc00700 1 -- 172.21.15.28:0/78757658 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd801c070 con 0x55d61c1165f0 2024-05-20T20:13:43.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.220+0000 7f8fdcc00700 1 -- 172.21.15.28:0/78757658 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fd800e660 con 0x55d61c1165f0 2024-05-20T20:13:43.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.221+0000 7f8fdcc00700 1 -- 172.21.15.28:0/78757658 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd801c070 con 0x55d61c1165f0 2024-05-20T20:13:43.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.223+0000 7f8ff9167b80 1 -- 172.21.15.28:0/78757658 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 msgr2=0x55d61c11a3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.223+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/78757658 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11a3c0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7f8fd800fa80 tx=0x7f8fd8009d20 comp rx=0 tx=0).stop 2024-05-20T20:13:43.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.223+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fcc0b2b20 0x7f8fcc0b4fe0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f8fe0009610 tx=0x7f8fe00096c0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:43.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.224+0000 7f8ff9167b80 1 -- 172.21.15.28:0/78757658 shutdown_connections 2024-05-20T20:13:43.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.224+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/78757658 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c11a900 0x55d61c11ad40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.224+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/78757658 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c115cd0 0x55d61c1160b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.224+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/78757658 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11a3c0 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.224+0000 7f8ff9167b80 1 -- 172.21.15.28:0/78757658 >> 172.21.15.28:0/78757658 conn(0x55d61c03bc60 msgr2=0x55d61c108090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:43.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.225+0000 7f8ff9167b80 1 -- 172.21.15.28:0/78757658 shutdown_connections 2024-05-20T20:13:43.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.226+0000 7f8ff9167b80 1 -- 172.21.15.28:0/78757658 wait complete. 2024-05-20T20:13:43.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.226+0000 7f8ff9167b80 1 -- start start 2024-05-20T20:13:43.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.227+0000 7f8ff9167b80 1 Processor -- start 2024-05-20T20:13:43.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.229+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c121de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.233+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11f100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.234+0000 7f8fe7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c121de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.234+0000 7f8fe7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c121de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37706/0 (socket says 172.21.15.28:37706) 2024-05-20T20:13:43.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.234+0000 7f8fe7400700 1 -- 172.21.15.28:0/1558100426 learned_addr learned my addr 172.21.15.28:0/1558100426 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.234+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c11f640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.234+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d61bf62020 con 0x55d61c11a900 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.235+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d61bffc7e0 con 0x55d61c115cd0 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.235+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d61c10d620 con 0x55d61c1165f0 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.235+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.236+0000 7f8fe7400700 1 -- 172.21.15.28:0/1558100426 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 msgr2=0x55d61c11f100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.236+0000 7f8fe7400700 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11f100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.236+0000 7f8fe7400700 1 -- 172.21.15.28:0/1558100426 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 msgr2=0x55d61c11f640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.236+0000 7f8fe7400700 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c11f640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.236+0000 7f8fe7400700 1 -- 172.21.15.28:0/1558100426 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d61bfab950 con 0x55d61c115cd0 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.241+0000 7f8ff9167b80 10 client.0 _ll_register_callbacks cb 0x55d61c005cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:43.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.241+0000 7f8fe7400700 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c121de0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f8fd40a4900 tx=0x7f8fd40df250 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f874e83424a1cdab server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:43.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.241+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8fd800e660 con 0x55d61c115cd0 2024-05-20T20:13:43.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.244+0000 7f8fd1600700 10 client.27143 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:43.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.244+0000 7f8fd1600700 1 -- 172.21.15.28:0/1558100426 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fd400ae00 con 0x55d61c115cd0 2024-05-20T20:13:43.244 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.244+0000 7f8fd1600700 1 -- 172.21.15.28:0/1558100426 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f8fd40b1460 con 0x55d61c115cd0 2024-05-20T20:13:43.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.250+0000 7f8fd1600700 1 -- 172.21.15.28:0/1558100426 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8fd40c4050 con 0x55d61c115cd0 2024-05-20T20:13:43.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.250+0000 7f8ff9167b80 10 client.27143 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:43.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.251+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x55d61c02c0e0 con 0x55d61c115cd0 2024-05-20T20:13:43.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.253+0000 7f8fd1600700 1 -- 172.21.15.28:0/1558100426 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8fd40a9bb0 con 0x55d61c115cd0 2024-05-20T20:13:43.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.253+0000 7f8ff9167b80 10 client.27143 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:43.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.253+0000 7f8ff9167b80 10 client.27143 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:43.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.253+0000 7f8ff9167b80 20 client.27143 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:43.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.253+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c137a10 0x55d61c139e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.253+0000 7f8ff9167b80 4 client.27143 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:43.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.253+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f8fd800e660 con 0x55d61c137a10 2024-05-20T20:13:43.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.254+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c137a10 0x55d61c139e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.258+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c137a10 0x55d61c139e00 crc :-1 s=READY pgs=183 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=59ce3f770019a01d server_cookie=d27bd04bb36440b7 in_seq=0 out_seq=0 2024-05-20T20:13:43.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.258+0000 7f8fd1600700 10 client.27143 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:43.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.259+0000 7f8fd1600700 1 -- 172.21.15.28:0/1558100426 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x7f8fd800e660 con 0x55d61c137a10 2024-05-20T20:13:43.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.259+0000 7f8fd1600700 10 client.27143 handle_command_reply: tid=0 2024-05-20T20:13:43.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.259+0000 7f8ff9167b80 1 client.27143 shutdown 2024-05-20T20:13:43.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 msgr2=0x55d61c121de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c121de0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f8fd40a4900 tx=0x7f8fd40df250 comp rx=0 tx=0).stop 2024-05-20T20:13:43.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 shutdown_connections 2024-05-20T20:13:43.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c137a10 0x55d61c139e00 crc :-1 s=READY pgs=183 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c11f640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c121de0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1558100426 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c11f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 >> 172.21.15.28:0/1558100426 conn(0x55d61c03bc60 msgr2=0x55d61c11b4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:43.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.260+0000 7f8fd1600700 0 client.27143 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:43.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.261+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 shutdown_connections 2024-05-20T20:13:43.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.261+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1558100426 wait complete. 2024-05-20T20:13:43.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.261+0000 7f8ff9167b80 20 client.27143 trim_cache size 0 max 16384 2024-05-20T20:13:43.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.296+0000 7f8ff9167b80 1 Processor -- start 2024-05-20T20:13:43.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.296+0000 7f8ff9167b80 1 -- start start 2024-05-20T20:13:43.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.297+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c11bcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.298+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c1165f0 0x55d61c11e2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:43 smithi028 ceph-mon[31178]: pgmap v249: 97 pgs: 97 active+clean; 49 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 226 MiB/s wr, 1.60k op/s 2024-05-20T20:13:43.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.298+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c11a900 0x55d61c13f5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.299+0000 7f8fe7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c11a900 0x55d61c13f5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.299+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe81aba70 con 0x55d61c1165f0 2024-05-20T20:13:43.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.299+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe81ad5b0 con 0x55d61c115cd0 2024-05-20T20:13:43.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.299+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe81ac140 con 0x55d61c11a900 2024-05-20T20:13:43.306 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.300+0000 7f8fe7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c11bcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.301+0000 7f8fe7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c11bcf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37712/0 (socket says 172.21.15.28:37712) 2024-05-20T20:13:43.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.301+0000 7f8fe7e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c11a900 0x55d61c13f5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43818/0 (socket says 172.21.15.28:43818) 2024-05-20T20:13:43.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.301+0000 7f8fe7e00700 1 -- 172.21.15.28:0/1860400724 learned_addr learned my addr 172.21.15.28:0/1860400724 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:43.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.301+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c1165f0 0x55d61c11e2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.304+0000 7f8fe7e00700 1 -- 172.21.15.28:0/1860400724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 msgr2=0x55d61c11bcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:43.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.304+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c11bcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.304+0000 7f8fe7e00700 1 -- 172.21.15.28:0/1860400724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c1165f0 msgr2=0x55d61c11e2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:43.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.304+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c1165f0 0x55d61c11e2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.308 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.304+0000 7f8fe7e00700 1 -- 172.21.15.28:0/1860400724 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d61c11d690 con 0x55d61c11a900 2024-05-20T20:13:43.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.309+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c11a900 0x55d61c13f5c0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f8fe0007df0 tx=0x7f8fe000b050 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c285368e656fc710 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:43.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.309+0000 7f8fd3e00700 1 -- 172.21.15.28:0/1860400724 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fe0007e20 con 0x55d61c11a900 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.311+0000 7f8fd3e00700 1 -- 172.21.15.28:0/1860400724 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fe0017c70 con 0x55d61c11a900 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.311+0000 7f8fd3e00700 1 -- 172.21.15.28:0/1860400724 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fe00207f0 con 0x55d61c11a900 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.312+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1860400724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c11a900 msgr2=0x55d61c13f5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.312+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c11a900 0x55d61c13f5c0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7f8fe0007df0 tx=0x7f8fe000b050 comp rx=0 tx=0).stop 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.313+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1860400724 shutdown_connections 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.313+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c11a900 0x55d61c13f5c0 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.313+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c11bcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.313+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/1860400724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c1165f0 0x55d61c11e2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.313+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1860400724 >> 172.21.15.28:0/1860400724 conn(0x55d61c03bc60 msgr2=0x55d61c11cb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:43.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.313+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1860400724 shutdown_connections 2024-05-20T20:13:43.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.313+0000 7f8ff9167b80 1 -- 172.21.15.28:0/1860400724 wait complete. 2024-05-20T20:13:43.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.314+0000 7f8ff9167b80 1 -- start start 2024-05-20T20:13:43.315 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.315+0000 7f8ff9167b80 1 Processor -- start 2024-05-20T20:13:43.317 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.317+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c111550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.318+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c111a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.318+0000 7f8ff9167b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c19c4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.319+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe81acee0 con 0x55d61c11a900 2024-05-20T20:13:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.319+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe81ac810 con 0x55d61c115cd0 2024-05-20T20:13:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.319+0000 7f8ff9167b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8fe81a9a30 con 0x55d61c1165f0 2024-05-20T20:13:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.319+0000 7f8ff9167b80 10 client.0 _ll_register_callbacks cb 0x55d61c005cf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.319+0000 7f8fe7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c19c4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.320+0000 7f8fe7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c19c4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40444/0 (socket says 172.21.15.28:40444) 2024-05-20T20:13:43.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.320+0000 7f8fe7e00700 1 -- 172.21.15.28:0/2535439095 learned_addr learned my addr 172.21.15.28:0/2535439095 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:43.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.321+0000 7f8fe6a00700 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c111a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.321+0000 7f8fe7e00700 1 -- 172.21.15.28:0/2535439095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 msgr2=0x55d61c111a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:43.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.321+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c111a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.321+0000 7f8fe7e00700 1 -- 172.21.15.28:0/2535439095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 msgr2=0x55d61c111550 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:13:43.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.321+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c111550 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.321+0000 7f8fe7e00700 1 -- 172.21.15.28:0/2535439095 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d61c19cad0 con 0x55d61c11a900 2024-05-20T20:13:43.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.322+0000 7f8fd1600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:13:43.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.322+0000 7f8fe7e00700 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c19c4e0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f8fe0017710 tx=0x7f8fe0017320 comp rx=0 tx=0).ready entity=mon.0 client_cookie=66a3e305888d219e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:43.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.324+0000 7f8fd1600700 1 -- 172.21.15.28:0/2535439095 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8fe0017e20 con 0x55d61c11a900 2024-05-20T20:13:43.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.324+0000 7f8fd1600700 1 -- 172.21.15.28:0/2535439095 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f8fe001d040 con 0x55d61c11a900 2024-05-20T20:13:43.328 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.328+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8fe0017c70 con 0x55d61c11a900 2024-05-20T20:13:43.329 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.329+0000 7f8fd1600700 1 -- 172.21.15.28:0/2535439095 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8fe0037050 con 0x55d61c11a900 2024-05-20T20:13:43.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.341+0000 7f8ff9167b80 10 client.16779 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:43.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.341+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d61c1a3eb0 con 0x55d61c11a900 2024-05-20T20:13:43.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.342+0000 7f8fd1600700 1 -- 172.21.15.28:0/2535439095 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8fe0017c70 con 0x55d61c11a900 2024-05-20T20:13:43.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.343+0000 7f8fd1600700 1 -- 172.21.15.28:0/2535439095 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8fe0015570 con 0x55d61c11a900 2024-05-20T20:13:43.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.344+0000 7f8ff9167b80 10 client.16779 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:43.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.344+0000 7f8ff9167b80 10 client.16779 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:43.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.344+0000 7f8ff9167b80 20 client.16779 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:43.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.344+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c1a4400 0x55d61c1a67f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:43.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.344+0000 7f8ff9167b80 4 client.16779 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:13:43.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.344+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x55d61c06de10 con 0x55d61c1a4400 2024-05-20T20:13:43.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.348+0000 7f8fe7400700 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c1a4400 0x55d61c1a67f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:43.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.349+0000 7f8fe7400700 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c1a4400 0x55d61c1a67f0 crc :-1 s=READY pgs=184 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a8d9b23e3525fa4f server_cookie=a80ae148a46ff204 in_seq=0 out_seq=0 2024-05-20T20:13:43.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.350+0000 7f8fd1600700 10 client.16779 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:43.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.351+0000 7f8fd1600700 1 -- 172.21.15.28:0/2535439095 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x55d61c06de10 con 0x55d61c1a4400 2024-05-20T20:13:43.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.351+0000 7f8fd1600700 10 client.16779 handle_command_reply: tid=0 2024-05-20T20:13:43.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.351+0000 7f8ff9167b80 1 client.16779 shutdown 2024-05-20T20:13:43.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 msgr2=0x55d61c19c4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c19c4e0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f8fe0017710 tx=0x7f8fe0017320 comp rx=0 tx=0).stop 2024-05-20T20:13:43.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 shutdown_connections 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d61c1a4400 0x55d61c1a67f0 crc :-1 s=READY pgs=184 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d61c11a900 0x55d61c19c4e0 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d61c115cd0 0x55d61c111550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 --2- 172.21.15.28:0/2535439095 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d61c1165f0 0x55d61c111a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8fd1600700 0 client.16779 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.355+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 >> 172.21.15.28:0/2535439095 conn(0x55d61c03bc60 msgr2=0x55d61c119760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.358+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 shutdown_connections 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.358+0000 7f8ff9167b80 1 -- 172.21.15.28:0/2535439095 wait complete. 2024-05-20T20:13:43.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.359+0000 7f8ff9167b80 20 client.16779 trim_cache size 0 max 16384 2024-05-20T20:13:43.360 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:13:43.360 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:13:43.360 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "4fe8c40d-f4c6-4d18-8f0d-5dce6f312fd1", 2024-05-20T20:13:43.360 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:13:43.360 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fcc0b2b20 msgr2=0x7f8fcc0b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fcc0b2b20 0x7f8fcc0b4fe0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f8fe0009610 tx=0x7f8fe00096c0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 msgr2=0x7f8fe819d620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe819d620 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f8fd4007a00 tx=0x7f8fd4007dc0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 shutdown_connections 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8fe8108520 0x7f8fe81976f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8fcc0b2b20 0x7f8fcc0b4fe0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8fe8102440 0x7f8fe819d620 unknown :-1 s=CLOSED pgs=454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 --2- 172.21.15.28:0/2763572734 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8fe8102e00 0x7f8fe819db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.360+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 >> 172.21.15.28:0/2763572734 conn(0x7f8fe80793f0 msgr2=0x7f8fe81003b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:43.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.361+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 shutdown_connections 2024-05-20T20:13:43.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:43.361+0000 7f8fee800700 1 -- 172.21.15.28:0/2763572734 wait complete. 2024-05-20T20:13:43.374 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:43 smithi154 ceph-mon[36314]: pgmap v249: 97 pgs: 97 active+clean; 49 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 226 MiB/s wr, 1.60k op/s 2024-05-20T20:13:43.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:43 smithi033 ceph-mon[36179]: pgmap v249: 97 pgs: 97 active+clean; 49 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 226 MiB/s wr, 1.60k op/s 2024-05-20T20:13:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:45 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:13:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:45 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:13:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:45 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:13:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:45 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:13:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:45 smithi154 ceph-mon[36314]: pgmap v250: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 317 MiB/s wr, 2.04k op/s 2024-05-20T20:13:45.376 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:45 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:13:45.376 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:45 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:13:45.376 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:45 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:13:45.376 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:45 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:13:45.376 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:45 smithi028 ceph-mon[31178]: pgmap v250: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 317 MiB/s wr, 2.04k op/s 2024-05-20T20:13:45.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:45 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:13:45.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:45 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:13:45.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:45 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:13:45.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:45 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:13:45.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:45 smithi033 ceph-mon[36179]: pgmap v250: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 317 MiB/s wr, 2.04k op/s 2024-05-20T20:13:46.279 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:13:47.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:47 smithi154 ceph-mon[36314]: pgmap v251: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 237 MiB/s wr, 1.38k op/s 2024-05-20T20:13:47.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:47 smithi033 ceph-mon[36179]: pgmap v251: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 237 MiB/s wr, 1.38k op/s 2024-05-20T20:13:47.440 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:47 smithi028 ceph-mon[31178]: pgmap v251: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 237 MiB/s wr, 1.38k op/s 2024-05-20T20:13:47.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 -- 172.21.15.28:0/1610441483 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0102a60 msgr2=0x7fa6e0102ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 --2- 172.21.15.28:0/1610441483 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0102a60 0x7fa6e0102ee0 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fa6cc009860 tx=0x7fa6cc02a340 comp rx=0 tx=0).stop 2024-05-20T20:13:47.658 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 -- 172.21.15.28:0/1610441483 shutdown_connections 2024-05-20T20:13:47.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 --2- 172.21.15.28:0/1610441483 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0102a60 0x7fa6e0102ee0 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 --2- 172.21.15.28:0/1610441483 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e0103420 0x7fa6e010f930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 --2- 172.21.15.28:0/1610441483 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0108a70 0x7fa6e0108e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 -- 172.21.15.28:0/1610441483 >> 172.21.15.28:0/1610441483 conn(0x7fa6e0079ee0 msgr2=0x7fa6e007a2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 -- 172.21.15.28:0/1610441483 shutdown_connections 2024-05-20T20:13:47.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.658+0000 7fa6e7800640 1 -- 172.21.15.28:0/1610441483 wait complete. 2024-05-20T20:13:47.659 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.659+0000 7fa6e7800640 1 Processor -- start 2024-05-20T20:13:47.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.660+0000 7fa6e7800640 1 -- start start 2024-05-20T20:13:47.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.660+0000 7fa6e7800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0102a60 0x7fa6e019b370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.660+0000 7fa6e7800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e0103420 0x7fa6e019b8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.660 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.660+0000 7fa6e7800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0108a70 0x7fa6e019fc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.660+0000 7fa6e7800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e0112d00 con 0x7fa6e0103420 2024-05-20T20:13:47.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.660+0000 7fa6e7800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e0112b80 con 0x7fa6e0108a70 2024-05-20T20:13:47.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.660+0000 7fa6e7800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e0112e80 con 0x7fa6e0102a60 2024-05-20T20:13:47.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.661+0000 7fa6e4e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0102a60 0x7fa6e019b370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.661+0000 7fa6e4e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0102a60 0x7fa6e019b370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59512/0 (socket says 172.21.15.28:59512) 2024-05-20T20:13:47.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.661+0000 7fa6e4e00640 1 -- 172.21.15.28:0/2319757660 learned_addr learned my addr 172.21.15.28:0/2319757660 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.661 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.661+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e0103420 0x7fa6e019b8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.661+0000 7fa6e5800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0108a70 0x7fa6e019fc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.661+0000 7fa6e4e00640 1 -- 172.21.15.28:0/2319757660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0108a70 msgr2=0x7fa6e019fc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6e4e00640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0108a70 0x7fa6e019fc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6e4e00640 1 -- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e0103420 msgr2=0x7fa6e019b8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6e4e00640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e0103420 0x7fa6e019b8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6e4e00640 1 -- 172.21.15.28:0/2319757660 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa6e01a0360 con 0x7fa6e0102a60 2024-05-20T20:13:47.662 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e0103420 0x7fa6e019b8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:47.663 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6e5800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0108a70 0x7fa6e019fc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:47.663 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6e4e00640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0102a60 0x7fa6e019b370 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fa6d400e9c0 tx=0x7fa6d400ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.663 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.663+0000 7fa6dd600640 1 -- 172.21.15.28:0/2319757660 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d400cde0 con 0x7fa6e0102a60 2024-05-20T20:13:47.663 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.664+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa6e01a0650 con 0x7fa6e0102a60 2024-05-20T20:13:47.664 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.664+0000 7fa6dd600640 1 -- 172.21.15.28:0/2319757660 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6d40035f0 con 0x7fa6e0102a60 2024-05-20T20:13:47.664 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.664+0000 7fa6dd600640 1 -- 172.21.15.28:0/2319757660 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d4010690 con 0x7fa6e0102a60 2024-05-20T20:13:47.664 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.664+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa6e01a7ee0 con 0x7fa6e0102a60 2024-05-20T20:13:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.668+0000 7fa6ea7cb740 1 Processor -- start 2024-05-20T20:13:47.668 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.669+0000 7fa6ea7cb740 1 -- start start 2024-05-20T20:13:47.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.669+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b834a0 0x561134b83880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.669+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b8e140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.669+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b8e680 0x561134b90a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.669+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561134aebb90 con 0x561134b8e680 2024-05-20T20:13:47.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.669+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561134adb730 con 0x561134b83dc0 2024-05-20T20:13:47.669 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.669+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561134adf770 con 0x561134b834a0 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6e5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b834a0 0x561134b83880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6e5800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b834a0 0x561134b83880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59516/0 (socket says 172.21.15.28:59516) 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6e5800640 1 -- 172.21.15.28:0/3521528946 learned_addr learned my addr 172.21.15.28:0/3521528946 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6dd600640 1 -- 172.21.15.28:0/2319757660 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa6d4004000 con 0x7fa6e0102a60 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6e4e00640 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b8e140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6e5800640 1 -- 172.21.15.28:0/3521528946 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 msgr2=0x561134b8e140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6e5800640 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b8e140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.670 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.670+0000 7fa6e5800640 1 -- 172.21.15.28:0/3521528946 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b8e680 msgr2=0x561134b90a70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:13:47.671 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.672+0000 7fa6dd600640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6c80b2b10 0x7fa6c80b4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.671 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.672+0000 7fa6dd600640 1 -- 172.21.15.28:0/2319757660 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fa6d40e29d0 con 0x7fa6e0102a60 2024-05-20T20:13:47.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.672+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b8e680 0x561134b90a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.672 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.672+0000 7fa6e5800640 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b8e680 0x561134b90a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.672+0000 7fa6e5800640 1 -- 172.21.15.28:0/3521528946 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561134a748b0 con 0x561134b834a0 2024-05-20T20:13:47.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.672+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b8e680 0x561134b90a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:47.673 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.673+0000 7fa6e5800640 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b834a0 0x561134b83880 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8009f80 tx=0x7fa6d800f7f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=84a268d472995e85 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.673+0000 7fa6d2000640 1 -- 172.21.15.28:0/3521528946 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d8003b70 con 0x561134b834a0 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.673+0000 7fa6d2000640 1 -- 172.21.15.28:0/3521528946 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6d8003d10 con 0x561134b834a0 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.673+0000 7fa6d2000640 1 -- 172.21.15.28:0/3521528946 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d8011980 con 0x561134b834a0 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.674+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3521528946 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b834a0 msgr2=0x561134b83880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.674+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b834a0 0x561134b83880 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8009f80 tx=0x7fa6d800f7f0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.675+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3521528946 shutdown_connections 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.675+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b8e140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.675+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b8e680 0x561134b90a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.675 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.675+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3521528946 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b834a0 0x561134b83880 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.675+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3521528946 >> 172.21.15.28:0/3521528946 conn(0x561134b7f9b0 msgr2=0x561134b7fd90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.676+0000 7fa6e4e00640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6c80b2b10 0x7fa6c80b4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.677+0000 7fa6e4e00640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6c80b2b10 0x7fa6c80b4fd0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x561134b84a40 tx=0x7fa6d4005c80 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.677 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.677+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3521528946 shutdown_connections 2024-05-20T20:13:47.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.678+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3521528946 wait complete. 2024-05-20T20:13:47.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.678+0000 7fa6ea7cb740 1 -- start start 2024-05-20T20:13:47.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.679+0000 7fa6ea7cb740 1 Processor -- start 2024-05-20T20:13:47.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.680+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b8e0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.680+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b83dc0 0x561134b8b390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.680+0000 7fa6e5800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b8e0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.681+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b8e680 0x561134b8b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.681+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5611347c1790 con 0x561134b834a0 2024-05-20T20:13:47.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.681+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561134a83bc0 con 0x561134b8e680 2024-05-20T20:13:47.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.681+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561134b912b0 con 0x561134b83dc0 2024-05-20T20:13:47.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.681+0000 7fa6ea7cb740 10 client.0 _ll_register_callbacks cb 0x561134a61400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:47.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.682+0000 7fa6e5800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b8e0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42280/0 (socket says 172.21.15.28:42280) 2024-05-20T20:13:47.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.682+0000 7fa6e5800640 1 -- 172.21.15.28:0/3853019245 learned_addr learned my addr 172.21.15.28:0/3853019245 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.693 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.691+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b83dc0 0x561134b8b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.693 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.692+0000 7fa6e5800640 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b8e680 0x561134b8b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.692+0000 7fa6e5800640 1 -- 172.21.15.28:0/3853019245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b83dc0 msgr2=0x561134b8b390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.692+0000 7fa6e5800640 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b83dc0 0x561134b8b390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.692+0000 7fa6e5800640 1 -- 172.21.15.28:0/3853019245 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 msgr2=0x561134b8e0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.692+0000 7fa6e5800640 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b8e0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.692+0000 7fa6e5800640 1 -- 172.21.15.28:0/3853019245 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5611347f0210 con 0x561134b8e680 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.693+0000 7fa6e5800640 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b8e680 0x561134b8b9a0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fa6d80049d0 tx=0x7fa6d8005fd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=36e234449e22e4e5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.693+0000 7fa6c6a00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.693+0000 7fa6c6a00640 1 -- 172.21.15.28:0/3853019245 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d8025070 con 0x561134b8e680 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.693+0000 7fa6c6a00640 1 -- 172.21.15.28:0/3853019245 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fa6d8004500 con 0x561134b8e680 2024-05-20T20:13:47.694 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.694+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa6d8003d10 con 0x561134b8e680 2024-05-20T20:13:47.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.694+0000 7fa6c6a00640 1 -- 172.21.15.28:0/3853019245 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa6d8031050 con 0x561134b8e680 2024-05-20T20:13:47.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.694+0000 7fa6ea7cb740 10 client.27155 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:47.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.695+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5611348bc620 con 0x561134b8e680 2024-05-20T20:13:47.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.695+0000 7fa6c6a00640 1 -- 172.21.15.28:0/3853019245 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6d8003d10 con 0x561134b8e680 2024-05-20T20:13:47.695 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.695+0000 7fa6c6a00640 1 -- 172.21.15.28:0/3853019245 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa6d8005040 con 0x561134b8e680 2024-05-20T20:13:47.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.696+0000 7fa6ea7cb740 10 client.27155 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:47.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.696+0000 7fa6ea7cb740 10 client.27155 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:47.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.696+0000 7fa6ea7cb740 20 client.27155 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:47.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.696+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134b9dde0 0x561134ba01d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.696+0000 7fa6ea7cb740 4 client.27155 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:47.696 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.696+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561134aedae0 con 0x561134b9dde0 2024-05-20T20:13:47.699 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.699+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134b9dde0 0x561134ba01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.707+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134b9dde0 0x561134ba01d0 crc :-1 s=READY pgs=185 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4b5f99b4b54b1f96 server_cookie=bd412b468dfea6d8 in_seq=0 out_seq=0 2024-05-20T20:13:47.707 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.707+0000 7fa6c6a00640 10 client.27155 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.708 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.708+0000 7fa6c6a00640 1 -- 172.21.15.28:0/3853019245 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561134aedae0 con 0x561134b9dde0 2024-05-20T20:13:47.708 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.708+0000 7fa6c6a00640 10 client.27155 handle_command_reply: tid=0 2024-05-20T20:13:47.708 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.708+0000 7fa6ea7cb740 1 client.27155 shutdown 2024-05-20T20:13:47.711 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b8e680 msgr2=0x561134b8b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b8e680 0x561134b8b9a0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fa6d80049d0 tx=0x7fa6d8005fd0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 shutdown_connections 2024-05-20T20:13:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134b9dde0 0x561134ba01d0 crc :-1 s=READY pgs=185 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b8e680 0x561134b8b9a0 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b83dc0 0x561134b8b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/3853019245 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b8e0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6c6a00640 0 client.27155 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.712+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 >> 172.21.15.28:0/3853019245 conn(0x561134b7f9b0 msgr2=0x561134b76310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.713+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 shutdown_connections 2024-05-20T20:13:47.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.713+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/3853019245 wait complete. 2024-05-20T20:13:47.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.715+0000 7fa6ea7cb740 20 client.27155 trim_cache size 0 max 16384 2024-05-20T20:13:47.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.722+0000 7feb1ae00700 1 -- 172.21.15.28:0/4292146295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb1406b480 msgr2=0x7feb1406b860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.722+0000 7feb1ae00700 1 --2- 172.21.15.28:0/4292146295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb1406b480 0x7feb1406b860 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7feb04009920 tx=0x7feb04009c30 comp rx=0 tx=0).stop 2024-05-20T20:13:47.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.724+0000 7feb1ae00700 1 -- 172.21.15.28:0/4292146295 shutdown_connections 2024-05-20T20:13:47.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.724+0000 7feb1ae00700 1 --2- 172.21.15.28:0/4292146295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb14108900 0x7feb14110480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.724+0000 7feb1ae00700 1 --2- 172.21.15.28:0/4292146295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb1406b480 0x7feb1406b860 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.724+0000 7feb1ae00700 1 --2- 172.21.15.28:0/4292146295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb1406be30 0x7feb141081d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.724+0000 7feb1ae00700 1 -- 172.21.15.28:0/4292146295 >> 172.21.15.28:0/4292146295 conn(0x7feb140793e0 msgr2=0x7feb140797f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.725+0000 7feb1ae00700 1 -- 172.21.15.28:0/4292146295 shutdown_connections 2024-05-20T20:13:47.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.725+0000 7feb1ae00700 1 -- 172.21.15.28:0/4292146295 wait complete. 2024-05-20T20:13:47.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.727+0000 7feb1ae00700 1 Processor -- start 2024-05-20T20:13:47.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.727+0000 7feb1ae00700 1 -- start start 2024-05-20T20:13:47.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.727+0000 7feb1ae00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb1406b480 0x7feb14102440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.727+0000 7feb1ae00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb1406be30 0x7feb14102980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb1ae00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb14108900 0x7feb14102ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb1ae00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7feb14113790 con 0x7feb1406be30 2024-05-20T20:13:47.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb1ae00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7feb14113610 con 0x7feb14108900 2024-05-20T20:13:47.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb1ae00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7feb14113910 con 0x7feb1406b480 2024-05-20T20:13:47.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb13e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb1406b480 0x7feb14102440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb13e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb1406b480 0x7feb14102440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59542/0 (socket says 172.21.15.28:59542) 2024-05-20T20:13:47.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb13e00700 1 -- 172.21.15.28:0/256134345 learned_addr learned my addr 172.21.15.28:0/256134345 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.728+0000 7feb13400700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb1406be30 0x7feb14102980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.729+0000 7feb18e00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb14108900 0x7feb14102ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.729+0000 7feb13e00700 1 -- 172.21.15.28:0/256134345 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb14108900 msgr2=0x7feb14102ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.729+0000 7feb13e00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb14108900 0x7feb14102ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.730+0000 7feb13e00700 1 -- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb1406be30 msgr2=0x7feb14102980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.730+0000 7feb13e00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb1406be30 0x7feb14102980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.730+0000 7feb13e00700 1 -- 172.21.15.28:0/256134345 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feb14074090 con 0x7feb1406b480 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.730+0000 7feb13400700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb1406be30 0x7feb14102980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.730+0000 7feb18e00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb14108900 0x7feb14102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.730+0000 7feb13e00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb1406b480 0x7feb14102440 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7feb0400fa90 tx=0x7feb0400fac0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.731+0000 7feb10c00700 1 -- 172.21.15.28:0/256134345 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb04027070 con 0x7feb1406b480 2024-05-20T20:13:47.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.731+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feb14074320 con 0x7feb1406b480 2024-05-20T20:13:47.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.731+0000 7feb10c00700 1 -- 172.21.15.28:0/256134345 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb04005570 con 0x7feb1406b480 2024-05-20T20:13:47.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.731+0000 7feb10c00700 1 -- 172.21.15.28:0/256134345 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb04004930 con 0x7feb1406b480 2024-05-20T20:13:47.732 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.731+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7feb14074800 con 0x7feb1406b480 2024-05-20T20:13:47.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.736+0000 7feb10c00700 1 -- 172.21.15.28:0/256134345 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7feb0402a070 con 0x7feb1406b480 2024-05-20T20:13:47.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.736+0000 7feb257dfb80 1 Processor -- start 2024-05-20T20:13:47.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.736+0000 7feb257dfb80 1 -- start start 2024-05-20T20:13:47.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4b7850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7d90 0x5566de4b19f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b2010 0x5566de4b2450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb257dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5566de40ffd0 con 0x5566de4b7d90 2024-05-20T20:13:47.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb257dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5566de3ffb70 con 0x5566de4b7470 2024-05-20T20:13:47.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb257dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5566de403bb0 con 0x5566de4b2010 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb18e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4b7850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb18e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4b7850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38318/0 (socket says 172.21.15.28:38318) 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.737+0000 7feb18e00700 1 -- 172.21.15.28:0/1770147504 learned_addr learned my addr 172.21.15.28:0/1770147504 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.738+0000 7feb18e00700 1 -- 172.21.15.28:0/1770147504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b2010 msgr2=0x5566de4b2450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.738+0000 7feb18e00700 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b2010 0x5566de4b2450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.738+0000 7feb18e00700 1 -- 172.21.15.28:0/1770147504 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7d90 msgr2=0x5566de4b19f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.738+0000 7feb13e00700 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7d90 0x5566de4b19f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.738+0000 7feb18e00700 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7d90 0x5566de4b19f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.738+0000 7feb18e00700 1 -- 172.21.15.28:0/1770147504 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5566de3c85b0 con 0x5566de4b7470 2024-05-20T20:13:47.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.739+0000 7feb18e00700 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4b7850 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7feb0c003eb0 tx=0x7feb0c0099e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a0d6d191eb58baba server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.739+0000 7feb09600700 1 -- 172.21.15.28:0/1770147504 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb0c019020 con 0x5566de4b7470 2024-05-20T20:13:47.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.739+0000 7feb09600700 1 -- 172.21.15.28:0/1770147504 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb0c0092d0 con 0x5566de4b7470 2024-05-20T20:13:47.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.739+0000 7feb09600700 1 -- 172.21.15.28:0/1770147504 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb0c00e800 con 0x5566de4b7470 2024-05-20T20:13:47.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.740+0000 7feb10c00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feaf80b2ba0 0x7feaf80b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.740+0000 7feb257dfb80 1 -- 172.21.15.28:0/1770147504 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 msgr2=0x5566de4b7850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.740+0000 7feb257dfb80 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4b7850 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7feb0c003eb0 tx=0x7feb0c0099e0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.740+0000 7feb13e00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feaf80b2ba0 0x7feaf80b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.741+0000 7feb10c00700 1 -- 172.21.15.28:0/256134345 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7feb040e3170 con 0x7feb1406b480 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.741+0000 7feb257dfb80 1 -- 172.21.15.28:0/1770147504 shutdown_connections 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.741+0000 7feb257dfb80 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b2010 0x5566de4b2450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.741+0000 7feb257dfb80 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4b7850 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.741+0000 7feb257dfb80 1 --2- 172.21.15.28:0/1770147504 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7d90 0x5566de4b19f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.741+0000 7feb257dfb80 1 -- 172.21.15.28:0/1770147504 >> 172.21.15.28:0/1770147504 conn(0x5566de3dfd70 msgr2=0x5566de4ac4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.741+0000 7feb257dfb80 1 -- 172.21.15.28:0/1770147504 shutdown_connections 2024-05-20T20:13:47.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.742+0000 7feb257dfb80 1 -- 172.21.15.28:0/1770147504 wait complete. 2024-05-20T20:13:47.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.742+0000 7feb13e00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feaf80b2ba0 0x7feaf80b5060 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7feb0401f020 tx=0x7feb040a9040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.742+0000 7feb257dfb80 1 -- start start 2024-05-20T20:13:47.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.743+0000 7feb257dfb80 1 Processor -- start 2024-05-20T20:13:47.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.743+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b2010 0x5566de4bda80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.744+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7470 0x5566de4bae20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.744+0000 7feb18e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b2010 0x5566de4bda80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.744+0000 7feb18e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b2010 0x5566de4bda80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38326/0 (socket says 172.21.15.28:38326) 2024-05-20T20:13:47.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.744+0000 7feb18e00700 1 -- 172.21.15.28:0/579358203 learned_addr learned my addr 172.21.15.28:0/579358203 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb257dfb80 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4bb360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5566de306130 con 0x5566de4b7470 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5566de3a08f0 con 0x5566de4b2010 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5566de4be870 con 0x5566de4b7d90 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb18e00700 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4bb360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb18e00700 1 -- 172.21.15.28:0/579358203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 msgr2=0x5566de4bb360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb18e00700 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4bb360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb18e00700 1 -- 172.21.15.28:0/579358203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7470 msgr2=0x5566de4bae20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb18e00700 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7470 0x5566de4bae20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.745+0000 7feb18e00700 1 -- 172.21.15.28:0/579358203 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5566de34fa60 con 0x5566de4b2010 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.746+0000 7feb257dfb80 10 client.0 _ll_register_callbacks cb 0x5566de3a9e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.746+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7feb0c0092d0 con 0x5566de4b2010 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.747+0000 7feb18e00700 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b2010 0x5566de4bda80 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7feb0c00b010 tx=0x7feb0c0094f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=947f2f71020ced87 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.747+0000 7feafe000700 10 client.27167 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:47.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.748+0000 7feafe000700 1 -- 172.21.15.28:0/579358203 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb0c031050 con 0x5566de4b2010 2024-05-20T20:13:47.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.748+0000 7feafe000700 1 -- 172.21.15.28:0/579358203 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7feb0c00e670 con 0x5566de4b2010 2024-05-20T20:13:47.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.749+0000 7feafe000700 1 -- 172.21.15.28:0/579358203 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7feb0c019030 con 0x5566de4b2010 2024-05-20T20:13:47.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.749+0000 7feb257dfb80 10 client.27167 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:47.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.749+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x5566de3d01f0 con 0x5566de4b2010 2024-05-20T20:13:47.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.749+0000 7feafe000700 1 -- 172.21.15.28:0/579358203 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7feb0c00e8b0 con 0x5566de4b2010 2024-05-20T20:13:47.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.750+0000 7feb257dfb80 10 client.27167 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:47.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.750+0000 7feb257dfb80 10 client.27167 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:13:47.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.750+0000 7feb257dfb80 20 client.27167 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:47.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.750+0000 7feb257dfb80 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de4cb110 0x5566de4cd500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.750+0000 7feb257dfb80 4 client.27167 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:47.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.750+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5566de411f20 con 0x5566de4cb110 2024-05-20T20:13:47.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.750+0000 7feb13400700 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de4cb110 0x5566de4cd500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.753+0000 7feb13400700 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de4cb110 0x5566de4cd500 crc :-1 s=READY pgs=186 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3d659274b94ffdc server_cookie=d6f189c1bb53850c in_seq=0 out_seq=0 2024-05-20T20:13:47.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.753+0000 7feafe000700 10 client.27167 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.754+0000 7feafe000700 1 -- 172.21.15.28:0/579358203 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5566de411f20 con 0x5566de4cb110 2024-05-20T20:13:47.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.754+0000 7feafe000700 10 client.27167 handle_command_reply: tid=0 2024-05-20T20:13:47.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.755+0000 7feb257dfb80 1 client.27167 shutdown 2024-05-20T20:13:47.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b2010 msgr2=0x5566de4bda80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b2010 0x5566de4bda80 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7feb0c00b010 tx=0x7feb0c0094f0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 shutdown_connections 2024-05-20T20:13:47.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de4cb110 0x5566de4cd500 crc :-1 s=READY pgs=186 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4bb360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b2010 0x5566de4bda80 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 --2- 172.21.15.28:0/579358203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b7470 0x5566de4bae20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.760+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 >> 172.21.15.28:0/579358203 conn(0x5566de3dfd70 msgr2=0x5566de4b09e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.761+0000 7feafe000700 0 client.27167 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.762+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 shutdown_connections 2024-05-20T20:13:47.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.762+0000 7feb257dfb80 1 -- 172.21.15.28:0/579358203 wait complete. 2024-05-20T20:13:47.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.762+0000 7feb257dfb80 20 client.27167 trim_cache size 0 max 16384 2024-05-20T20:13:47.812 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.811+0000 7fa6ea7cb740 1 Processor -- start 2024-05-20T20:13:47.814 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.812+0000 7fa6ea7cb740 1 -- start start 2024-05-20T20:13:47.814 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.812+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b834a0 0x561134b860e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.818+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b83dc0 0x561134b9a310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.817+0000 7feb257dfb80 1 Processor -- start 2024-05-20T20:13:47.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.818+0000 7feb257dfb80 1 -- start start 2024-05-20T20:13:47.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.818+0000 7fa6e5800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b834a0 0x561134b860e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.819+0000 7fa6e5800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b834a0 0x561134b860e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38334/0 (socket says 172.21.15.28:38334) 2024-05-20T20:13:47.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.820+0000 7fa6e5800640 1 -- 172.21.15.28:0/41805888 learned_addr learned my addr 172.21.15.28:0/41805888 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.818+0000 7fa6dfe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b83dc0 0x561134b9a310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.820+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b8df40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.820+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e01ac620 con 0x561134b83dc0 2024-05-20T20:13:47.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.820+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e01ae160 con 0x561134b834a0 2024-05-20T20:13:47.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.820+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e01accf0 con 0x561134b8dac0 2024-05-20T20:13:47.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.823+0000 7fa6e5800640 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b8df40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.819+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4b5560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.819+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4bb310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.819+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4d2aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.819+0000 7feb257dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7feb141bcc60 con 0x5566de4b2010 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.819+0000 7feb257dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7feb141be7a0 con 0x5566de4b7470 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.819+0000 7feb257dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7feb141bd330 con 0x5566de4b7d90 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.820+0000 7feb13400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4bb310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.820+0000 7feb13400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4bb310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38340/0 (socket says 172.21.15.28:38340) 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.820+0000 7feb13400700 1 -- 172.21.15.28:0/665871619 learned_addr learned my addr 172.21.15.28:0/665871619 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.821+0000 7feb18e00700 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4d2aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.821+0000 7feb13400700 1 -- 172.21.15.28:0/665871619 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 msgr2=0x5566de4d2aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.821+0000 7feb13400700 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4d2aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.821+0000 7feb13400700 1 -- 172.21.15.28:0/665871619 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 msgr2=0x5566de4b5560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.821+0000 7feb13400700 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4b5560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.821+0000 7feb13400700 1 -- 172.21.15.28:0/665871619 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5566de4b9390 con 0x5566de4b7470 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.822+0000 7feb13400700 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4bb310 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7feb00006b30 tx=0x7feb00009e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5fbf3a5c8825deb2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.823+0000 7feb08c00700 1 -- 172.21.15.28:0/665871619 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb00024070 con 0x5566de4b7470 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.823+0000 7feb08c00700 1 -- 172.21.15.28:0/665871619 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb00015dc0 con 0x5566de4b7470 2024-05-20T20:13:47.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.823+0000 7feb08c00700 1 -- 172.21.15.28:0/665871619 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb00024070 con 0x5566de4b7470 2024-05-20T20:13:47.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.825+0000 7fa6e5800640 1 -- 172.21.15.28:0/41805888 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 msgr2=0x561134b8df40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.825+0000 7fa6e5800640 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b8df40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.825+0000 7fa6e5800640 1 -- 172.21.15.28:0/41805888 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b83dc0 msgr2=0x561134b9a310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.825+0000 7fa6e5800640 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b83dc0 0x561134b9a310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.825+0000 7fa6e5800640 1 -- 172.21.15.28:0/41805888 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561134b86f90 con 0x561134b834a0 2024-05-20T20:13:47.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.827+0000 7fa6e5800640 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b834a0 0x561134b860e0 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8005980 tx=0x7fa6d8004d20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e8b3fe86df4a68db server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.828+0000 7fa6d1600640 1 -- 172.21.15.28:0/41805888 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d8025040 con 0x561134b834a0 2024-05-20T20:13:47.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.828+0000 7fa6d1600640 1 -- 172.21.15.28:0/41805888 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6d8026040 con 0x561134b834a0 2024-05-20T20:13:47.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.828+0000 7fa6d1600640 1 -- 172.21.15.28:0/41805888 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6d8004640 con 0x561134b834a0 2024-05-20T20:13:47.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.825+0000 7feb257dfb80 1 -- 172.21.15.28:0/665871619 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 msgr2=0x5566de4bb310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.825+0000 7feb257dfb80 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4bb310 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7feb00006b30 tx=0x7feb00009e80 comp rx=0 tx=0).stop 2024-05-20T20:13:47.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.825+0000 7feb257dfb80 1 -- 172.21.15.28:0/665871619 shutdown_connections 2024-05-20T20:13:47.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.826+0000 7feb257dfb80 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de4d2aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.826+0000 7feb257dfb80 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4b5560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.826+0000 7feb257dfb80 1 --2- 172.21.15.28:0/665871619 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4bb310 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.826+0000 7feb257dfb80 1 -- 172.21.15.28:0/665871619 >> 172.21.15.28:0/665871619 conn(0x5566de3dfd70 msgr2=0x5566de4bac00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.826+0000 7feb257dfb80 1 -- 172.21.15.28:0/665871619 shutdown_connections 2024-05-20T20:13:47.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.827+0000 7feb257dfb80 1 -- 172.21.15.28:0/665871619 wait complete. 2024-05-20T20:13:47.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.828+0000 7feb257dfb80 1 -- start start 2024-05-20T20:13:47.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.830+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b834a0 msgr2=0x561134b860e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.830+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b834a0 0x561134b860e0 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fa6d8005980 tx=0x7fa6d8004d20 comp rx=0 tx=0).stop 2024-05-20T20:13:47.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.829+0000 7feb257dfb80 1 Processor -- start 2024-05-20T20:13:47.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.830+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4c10e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.832+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 shutdown_connections 2024-05-20T20:13:47.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.831+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4c1620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.832+0000 7feb257dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de535980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.832+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b8df40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.832+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b83dc0 0x561134b9a310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.832+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/41805888 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b834a0 0x561134b860e0 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.832+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 >> 172.21.15.28:0/41805888 conn(0x561134b7f9b0 msgr2=0x561134b8a3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.832+0000 7feb18e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4c10e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.832+0000 7feb257dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7feb141be0d0 con 0x5566de4b2010 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.832+0000 7feb13400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4c1620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.832+0000 7feb18e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4c10e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42368/0 (socket says 172.21.15.28:42368) 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.833+0000 7feb18e00700 1 -- 172.21.15.28:0/3085927127 learned_addr learned my addr 172.21.15.28:0/3085927127 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.833+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7feb141bda00 con 0x5566de4b7470 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.833+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7feb141bac40 con 0x5566de4b7d90 2024-05-20T20:13:47.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.833+0000 7feb257dfb80 10 client.0 _ll_register_callbacks cb 0x5566de4acb10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:47.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.835+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 shutdown_connections 2024-05-20T20:13:47.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.835+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/41805888 wait complete. 2024-05-20T20:13:47.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.835+0000 7feb18e00700 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de535980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.836+0000 7feb13400700 1 -- 172.21.15.28:0/3085927127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 msgr2=0x5566de535980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.836+0000 7feb13400700 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de535980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.836+0000 7feb13400700 1 -- 172.21.15.28:0/3085927127 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 msgr2=0x5566de4c10e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.836+0000 7feb13400700 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4c10e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.837+0000 7feb13400700 1 -- 172.21.15.28:0/3085927127 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5566de535ec0 con 0x5566de4b7470 2024-05-20T20:13:47.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.837+0000 7feb18e00700 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de535980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:47.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.838+0000 7feafe000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:47.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.839+0000 7feb13400700 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4c1620 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x5566de4c81d0 tx=0x7feb00004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fd24347cb356f8ee server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.837+0000 7fa6ea7cb740 1 -- start start 2024-05-20T20:13:47.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.839+0000 7fa6ea7cb740 1 Processor -- start 2024-05-20T20:13:47.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.839+0000 7feafe000700 1 -- 172.21.15.28:0/3085927127 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb00024070 con 0x5566de4b7470 2024-05-20T20:13:47.841 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.840+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b969b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.841 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.841+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b87ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.842 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.841+0000 7fa6ea7cb740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b88420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.842 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.841+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e01ada90 con 0x561134b834a0 2024-05-20T20:13:47.842 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.841+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e01ad3c0 con 0x561134b83dc0 2024-05-20T20:13:47.842 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.841+0000 7fa6ea7cb740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa6e01aa660 con 0x561134b8dac0 2024-05-20T20:13:47.842 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.841+0000 7fa6ea7cb740 10 client.0 _ll_register_callbacks cb 0x7fa6d8009e50 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.840+0000 7feafe000700 1 -- 172.21.15.28:0/3085927127 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7feb0001c070 con 0x5566de4b7470 2024-05-20T20:13:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.840+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7feb00015dc0 con 0x5566de4b7470 2024-05-20T20:13:47.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.841+0000 7feafe000700 1 -- 172.21.15.28:0/3085927127 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7feb00024070 con 0x5566de4b7470 2024-05-20T20:13:47.847 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.847+0000 7fa6dfe00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b87ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.847 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.847+0000 7fa6dfe00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b87ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38358/0 (socket says 172.21.15.28:38358) 2024-05-20T20:13:47.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.847+0000 7fa6dfe00640 1 -- 172.21.15.28:0/4114227027 learned_addr learned my addr 172.21.15.28:0/4114227027 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:47.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.847+0000 7fa6e5800640 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b88420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.847+0000 7fa6e5800640 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b969b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.842+0000 7feb257dfb80 10 client.27176 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:47.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.843+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5566de537150 con 0x5566de4b7470 2024-05-20T20:13:47.849 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.848+0000 7fa6dfe00640 1 -- 172.21.15.28:0/4114227027 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 msgr2=0x561134b88420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.849 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.848+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b88420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.849 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.848+0000 7fa6dfe00640 1 -- 172.21.15.28:0/4114227027 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 msgr2=0x561134b969b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:47.849 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.848+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b969b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.849 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.848+0000 7fa6dfe00640 1 -- 172.21.15.28:0/4114227027 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561134b88a10 con 0x561134b83dc0 2024-05-20T20:13:47.849 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.848+0000 7fa6dfe00640 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b87ee0 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7fa6cc009830 tx=0x7fa6cc005780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ec5d727ca49c9213 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:47.852 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.848+0000 7fa6c6a00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:13:47.852 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.849+0000 7fa6c6a00640 1 -- 172.21.15.28:0/4114227027 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6cc037040 con 0x561134b83dc0 2024-05-20T20:13:47.852 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.849+0000 7fa6c6a00640 1 -- 172.21.15.28:0/4114227027 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fa6cc030680 con 0x561134b83dc0 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.846+0000 7feafe000700 1 -- 172.21.15.28:0/3085927127 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7feb00015dc0 con 0x5566de4b7470 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.846+0000 7feafe000700 1 -- 172.21.15.28:0/3085927127 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7feb00006340 con 0x5566de4b7470 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.846+0000 7feb257dfb80 10 client.27176 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.846+0000 7feb257dfb80 10 client.27176 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.847+0000 7feb257dfb80 20 client.27176 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.847+0000 7feb257dfb80 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de537710 0x5566de539b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.847+0000 7feb257dfb80 4 client.27176 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 143.3, "await_for": 45.0}] 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.847+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 143.3, "await_for": 45.0}) -- 0x5566de40b700 con 0x5566de537710 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.847+0000 7feb18e00700 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de537710 0x5566de539b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.849+0000 7feafe000700 10 client.27176 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.855 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.856+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa6d8026040 con 0x561134b83dc0 2024-05-20T20:13:47.856 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.856+0000 7fa6c6a00640 1 -- 172.21.15.28:0/4114227027 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa6cc030880 con 0x561134b83dc0 2024-05-20T20:13:47.856 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.856+0000 7fa6ea7cb740 10 client.27179 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:47.856 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.857+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561134c19610 con 0x561134b83dc0 2024-05-20T20:13:47.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.857+0000 7fa6c6a00640 1 -- 172.21.15.28:0/4114227027 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa6d8026040 con 0x561134b83dc0 2024-05-20T20:13:47.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.857+0000 7fa6c6a00640 1 -- 172.21.15.28:0/4114227027 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa6cc003bd0 con 0x561134b83dc0 2024-05-20T20:13:47.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.859+0000 7fa6ea7cb740 10 client.27179 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:47.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.859+0000 7fa6ea7cb740 10 client.27179 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:47.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.859+0000 7fa6ea7cb740 20 client.27179 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:47.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.859+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134c19cf0 0x561134c1c110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:47.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.859+0000 7fa6ea7cb740 4 client.27179 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:13:47.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.859+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x561134ae72c0 con 0x561134c19cf0 2024-05-20T20:13:47.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.859+0000 7feb18e00700 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de537710 0x5566de539b00 crc :-1 s=READY pgs=187 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a02d79b0126cee5e server_cookie=3afa149804d47bbb in_seq=0 out_seq=0 2024-05-20T20:13:47.860 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.859+0000 7fa6e5800640 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134c19cf0 0x561134c1c110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:47.861 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.862+0000 7fa6e5800640 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134c19cf0 0x561134c1c110 crc :-1 s=READY pgs=188 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5dd95951598c3bd server_cookie=60ba704edd5e4da7 in_seq=0 out_seq=0 2024-05-20T20:13:47.861 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.862+0000 7fa6c6a00640 10 client.27179 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.910+0000 7fa6c6a00640 1 -- 172.21.15.28:0/4114227027 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x561134ae72c0 con 0x561134c19cf0 2024-05-20T20:13:47.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.910+0000 7fa6c6a00640 10 client.27179 handle_command_reply: tid=0 2024-05-20T20:13:47.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.910+0000 7fa6ea7cb740 1 client.27179 shutdown 2024-05-20T20:13:47.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 msgr2=0x561134b87ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.911 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b87ee0 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7fa6cc009830 tx=0x7fa6cc005780 comp rx=0 tx=0).stop 2024-05-20T20:13:47.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 shutdown_connections 2024-05-20T20:13:47.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561134c19cf0 0x561134c1c110 crc :-1 s=READY pgs=188 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561134b83dc0 0x561134b87ee0 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561134b8dac0 0x561134b88420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 --2- 172.21.15.28:0/4114227027 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561134b834a0 0x561134b969b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 >> 172.21.15.28:0/4114227027 conn(0x561134b7f9b0 msgr2=0x561134b8a3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.912+0000 7fa6c6a00640 0 client.27179 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.914+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 shutdown_connections 2024-05-20T20:13:47.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.914+0000 7fa6ea7cb740 1 -- 172.21.15.28:0/4114227027 wait complete. 2024-05-20T20:13:47.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.915+0000 7fa6ea7cb740 20 client.27179 trim_cache size 0 max 16384 2024-05-20T20:13:47.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:13:47.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:13:47.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:13:47.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.917+0000 7feafe000700 1 -- 172.21.15.28:0/3085927127 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x5566de40b700 con 0x5566de537710 2024-05-20T20:13:47.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.917+0000 7feafe000700 10 client.27176 handle_command_reply: tid=0 2024-05-20T20:13:47.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.917+0000 7feb257dfb80 1 client.27176 shutdown 2024-05-20T20:13:47.921 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.921+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6c80b2b10 msgr2=0x7fa6c80b4fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.921 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.921+0000 7fa6e7800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6c80b2b10 0x7fa6c80b4fd0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x561134b84a40 tx=0x7fa6d4005c80 comp rx=0 tx=0).stop 2024-05-20T20:13:47.921 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.921+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0102a60 msgr2=0x7fa6e019b370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.921 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.921+0000 7fa6e7800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0102a60 0x7fa6e019b370 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fa6d400e9c0 tx=0x7fa6d400ee80 comp rx=0 tx=0).stop 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 msgr2=0x5566de4c1620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4c1620 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x5566de4c81d0 tx=0x7feb00004750 comp rx=0 tx=0).stop 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 shutdown_connections 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5566de537710 0x5566de539b00 crc :-1 s=READY pgs=187 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5566de4b7d90 0x5566de535980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5566de4b2010 0x5566de4c10e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 --2- 172.21.15.28:0/3085927127 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5566de4b7470 0x5566de4c1620 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 >> 172.21.15.28:0/3085927127 conn(0x5566de3dfd70 msgr2=0x5566de4a5e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feafe000700 0 client.27176 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.918+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 shutdown_connections 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.919+0000 7feb257dfb80 1 -- 172.21.15.28:0/3085927127 wait complete. 2024-05-20T20:13:47.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.919+0000 7feb257dfb80 20 client.27176 trim_cache size 0 max 16384 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 22, 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "c71942f8": { 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 22, 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:13:47.923 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 143.3, 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:47.924 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:47.925 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:47.925 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:13:47.925 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:13:47.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.923+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 shutdown_connections 2024-05-20T20:13:47.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.923+0000 7fa6e7800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa6e0108a70 0x7fa6e019fc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.923+0000 7fa6e7800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa6c80b2b10 0x7fa6c80b4fd0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.923+0000 7fa6e7800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa6e0103420 0x7fa6e019b8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.923+0000 7fa6e7800640 1 --2- 172.21.15.28:0/2319757660 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa6e0102a60 0x7fa6e019b370 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.923+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 >> 172.21.15.28:0/2319757660 conn(0x7fa6e0079ee0 msgr2=0x7fa6e010e620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.924+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feaf80b2ba0 msgr2=0x7feaf80b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.924+0000 7feb1ae00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feaf80b2ba0 0x7feaf80b5060 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7feb0401f020 tx=0x7feb040a9040 comp rx=0 tx=0).stop 2024-05-20T20:13:47.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.924+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb1406b480 msgr2=0x7feb14102440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:47.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.924+0000 7feb1ae00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb1406b480 0x7feb14102440 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7feb0400fa90 tx=0x7feb0400fac0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.925+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 shutdown_connections 2024-05-20T20:13:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.925+0000 7feb1ae00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7feaf80b2ba0 0x7feaf80b5060 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.925+0000 7feb1ae00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feb14108900 0x7feb14102ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.925+0000 7feb1ae00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feb1406b480 0x7feb14102440 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.925+0000 7feb1ae00700 1 --2- 172.21.15.28:0/256134345 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feb1406be30 0x7feb14102980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.925+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 >> 172.21.15.28:0/256134345 conn(0x7feb140793e0 msgr2=0x7feb14100730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:47.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.926+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 shutdown_connections 2024-05-20T20:13:47.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.926+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 shutdown_connections 2024-05-20T20:13:47.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:13:47.926+0000 7fa6e7800640 1 -- 172.21.15.28:0/2319757660 wait complete. 2024-05-20T20:13:47.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:47.926+0000 7feb1ae00700 1 -- 172.21.15.28:0/256134345 wait complete. 2024-05-20T20:13:48.495 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:49.449 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:49 smithi028 ceph-mon[31178]: pgmap v252: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 260 MiB/s wr, 1.45k op/s 2024-05-20T20:13:49.449 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:49 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:13:49.526 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set 'c71942f8', quiesce took 20.5 seconds. Will release after: 2.8000000000000007 2024-05-20T20:13:49.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:49 smithi154 ceph-mon[36314]: pgmap v252: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 260 MiB/s wr, 1.45k op/s 2024-05-20T20:13:49.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:49 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:13:49.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:49 smithi033 ceph-mon[36179]: pgmap v252: 97 pgs: 97 active+clean; 50 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 260 MiB/s wr, 1.45k op/s 2024-05-20T20:13:49.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:49 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:13:51.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:51 smithi028 ceph-mon[31178]: pgmap v253: 97 pgs: 97 active+clean; 51 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 337 MiB/s wr, 1.67k op/s 2024-05-20T20:13:51.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:51 smithi154 ceph-mon[36314]: pgmap v253: 97 pgs: 97 active+clean; 51 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 337 MiB/s wr, 1.67k op/s 2024-05-20T20:13:51.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:51 smithi033 ceph-mon[36179]: pgmap v253: 97 pgs: 97 active+clean; 51 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 337 MiB/s wr, 1.67k op/s 2024-05-20T20:13:52.327 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:13:53.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.317+0000 7f3283c00700 1 -- 172.21.15.28:0/4273042535 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c06bbe0 msgr2=0x7f327c1103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.317+0000 7f3283c00700 1 --2- 172.21.15.28:0/4273042535 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c06bbe0 0x7f327c1103d0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f3274009f90 tx=0x7f327400b330 comp rx=0 tx=0).stop 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.318+0000 7f3283c00700 1 -- 172.21.15.28:0/4273042535 shutdown_connections 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.318+0000 7f3283c00700 1 --2- 172.21.15.28:0/4273042535 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c06bbe0 0x7f327c1103d0 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.318+0000 7f3283c00700 1 --2- 172.21.15.28:0/4273042535 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c1095b0 0x7f327c109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.318+0000 7f3283c00700 1 --2- 172.21.15.28:0/4273042535 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f327c06b220 0x7f327c06b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.318+0000 7f3283c00700 1 -- 172.21.15.28:0/4273042535 >> 172.21.15.28:0/4273042535 conn(0x7f327c0793f0 msgr2=0x7f327c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.318+0000 7f3283c00700 1 -- 172.21.15.28:0/4273042535 shutdown_connections 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.318+0000 7f3283c00700 1 -- 172.21.15.28:0/4273042535 wait complete. 2024-05-20T20:13:53.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.320+0000 7f3283c00700 1 Processor -- start 2024-05-20T20:13:53.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.320+0000 7f3283c00700 1 -- start start 2024-05-20T20:13:53.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.320+0000 7f3283c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c06b220 0x7f327c074d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3283c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f327c06bbe0 0x7f327c06de10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3281200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c06b220 0x7f327c074d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3283c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c1095b0 0x7f327c06e350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3283c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f327c111530 con 0x7f327c06b220 2024-05-20T20:13:53.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3281200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c06b220 0x7f327c074d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42406/0 (socket says 172.21.15.28:42406) 2024-05-20T20:13:53.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3281200700 1 -- 172.21.15.28:0/3704696737 learned_addr learned my addr 172.21.15.28:0/3704696737 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:53.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1113b0 con 0x7f327c06bbe0 2024-05-20T20:13:53.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1116b0 con 0x7f327c1095b0 2024-05-20T20:13:53.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.321+0000 7f327be00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f327c06bbe0 0x7f327c06de10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.322+0000 7f3281c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c1095b0 0x7f327c06e350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f327be00700 1 -- 172.21.15.28:0/3704696737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c1095b0 msgr2=0x7f327c06e350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f327be00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c1095b0 0x7f327c06e350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f327be00700 1 -- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c06b220 msgr2=0x7f327c074d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f327be00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c06b220 0x7f327c074d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f327be00700 1 -- 172.21.15.28:0/3704696737 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f327c06e990 con 0x7f327c06bbe0 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f3281200700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c06b220 0x7f327c074d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f327be00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f327c06bbe0 0x7f327c06de10 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f326800d750 tx=0x7f326800da60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.323+0000 7f3279600700 1 -- 172.21.15.28:0/3704696737 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3268016070 con 0x7f327c06bbe0 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.324+0000 7f3279600700 1 -- 172.21.15.28:0/3704696737 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3268004830 con 0x7f327c06bbe0 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.324+0000 7f3279600700 1 -- 172.21.15.28:0/3704696737 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3268005560 con 0x7f327c06bbe0 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.324+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f327c072860 con 0x7f327c06bbe0 2024-05-20T20:13:53.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.324+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f327c072d50 con 0x7f327c06bbe0 2024-05-20T20:13:53.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.330+0000 7f3279600700 1 -- 172.21.15.28:0/3704696737 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3268004220 con 0x7f327c06bbe0 2024-05-20T20:13:53.330 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.331+0000 7f328e633b80 1 Processor -- start 2024-05-20T20:13:53.331 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.331+0000 7f328e633b80 1 -- start start 2024-05-20T20:13:53.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f328e633b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x56207530f290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f328e633b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075309260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f328e633b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562075309880 0x562075309ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f328e633b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562075267e40 con 0x56207530f7d0 2024-05-20T20:13:53.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f328e633b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5620752579e0 con 0x56207530eeb0 2024-05-20T20:13:53.332 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f328e633b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56207525ba20 con 0x562075309880 2024-05-20T20:13:53.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f3281c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x56207530f290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f3281c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x56207530f290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38384/0 (socket says 172.21.15.28:38384) 2024-05-20T20:13:53.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f3281c00700 1 -- 172.21.15.28:0/4279675220 learned_addr learned my addr 172.21.15.28:0/4279675220 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:53.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.332+0000 7f3279600700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f32640b6eb0 0x7f32640b9370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.333+0000 7f327be00700 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562075309880 0x562075309ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.333 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.333+0000 7f3279600700 1 -- 172.21.15.28:0/3704696737 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f32680e2280 con 0x7f327c06bbe0 2024-05-20T20:13:53.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.333+0000 7f3281c00700 1 -- 172.21.15.28:0/4279675220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562075309880 msgr2=0x562075309ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.333+0000 7f3281c00700 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562075309880 0x562075309ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.333+0000 7f3281c00700 1 -- 172.21.15.28:0/4279675220 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 msgr2=0x562075309260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.334+0000 7f3281200700 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075309260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.334+0000 7f3281c00700 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075309260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.334+0000 7f3281c00700 1 -- 172.21.15.28:0/4279675220 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562075220420 con 0x56207530eeb0 2024-05-20T20:13:53.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.334+0000 7f3281c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f32640b6eb0 0x7f32640b9370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.335+0000 7f3281c00700 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x56207530f290 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7f3274009490 tx=0x7f32740094c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=627c328905c463de server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:53.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.335+0000 7f326e000700 1 -- 172.21.15.28:0/4279675220 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f327402d070 con 0x56207530eeb0 2024-05-20T20:13:53.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.335+0000 7f3281c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f32640b6eb0 0x7f32640b9370 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f327402a000 tx=0x7f327402b000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:53.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.335+0000 7f326e000700 1 -- 172.21.15.28:0/4279675220 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3274007ad0 con 0x56207530eeb0 2024-05-20T20:13:53.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.335+0000 7f326e000700 1 -- 172.21.15.28:0/4279675220 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3274027960 con 0x56207530eeb0 2024-05-20T20:13:53.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.337+0000 7f328e633b80 1 -- 172.21.15.28:0/4279675220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 msgr2=0x56207530f290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.337+0000 7f328e633b80 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x56207530f290 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7f3274009490 tx=0x7f32740094c0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.338+0000 7f328e633b80 1 -- 172.21.15.28:0/4279675220 shutdown_connections 2024-05-20T20:13:53.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.338+0000 7f328e633b80 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562075309880 0x562075309ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.338+0000 7f328e633b80 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x56207530f290 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.338+0000 7f328e633b80 1 --2- 172.21.15.28:0/4279675220 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075309260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.338+0000 7f328e633b80 1 -- 172.21.15.28:0/4279675220 >> 172.21.15.28:0/4279675220 conn(0x562075237be0 msgr2=0x562075304010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:53.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.339+0000 7f328e633b80 1 -- 172.21.15.28:0/4279675220 shutdown_connections 2024-05-20T20:13:53.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.339+0000 7f328e633b80 1 -- 172.21.15.28:0/4279675220 wait complete. 2024-05-20T20:13:53.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.339+0000 7f328e633b80 1 -- start start 2024-05-20T20:13:53.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.340+0000 7f328e633b80 1 Processor -- start 2024-05-20T20:13:53.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.342+0000 7f328e633b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075315500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.342+0000 7f328e633b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753128a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.342+0000 7f328e633b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530f7d0 0x562075312de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.342+0000 7f327be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753128a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.342+0000 7f327be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753128a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59634/0 (socket says 172.21.15.28:59634) 2024-05-20T20:13:53.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.343+0000 7f3281c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530f7d0 0x562075312de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.344+0000 7f3281c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530f7d0 0x562075312de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38394/0 (socket says 172.21.15.28:38394) 2024-05-20T20:13:53.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.344+0000 7f328e633b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56207515dfa0 con 0x562075309880 2024-05-20T20:13:53.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.344+0000 7f328e633b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5620751f8760 con 0x56207530f7d0 2024-05-20T20:13:53.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.344+0000 7f328e633b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562075316240 con 0x56207530eeb0 2024-05-20T20:13:53.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.344+0000 7f328e633b80 10 client.0 _ll_register_callbacks cb 0x562075201c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:53.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.344+0000 7f327be00700 1 -- 172.21.15.28:0/1021726388 learned_addr learned my addr 172.21.15.28:0/1021726388 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:53.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.344+0000 7f3281200700 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075315500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.346+0000 7f327be00700 1 -- 172.21.15.28:0/1021726388 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530f7d0 msgr2=0x562075312de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.346+0000 7f327be00700 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530f7d0 0x562075312de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.346+0000 7f327be00700 1 -- 172.21.15.28:0/1021726388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 msgr2=0x562075315500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.346+0000 7f327be00700 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075315500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.346+0000 7f327be00700 1 -- 172.21.15.28:0/1021726388 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5620751a78d0 con 0x56207530eeb0 2024-05-20T20:13:53.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.346+0000 7f3281200700 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075315500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:53.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.347+0000 7f327be00700 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753128a0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f326800de60 tx=0x7f32680058e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d28f23f29cc8e411 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:53.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.347+0000 7f3262a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:13:53.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.348+0000 7f3262a00700 1 -- 172.21.15.28:0/1021726388 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f326805a020 con 0x56207530eeb0 2024-05-20T20:13:53.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.348+0000 7f3262a00700 1 -- 172.21.15.28:0/1021726388 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f3268004d00 con 0x56207530eeb0 2024-05-20T20:13:53.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.349+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3274007ad0 con 0x56207530eeb0 2024-05-20T20:13:53.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.349+0000 7f3262a00700 1 -- 172.21.15.28:0/1021726388 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f32680a9040 con 0x56207530eeb0 2024-05-20T20:13:53.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.349+0000 7f328e633b80 10 client.27088 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:53.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.349+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562075228060 con 0x56207530eeb0 2024-05-20T20:13:53.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.350+0000 7f3262a00700 1 -- 172.21.15.28:0/1021726388 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3274007ad0 con 0x56207530eeb0 2024-05-20T20:13:53.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.351+0000 7f3262a00700 1 -- 172.21.15.28:0/1021726388 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f32680a9790 con 0x56207530eeb0 2024-05-20T20:13:53.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.351+0000 7f328e633b80 10 client.27088 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:53.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.351+0000 7f328e633b80 10 client.27088 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:53.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.351+0000 7f328e633b80 20 client.27088 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:53.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.352+0000 7f328e633b80 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56207532b6a0 0x56207532da90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.352+0000 7f328e633b80 4 client.27088 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:13:53.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.352+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562075269d90 con 0x56207532b6a0 2024-05-20T20:13:53.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.353+0000 7f3281200700 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56207532b6a0 0x56207532da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.354+0000 7f3281200700 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56207532b6a0 0x56207532da90 crc :-1 s=READY pgs=189 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=372b1352fd57cae1 server_cookie=7a3187a12aba9ec5 in_seq=0 out_seq=0 2024-05-20T20:13:53.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.354+0000 7f3262a00700 10 client.27088 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:53.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.355+0000 7f3262a00700 1 -- 172.21.15.28:0/1021726388 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562075269d90 con 0x56207532b6a0 2024-05-20T20:13:53.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.355+0000 7f3262a00700 10 client.27088 handle_command_reply: tid=0 2024-05-20T20:13:53.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.356+0000 7f328e633b80 1 client.27088 shutdown 2024-05-20T20:13:53.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.357+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 msgr2=0x5620753128a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.357+0000 7f328e633b80 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753128a0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f326800de60 tx=0x7f32680058e0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.357+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 shutdown_connections 2024-05-20T20:13:53.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.358+0000 7f328e633b80 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56207532b6a0 0x56207532da90 crc :-1 s=READY pgs=189 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.358+0000 7f328e633b80 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530f7d0 0x562075312de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.358+0000 7f328e633b80 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075315500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.358+0000 7f328e633b80 1 --2- 172.21.15.28:0/1021726388 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753128a0 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.358+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 >> 172.21.15.28:0/1021726388 conn(0x562075237be0 msgr2=0x562075308250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:53.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.358+0000 7f3262a00700 0 client.27088 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:53.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.359+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 shutdown_connections 2024-05-20T20:13:53.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.359+0000 7f328e633b80 1 -- 172.21.15.28:0/1021726388 wait complete. 2024-05-20T20:13:53.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.360+0000 7f328e633b80 20 client.27088 trim_cache size 0 max 16384 2024-05-20T20:13:53.368 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.415+0000 7f328e633b80 1 Processor -- start 2024-05-20T20:13:53.416 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:53 smithi028 ceph-mon[31178]: pgmap v254: 97 pgs: 97 active+clean; 52 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 267 MiB/s wr, 1.18k op/s 2024-05-20T20:13:53.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.416+0000 7f328e633b80 1 -- start start 2024-05-20T20:13:53.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.416+0000 7f328e633b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075309260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.416+0000 7f328e633b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x5620753040f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.416+0000 7f328e633b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530f7d0 0x5620753332f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.417+0000 7f327be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x5620753040f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.417+0000 7f3281c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530f7d0 0x5620753332f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.418+0000 7f3281c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530f7d0 0x5620753332f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59640/0 (socket says 172.21.15.28:59640) 2024-05-20T20:13:53.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.418+0000 7f327be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x5620753040f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38408/0 (socket says 172.21.15.28:38408) 2024-05-20T20:13:53.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.418+0000 7f3281c00700 1 -- 172.21.15.28:0/2241660060 learned_addr learned my addr 172.21.15.28:0/2241660060 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:53.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.418+0000 7f3281200700 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075309260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.418+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1b4440 con 0x562075309880 2024-05-20T20:13:53.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.418+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1b5f80 con 0x56207530eeb0 2024-05-20T20:13:53.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.418+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1b4b10 con 0x56207530f7d0 2024-05-20T20:13:53.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.420+0000 7f3281c00700 1 -- 172.21.15.28:0/2241660060 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 msgr2=0x5620753040f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.420+0000 7f3281c00700 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x5620753040f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.420+0000 7f3281c00700 1 -- 172.21.15.28:0/2241660060 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 msgr2=0x562075309260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.420+0000 7f3281c00700 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075309260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.420+0000 7f3281c00700 1 -- 172.21.15.28:0/2241660060 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5620753290c0 con 0x56207530f7d0 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.424+0000 7f3281c00700 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530f7d0 0x5620753332f0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f3274009490 tx=0x7f327401d000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a902c4e9c593228c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.425+0000 7f326d600700 1 -- 172.21.15.28:0/2241660060 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f327402d070 con 0x56207530f7d0 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.425+0000 7f326d600700 1 -- 172.21.15.28:0/2241660060 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3274003d20 con 0x56207530f7d0 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.425+0000 7f326d600700 1 -- 172.21.15.28:0/2241660060 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3274007940 con 0x56207530f7d0 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.426+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530f7d0 msgr2=0x5620753332f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.426+0000 7f328e633b80 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530f7d0 0x5620753332f0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f3274009490 tx=0x7f327401d000 comp rx=0 tx=0).stop 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.427+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 shutdown_connections 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.427+0000 7f328e633b80 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530f7d0 0x5620753332f0 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.428+0000 7f328e633b80 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562075309880 0x562075309260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.428+0000 7f328e633b80 1 --2- 172.21.15.28:0/2241660060 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56207530eeb0 0x5620753040f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.428+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 >> 172.21.15.28:0/2241660060 conn(0x562075237be0 msgr2=0x5620753039e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:53.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.428+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 shutdown_connections 2024-05-20T20:13:53.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.428+0000 7f328e633b80 1 -- 172.21.15.28:0/2241660060 wait complete. 2024-05-20T20:13:53.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.429+0000 7f328e633b80 1 -- start start 2024-05-20T20:13:53.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.430+0000 7f328e633b80 1 Processor -- start 2024-05-20T20:13:53.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.432+0000 7f328e633b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562075309880 0x562075323f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.433+0000 7f328e633b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753244d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.433+0000 7f328e633b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075320d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.433+0000 7f328e633b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1b58b0 con 0x56207530f7d0 2024-05-20T20:13:53.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.433+0000 7f328e633b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1b51e0 con 0x562075309880 2024-05-20T20:13:53.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.433+0000 7f328e633b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f327c1b2400 con 0x56207530eeb0 2024-05-20T20:13:53.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.433+0000 7f328e633b80 10 client.0 _ll_register_callbacks cb 0x562075201c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:13:53.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.434+0000 7f3281200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562075309880 0x562075323f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.434+0000 7f327be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753244d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.435+0000 7f3281200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562075309880 0x562075323f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:38414/0 (socket says 172.21.15.28:38414) 2024-05-20T20:13:53.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.435+0000 7f3281200700 1 -- 172.21.15.28:0/2956713203 learned_addr learned my addr 172.21.15.28:0/2956713203 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:53.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.435+0000 7f3281c00700 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075320d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.436+0000 7f3281c00700 1 -- 172.21.15.28:0/2956713203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 msgr2=0x5620753244d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.436+0000 7f3281c00700 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753244d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.436+0000 7f3281c00700 1 -- 172.21.15.28:0/2956713203 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562075309880 msgr2=0x562075323f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:13:53.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.436+0000 7f3281c00700 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562075309880 0x562075323f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.437+0000 7f3281c00700 1 -- 172.21.15.28:0/2956713203 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562075321430 con 0x56207530f7d0 2024-05-20T20:13:53.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.440+0000 7f3281c00700 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075320d80 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f3274028fd0 tx=0x7f3274007610 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e26ec09b2096b834 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:53.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.440+0000 7f3262a00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:13:53.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.441+0000 7f3262a00700 1 -- 172.21.15.28:0/2956713203 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3274040070 con 0x56207530f7d0 2024-05-20T20:13:53.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.441+0000 7f3262a00700 1 -- 172.21.15.28:0/2956713203 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f3274027dd0 con 0x56207530f7d0 2024-05-20T20:13:53.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.441+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3274003d20 con 0x56207530f7d0 2024-05-20T20:13:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.442+0000 7f3262a00700 1 -- 172.21.15.28:0/2956713203 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f327402d040 con 0x56207530f7d0 2024-05-20T20:13:53.443 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.442+0000 7f328e633b80 10 client.16809 fetch_fsmap learned FSMap version 29 2024-05-20T20:13:53.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.442+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562075396b60 con 0x56207530f7d0 2024-05-20T20:13:53.444 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.443+0000 7f3262a00700 1 -- 172.21.15.28:0/2956713203 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3274003d20 con 0x56207530f7d0 2024-05-20T20:13:53.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.444+0000 7f3262a00700 1 -- 172.21.15.28:0/2956713203 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3274043030 con 0x56207530f7d0 2024-05-20T20:13:53.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.448+0000 7f328e633b80 10 client.16809 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:13:53.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.448+0000 7f328e633b80 10 client.16809 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:13:53.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.448+0000 7f328e633b80 20 client.16809 populate_metadata read hostname 'smithi028' 2024-05-20T20:13:53.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.448+0000 7f328e633b80 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562075396d30 0x562075399120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:53.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.448+0000 7f328e633b80 4 client.16809 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:13:53.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.448+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x562075263570 con 0x562075396d30 2024-05-20T20:13:53.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.448+0000 7f3281200700 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562075396d30 0x562075399120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:53.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.449+0000 7f3281200700 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562075396d30 0x562075399120 crc :-1 s=READY pgs=190 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f829022337f69973 server_cookie=bc9ae376c72e740d in_seq=0 out_seq=0 2024-05-20T20:13:53.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.449+0000 7f3262a00700 10 client.16809 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:53.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f3262a00700 1 -- 172.21.15.28:0/2956713203 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x562075263570 con 0x562075396d30 2024-05-20T20:13:53.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f3262a00700 10 client.16809 handle_command_reply: tid=0 2024-05-20T20:13:53.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 client.16809 shutdown 2024-05-20T20:13:53.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 msgr2=0x562075320d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075320d80 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f3274028fd0 tx=0x7f3274007610 comp rx=0 tx=0).stop 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 shutdown_connections 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562075396d30 0x562075399120 crc :-1 s=READY pgs=190 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56207530f7d0 0x562075320d80 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562075309880 0x562075323f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 --2- 172.21.15.28:0/2956713203 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56207530eeb0 0x5620753244d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 >> 172.21.15.28:0/2956713203 conn(0x562075237be0 msgr2=0x56207530fbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.450+0000 7f3262a00700 0 client.16809 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.451+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 shutdown_connections 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.451+0000 7f328e633b80 1 -- 172.21.15.28:0/2956713203 wait complete. 2024-05-20T20:13:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.451+0000 7f328e633b80 20 client.16809 trim_cache size 0 max 16384 2024-05-20T20:13:53.454 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:13:53.454 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:13:53.454 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:13:53.455 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f32640b6eb0 msgr2=0x7f32640b9370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f32640b6eb0 0x7f32640b9370 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f327402a000 tx=0x7f327402b000 comp rx=0 tx=0).stop 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f327c06bbe0 msgr2=0x7f327c06de10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f327c06bbe0 0x7f327c06de10 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f326800d750 tx=0x7f326800da60 comp rx=0 tx=0).stop 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 shutdown_connections 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f327c1095b0 0x7f327c06e350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f327c06b220 0x7f327c074d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f32640b6eb0 0x7f32640b9370 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 --2- 172.21.15.28:0/3704696737 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f327c06bbe0 0x7f327c06de10 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.458+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 >> 172.21.15.28:0/3704696737 conn(0x7f327c0793f0 msgr2=0x7f327c1010a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.459+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 shutdown_connections 2024-05-20T20:13:53.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:53.459+0000 7f3283c00700 1 -- 172.21.15.28:0/3704696737 wait complete. 2024-05-20T20:13:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:53 smithi154 ceph-mon[36314]: pgmap v254: 97 pgs: 97 active+clean; 52 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 267 MiB/s wr, 1.18k op/s 2024-05-20T20:13:53.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:53 smithi033 ceph-mon[36179]: pgmap v254: 97 pgs: 97 active+clean; 52 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 267 MiB/s wr, 1.18k op/s 2024-05-20T20:13:54.241 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:54 smithi028 ceph-mon[31178]: pgmap v255: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 366 MiB/s wr, 1.66k op/s 2024-05-20T20:13:54.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:54 smithi154 ceph-mon[36314]: pgmap v255: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 366 MiB/s wr, 1.66k op/s 2024-05-20T20:13:54.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:54 smithi033 ceph-mon[36179]: pgmap v255: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 366 MiB/s wr, 1.66k op/s 2024-05-20T20:13:54.876 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:13:54.877 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:13:55.855 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:13:56.598 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:56 smithi028 ceph-mon[31178]: pgmap v256: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 270 MiB/s wr, 1.17k op/s 2024-05-20T20:13:56.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:56 smithi154 ceph-mon[36314]: pgmap v256: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 270 MiB/s wr, 1.17k op/s 2024-05-20T20:13:56.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:56 smithi033 ceph-mon[36179]: pgmap v256: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 270 MiB/s wr, 1.17k op/s 2024-05-20T20:13:57.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.204+0000 7fb862200700 1 -- 172.21.15.28:0/1809452439 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 msgr2=0x7fb85c10bb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:57.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.204+0000 7fb862200700 1 --2- 172.21.15.28:0/1809452439 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 0x7fb85c10bb50 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fb84c009990 tx=0x7fb84c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.207+0000 7fb862200700 1 -- 172.21.15.28:0/1809452439 shutdown_connections 2024-05-20T20:13:57.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.207+0000 7fb862200700 1 --2- 172.21.15.28:0/1809452439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb85c106130 0x7fb85c112640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.207+0000 7fb862200700 1 --2- 172.21.15.28:0/1809452439 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 0x7fb85c10bb50 unknown :-1 s=CLOSED pgs=467 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.207+0000 7fb862200700 1 --2- 172.21.15.28:0/1809452439 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85c105770 0x7fb85c105bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.207+0000 7fb862200700 1 -- 172.21.15.28:0/1809452439 >> 172.21.15.28:0/1809452439 conn(0x7fb85c101260 msgr2=0x7fb85c103680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:57.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.209+0000 7fb862200700 1 -- 172.21.15.28:0/1809452439 shutdown_connections 2024-05-20T20:13:57.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.210+0000 7fb862200700 1 -- 172.21.15.28:0/1809452439 wait complete. 2024-05-20T20:13:57.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.215+0000 7fb862200700 1 Processor -- start 2024-05-20T20:13:57.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.215+0000 7fb862200700 1 -- start start 2024-05-20T20:13:57.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb862200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb85c105770 0x7fb85c11bc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb862200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85c106130 0x7fb85c11c170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb862200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 0x7fb85c111ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb862200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb85c079500 con 0x7fb85c106130 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb862200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb85c079380 con 0x7fb85c10b770 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb862200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb85c079680 con 0x7fb85c105770 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb853e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb85c105770 0x7fb85c11bc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb85be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 0x7fb85c111ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:57.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb853e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb85c105770 0x7fb85c11bc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33260/0 (socket says 172.21.15.28:33260) 2024-05-20T20:13:57.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.216+0000 7fb853e00700 1 -- 172.21.15.28:0/1885375661 learned_addr learned my addr 172.21.15.28:0/1885375661 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:57.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.217+0000 7fb85b400700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85c106130 0x7fb85c11c170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:57.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.217+0000 7fb85be00700 1 -- 172.21.15.28:0/1885375661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb85c105770 msgr2=0x7fb85c11bc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:57.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.217+0000 7fb85be00700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb85c105770 0x7fb85c11bc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.217+0000 7fb85be00700 1 -- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85c106130 msgr2=0x7fb85c11c170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:57.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.218+0000 7fb85be00700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85c106130 0x7fb85c11c170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.218+0000 7fb85be00700 1 -- 172.21.15.28:0/1885375661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb85c1123b0 con 0x7fb85c10b770 2024-05-20T20:13:57.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.218+0000 7fb85b400700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85c106130 0x7fb85c11c170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:13:57.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.219+0000 7fb85be00700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 0x7fb85c111ab0 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fb85400efc0 tx=0x7fb85400c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:57.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.219+0000 7fb858c00700 1 -- 172.21.15.28:0/1885375661 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8540092e0 con 0x7fb85c10b770 2024-05-20T20:13:57.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.219+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb85c112640 con 0x7fb85c10b770 2024-05-20T20:13:57.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.219+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb85c1179d0 con 0x7fb85c10b770 2024-05-20T20:13:57.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.220+0000 7fb858c00700 1 -- 172.21.15.28:0/1885375661 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb854004bb0 con 0x7fb85c10b770 2024-05-20T20:13:57.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.220+0000 7fb858c00700 1 -- 172.21.15.28:0/1885375661 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb854004020 con 0x7fb85c10b770 2024-05-20T20:13:57.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.223+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb85c06a230 con 0x7fb85c10b770 2024-05-20T20:13:57.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.229+0000 7fb858c00700 1 -- 172.21.15.28:0/1885375661 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb85401b030 con 0x7fb85c10b770 2024-05-20T20:13:57.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.234+0000 7fb858c00700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8400b2aa0 0x7fb8400b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:57.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.235+0000 7fb858c00700 1 -- 172.21.15.28:0/1885375661 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7fb8540e2100 con 0x7fb85c10b770 2024-05-20T20:13:57.239 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.239+0000 7fb858c00700 1 -- 172.21.15.28:0/1885375661 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fb854022020 con 0x7fb85c10b770 2024-05-20T20:13:57.240 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.240+0000 7fb853e00700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8400b2aa0 0x7fb8400b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:57.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.241+0000 7fb853e00700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8400b2aa0 0x7fb8400b4f60 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fb84c009750 tx=0x7fb84c005dc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:57.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.825+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7fb85c1184c0 con 0x7fb85c10b770 2024-05-20T20:13:57.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.826+0000 7fb858c00700 1 -- 172.21.15.28:0/1885375661 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fb854010040 con 0x7fb85c10b770 2024-05-20T20:13:57.827 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:13:57.827 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:13:57.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8400b2aa0 msgr2=0x7fb8400b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:57.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8400b2aa0 0x7fb8400b4f60 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fb84c009750 tx=0x7fb84c005dc0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 msgr2=0x7fb85c111ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:57.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 0x7fb85c111ab0 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fb85400efc0 tx=0x7fb85400c3a0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 shutdown_connections 2024-05-20T20:13:57.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb8400b2aa0 0x7fb8400b4f60 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb85c10b770 0x7fb85c111ab0 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb85c105770 0x7fb85c11bc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 --2- 172.21.15.28:0/1885375661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb85c106130 0x7fb85c11c170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:57.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.830+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 >> 172.21.15.28:0/1885375661 conn(0x7fb85c101260 msgr2=0x7fb85c10aa30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:57.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.831+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 shutdown_connections 2024-05-20T20:13:57.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:57.831+0000 7fb862200700 1 -- 172.21.15.28:0/1885375661 wait complete. 2024-05-20T20:13:57.838 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:13:58.057 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:57 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1885375661' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:58.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:57 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1885375661' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:58.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:57 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1885375661' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:13:58.983 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id c71942f8 --release --await' 2024-05-20T20:13:58.983 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id c71942f8 --release --await 2024-05-20T20:13:59.278 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:13:58 smithi033 ceph-mon[36179]: pgmap v257: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 291 MiB/s wr, 1.32k op/s 2024-05-20T20:13:59.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:13:58 smithi028 ceph-mon[31178]: pgmap v257: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 291 MiB/s wr, 1.32k op/s 2024-05-20T20:13:59.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:13:58 smithi154 ceph-mon[36314]: pgmap v257: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 291 MiB/s wr, 1.32k op/s 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 -- 172.21.15.28:0/1011457589 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601041a0 msgr2=0x7f82601081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 --2- 172.21.15.28:0/1011457589 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601041a0 0x7f82601081c0 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f824c009920 tx=0x7f824c009c30 comp rx=0 tx=0).stop 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 -- 172.21.15.28:0/1011457589 shutdown_connections 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 --2- 172.21.15.28:0/1011457589 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601088f0 0x7f8260110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 --2- 172.21.15.28:0/1011457589 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82601037f0 0x7f8260103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 --2- 172.21.15.28:0/1011457589 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601041a0 0x7f82601081c0 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 -- 172.21.15.28:0/1011457589 >> 172.21.15.28:0/1011457589 conn(0x7f826007a6a0 msgr2=0x7f826007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:13:59.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 -- 172.21.15.28:0/1011457589 shutdown_connections 2024-05-20T20:13:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.724+0000 7f8265a00700 1 -- 172.21.15.28:0/1011457589 wait complete. 2024-05-20T20:13:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.725+0000 7f8265a00700 1 Processor -- start 2024-05-20T20:13:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.725+0000 7f8265a00700 1 -- start start 2024-05-20T20:13:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f8265a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82601037f0 0x7f826019be50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:59.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f8265a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601041a0 0x7f826019c390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f8265a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601088f0 0x7f82601a0720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f8265a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8260113700 con 0x7f82601088f0 2024-05-20T20:13:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f8265a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8260113580 con 0x7f82601041a0 2024-05-20T20:13:59.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f8265a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8260113880 con 0x7f82601037f0 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825e000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601041a0 0x7f826019c390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825e000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601041a0 0x7f826019c390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49290/0 (socket says 172.21.15.28:49290) 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825e000700 1 -- 172.21.15.28:0/4057293613 learned_addr learned my addr 172.21.15.28:0/4057293613 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825ea00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82601037f0 0x7f826019be50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825e000700 1 -- 172.21.15.28:0/4057293613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82601037f0 msgr2=0x7f826019be50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825e000700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82601037f0 0x7f826019be50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825e000700 1 -- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601088f0 msgr2=0x7f82601a0720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.726+0000 7f825f400700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601088f0 0x7f82601a0720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f825e000700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601088f0 0x7f82601a0720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f825e000700 1 -- 172.21.15.28:0/4057293613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f82601a0e60 con 0x7f82601041a0 2024-05-20T20:13:59.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f825f400700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601088f0 0x7f82601a0720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:13:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f825e000700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601041a0 0x7f826019c390 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f824c000c00 tx=0x7f824c00f990 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f8257400700 1 -- 172.21.15.28:0/4057293613 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f824c027070 con 0x7f82601041a0 2024-05-20T20:13:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f8257400700 1 -- 172.21.15.28:0/4057293613 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f824c0055f0 con 0x7f82601041a0 2024-05-20T20:13:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f82601a10f0 con 0x7f82601041a0 2024-05-20T20:13:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f8257400700 1 -- 172.21.15.28:0/4057293613 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f824c004930 con 0x7f82601041a0 2024-05-20T20:13:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.727+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f82601a89d0 con 0x7f82601041a0 2024-05-20T20:13:59.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.729+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8260104c80 con 0x7f82601041a0 2024-05-20T20:13:59.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.732+0000 7f8257400700 1 -- 172.21.15.28:0/4057293613 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f824c02a070 con 0x7f82601041a0 2024-05-20T20:13:59.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.734+0000 7f8257400700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82440b29f0 0x7f82440b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:13:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.734+0000 7f8257400700 1 -- 172.21.15.28:0/4057293613 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f824c0e2ff0 con 0x7f82601041a0 2024-05-20T20:13:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.734+0000 7f825ea00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82440b29f0 0x7f82440b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:13:59.735 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.735+0000 7f825ea00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82440b29f0 0x7f82440b4eb0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f8250009680 tx=0x7f8250009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:13:59.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:13:59.739+0000 7f8257400700 1 -- 172.21.15.28:0/4057293613 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f824c05fdf0 con 0x7f82601041a0 2024-05-20T20:13:59.768 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:00.307 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.306+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f826006a230 con 0x7f82601041a0 2024-05-20T20:14:00.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.311+0000 7f8257400700 1 -- 172.21.15.28:0/4057293613 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f826006a230 con 0x7f82601041a0 2024-05-20T20:14:00.314 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:14:00.314 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:14:00.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.317+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82440b29f0 msgr2=0x7f82440b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:00.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.317+0000 7f8265a00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82440b29f0 0x7f82440b4eb0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f8250009680 tx=0x7f8250009260 comp rx=0 tx=0).stop 2024-05-20T20:14:00.318 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.317+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601041a0 msgr2=0x7f826019c390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:00.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.317+0000 7f8265a00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601041a0 0x7f826019c390 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f824c000c00 tx=0x7f824c00f990 comp rx=0 tx=0).stop 2024-05-20T20:14:00.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.318+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 shutdown_connections 2024-05-20T20:14:00.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.318+0000 7f8265a00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f82601088f0 0x7f82601a0720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:00.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.318+0000 7f8265a00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82601037f0 0x7f826019be50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:00.319 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.318+0000 7f8265a00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82440b29f0 0x7f82440b4eb0 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:00.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.318+0000 7f8265a00700 1 --2- 172.21.15.28:0/4057293613 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f82601041a0 0x7f826019c390 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:00.320 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.318+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 >> 172.21.15.28:0/4057293613 conn(0x7f826007a6a0 msgr2=0x7f8260101680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:00.321 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.320+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 shutdown_connections 2024-05-20T20:14:00.322 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:00.321+0000 7f8265a00700 1 -- 172.21.15.28:0/4057293613 wait complete. 2024-05-20T20:14:00.327 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:14:01.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.139+0000 7f756a800640 1 -- 172.21.15.28:0/1120722602 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f75641019f0 msgr2=0x7f7564101dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.139+0000 7f756a800640 1 --2- 172.21.15.28:0/1120722602 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f75641019f0 0x7f7564101dd0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f755c009ca0 tx=0x7f755c02a940 comp rx=0 tx=0).stop 2024-05-20T20:14:01.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.140+0000 7f756a800640 1 -- 172.21.15.28:0/1120722602 shutdown_connections 2024-05-20T20:14:01.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.140+0000 7f756a800640 1 --2- 172.21.15.28:0/1120722602 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 0x7f75641067d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.141+0000 7f756a800640 1 --2- 172.21.15.28:0/1120722602 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7564106df0 0x7f7564107280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.142 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.141+0000 7f756a800640 1 --2- 172.21.15.28:0/1120722602 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f75641019f0 0x7f7564101dd0 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.141+0000 7f756a800640 1 -- 172.21.15.28:0/1120722602 >> 172.21.15.28:0/1120722602 conn(0x7f75640fd420 msgr2=0x7f75640ff840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:01.143 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.142+0000 7f756a800640 1 -- 172.21.15.28:0/1120722602 shutdown_connections 2024-05-20T20:14:01.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.142+0000 7f756a800640 1 -- 172.21.15.28:0/1120722602 wait complete. 2024-05-20T20:14:01.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.143+0000 7f756a800640 1 Processor -- start 2024-05-20T20:14:01.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.145+0000 7f756a800640 1 -- start start 2024-05-20T20:14:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f756a800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f75641019f0 0x7f7564115c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f756a800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 0x7f756410fc80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f756a800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7564106df0 0x7f75641101c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f756a800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f75641086a0 con 0x7f75641019f0 2024-05-20T20:14:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f756a800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7564108520 con 0x7f7564102310 2024-05-20T20:14:01.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f756a800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7564108820 con 0x7f7564106df0 2024-05-20T20:14:01.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f7562a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 0x7f756410fc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f7562a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 0x7f756410fc80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49316/0 (socket says 172.21.15.28:49316) 2024-05-20T20:14:01.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f7562a00640 1 -- 172.21.15.28:0/2570519635 learned_addr learned my addr 172.21.15.28:0/2570519635 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:01.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f7563e00640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7564106df0 0x7f75641101c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.146+0000 7f7563400640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f75641019f0 0x7f7564115c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.147+0000 7f7562a00640 1 -- 172.21.15.28:0/2570519635 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7564106df0 msgr2=0x7f75641101c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.147+0000 7f7562a00640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7564106df0 0x7f75641101c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.147+0000 7f7562a00640 1 -- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f75641019f0 msgr2=0x7f7564115c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.147+0000 7f7562a00640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f75641019f0 0x7f7564115c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.147+0000 7f7562a00640 1 -- 172.21.15.28:0/2570519635 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f75641107b0 con 0x7f7564102310 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.147+0000 7f7562a00640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 0x7f756410fc80 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f755000d770 tx=0x7f755000dc30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.148+0000 7f755be00640 1 -- 172.21.15.28:0/2570519635 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7550017070 con 0x7f7564102310 2024-05-20T20:14:01.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.148+0000 7f755be00640 1 -- 172.21.15.28:0/2570519635 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f755000de80 con 0x7f7564102310 2024-05-20T20:14:01.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.148+0000 7f755be00640 1 -- 172.21.15.28:0/2570519635 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f755000f620 con 0x7f7564102310 2024-05-20T20:14:01.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.148+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7564110aa0 con 0x7f7564102310 2024-05-20T20:14:01.150 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.148+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f75641b4a70 con 0x7f7564102310 2024-05-20T20:14:01.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.154+0000 7f755be00640 1 -- 172.21.15.28:0/2570519635 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7550009c30 con 0x7f7564102310 2024-05-20T20:14:01.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.156+0000 7f756d62b740 1 Processor -- start 2024-05-20T20:14:01.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.157+0000 7f755be00640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75480b2ae0 0x7f75480b4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.157+0000 7f755be00640 1 -- 172.21.15.28:0/2570519635 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f75500e2140 con 0x7f7564102310 2024-05-20T20:14:01.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f756d62b740 1 -- start start 2024-05-20T20:14:01.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f7563e00640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75480b2ae0 0x7f75480b4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f756d62b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc3c070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f756d62b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3c5b0 0x5596fbc46930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f756d62b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc49260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f756d62b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5596fbbacb90 con 0x5596fbc3bc90 2024-05-20T20:14:01.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f756d62b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5596fbb9c730 con 0x5596fbc3c5b0 2024-05-20T20:14:01.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.158+0000 7f756d62b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5596fbba0770 con 0x5596fbc46e70 2024-05-20T20:14:01.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.159+0000 7f7563400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc3c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.159+0000 7f7563400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc3c070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58496/0 (socket says 172.21.15.28:58496) 2024-05-20T20:14:01.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.159+0000 7f7563400640 1 -- 172.21.15.28:0/3921902597 learned_addr learned my addr 172.21.15.28:0/3921902597 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:01.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.160+0000 7f7562a00640 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3c5b0 0x5596fbc46930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.161 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.160+0000 7f7563e00640 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc49260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.161 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.160+0000 7f7563400640 1 -- 172.21.15.28:0/3921902597 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 msgr2=0x5596fbc49260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:01.161 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.160+0000 7f7563400640 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc49260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.161 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.160+0000 7f7563400640 1 -- 172.21.15.28:0/3921902597 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3c5b0 msgr2=0x5596fbc46930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:01.161 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.160+0000 7f7563400640 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3c5b0 0x5596fbc46930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.161 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.160+0000 7f7563400640 1 -- 172.21.15.28:0/3921902597 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5596fbb358b0 con 0x5596fbc3bc90 2024-05-20T20:14:01.166 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.161+0000 7f7563400640 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc3c070 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f755c009ca0 tx=0x7f755c02f440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9bc5b5393316fb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:01.166 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.161+0000 7f7563e00640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75480b2ae0 0x7f75480b4fa0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f755400ab10 tx=0x7f755400a250 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:01.166 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.161+0000 7f7558c00640 1 -- 172.21.15.28:0/3921902597 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f755c034040 con 0x5596fbc3bc90 2024-05-20T20:14:01.166 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.162+0000 7f7558c00640 1 -- 172.21.15.28:0/3921902597 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f755c003e20 con 0x5596fbc3bc90 2024-05-20T20:14:01.166 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.162+0000 7f7558c00640 1 -- 172.21.15.28:0/3921902597 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f755c034040 con 0x5596fbc3bc90 2024-05-20T20:14:01.169 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 -- 172.21.15.28:0/3921902597 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 msgr2=0x5596fbc3c070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.169 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc3c070 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f755c009ca0 tx=0x7f755c02f440 comp rx=0 tx=0).stop 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 -- 172.21.15.28:0/3921902597 shutdown_connections 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3c5b0 0x5596fbc46930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc49260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 --2- 172.21.15.28:0/3921902597 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc3c070 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 -- 172.21.15.28:0/3921902597 >> 172.21.15.28:0/3921902597 conn(0x5596fbc381a0 msgr2=0x5596fbc38580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 -- 172.21.15.28:0/3921902597 shutdown_connections 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.163+0000 7f756d62b740 1 -- 172.21.15.28:0/3921902597 wait complete. 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.164+0000 7f756d62b740 1 -- start start 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.164+0000 7f756d62b740 1 Processor -- start 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f756d62b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3bc90 0x5596fbc467a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f756d62b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f756d62b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc43fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f756d62b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5596fb882790 con 0x5596fbc3c5b0 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f756d62b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5596fbb44bc0 con 0x5596fbc3bc90 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f756d62b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5596fbc49a90 con 0x5596fbc46e70 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f7563400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3bc90 0x5596fbc467a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f7563400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3bc90 0x5596fbc467a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49342/0 (socket says 172.21.15.28:49342) 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f7563400640 1 -- 172.21.15.28:0/717339855 learned_addr learned my addr 172.21.15.28:0/717339855 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f7563e00640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc43fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.165+0000 7f756d62b740 10 client.0 _ll_register_callbacks cb 0x5596fbb22400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.166+0000 7f7562a00640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.166+0000 7f7563e00640 1 -- 172.21.15.28:0/717339855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3bc90 msgr2=0x5596fbc467a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.166+0000 7f7563e00640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3bc90 0x5596fbc467a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.166+0000 7f7563e00640 1 -- 172.21.15.28:0/717339855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 msgr2=0x5596fbc43a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:01.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.166+0000 7f7563e00640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.166+0000 7f7563e00640 1 -- 172.21.15.28:0/717339855 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5596fb8b1210 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.166+0000 7f7562a00640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.167+0000 7f7563e00640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc43fd0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x5596fbc39330 tx=0x7f7554004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=575f41880012e988 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.167+0000 7f754d600640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.167+0000 7f754d600640 1 -- 172.21.15.28:0/717339855 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7554016070 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.167+0000 7f754d600640 1 -- 172.21.15.28:0/717339855 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f7554010d90 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.168+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f755c003e20 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.168+0000 7f754d600640 1 -- 172.21.15.28:0/717339855 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f755401ce10 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.168+0000 7f756d62b740 10 client.27115 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.168+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5596fb97d620 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.168+0000 7f754d600640 1 -- 172.21.15.28:0/717339855 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f755c003e20 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.169+0000 7f754d600640 1 -- 172.21.15.28:0/717339855 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f755401a560 con 0x5596fbc46e70 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.169+0000 7f756d62b740 10 client.27115 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:01.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.169+0000 7f756d62b740 10 client.27115 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:01.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.169+0000 7f756d62b740 20 client.27115 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:01.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.169+0000 7f756d62b740 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbc56820 0x5596fbc58c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.169+0000 7f756d62b740 4 client.27115 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:01.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.169+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5596fbbaeae0 con 0x5596fbc56820 2024-05-20T20:14:01.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.170+0000 7f7563400640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbc56820 0x5596fbc58c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.170+0000 7f7563400640 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbc56820 0x5596fbc58c10 crc :-1 s=READY pgs=191 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=97baee0203d639c2 server_cookie=7b9145fe0ed0fc6a in_seq=0 out_seq=0 2024-05-20T20:14:01.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.170+0000 7f754d600640 10 client.27115 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:01.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.171+0000 7f754d600640 1 -- 172.21.15.28:0/717339855 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5596fbbaeae0 con 0x5596fbc56820 2024-05-20T20:14:01.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.171+0000 7f754d600640 10 client.27115 handle_command_reply: tid=0 2024-05-20T20:14:01.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.171+0000 7f756d62b740 1 client.27115 shutdown 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 msgr2=0x5596fbc43fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc43fd0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x5596fbc39330 tx=0x7f7554004730 comp rx=0 tx=0).stop 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 shutdown_connections 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbc56820 0x5596fbc58c10 crc :-1 s=READY pgs=191 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc43fd0 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 --2- 172.21.15.28:0/717339855 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc3bc90 0x5596fbc467a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 >> 172.21.15.28:0/717339855 conn(0x5596fbc381a0 msgr2=0x5596fbc3df20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f754d600640 0 client.27115 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 shutdown_connections 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 1 -- 172.21.15.28:0/717339855 wait complete. 2024-05-20T20:14:01.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.172+0000 7f756d62b740 20 client.27115 trim_cache size 0 max 16384 2024-05-20T20:14:01.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.206+0000 7f756d62b740 1 Processor -- start 2024-05-20T20:14:01.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.206+0000 7f756d62b740 1 -- start start 2024-05-20T20:14:01.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f756d62b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc429d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f756d62b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f756d62b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc46e70 0x5596fbc52af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f756d62b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f75641bd450 con 0x5596fbc3bc90 2024-05-20T20:14:01.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f756d62b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f75641bef90 con 0x5596fbc46e70 2024-05-20T20:14:01.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f756d62b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f75641bdb20 con 0x5596fbc3c5b0 2024-05-20T20:14:01.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f7563e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc46e70 0x5596fbc52af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f7563400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc429d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f7563e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc46e70 0x5596fbc52af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:49354/0 (socket says 172.21.15.28:49354) 2024-05-20T20:14:01.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f7563e00640 1 -- 172.21.15.28:0/2134163779 learned_addr learned my addr 172.21.15.28:0/2134163779 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:01.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.207+0000 7f7563400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc429d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58516/0 (socket says 172.21.15.28:58516) 2024-05-20T20:14:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.208+0000 7f7562a00640 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.209+0000 7f7563400640 1 -- 172.21.15.28:0/2134163779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc3c5b0 msgr2=0x5596fbc43eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:01.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.209+0000 7f7563400640 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.209+0000 7f7563400640 1 -- 172.21.15.28:0/2134163779 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc46e70 msgr2=0x5596fbc52af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.209+0000 7f7563400640 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc46e70 0x5596fbc52af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.209+0000 7f7563400640 1 -- 172.21.15.28:0/2134163779 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5596fbc59290 con 0x5596fbc3bc90 2024-05-20T20:14:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.209+0000 7f7562a00640 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:01.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.210+0000 7f7563400640 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc429d0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x5596fbc59510 tx=0x7f755c02feb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=814a77767aad8fc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.210+0000 7f754fe00640 1 -- 172.21.15.28:0/2134163779 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f755c034040 con 0x5596fbc3bc90 2024-05-20T20:14:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.211+0000 7f754fe00640 1 -- 172.21.15.28:0/2134163779 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f755c052070 con 0x5596fbc3bc90 2024-05-20T20:14:01.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.211+0000 7f754fe00640 1 -- 172.21.15.28:0/2134163779 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f755c04d2c0 con 0x5596fbc3bc90 2024-05-20T20:14:01.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.214+0000 7f756d62b740 1 -- 172.21.15.28:0/2134163779 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 msgr2=0x5596fbc429d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.214 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.214+0000 7f756d62b740 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc429d0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x5596fbc59510 tx=0x7f755c02feb0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.214+0000 7f756d62b740 1 -- 172.21.15.28:0/2134163779 shutdown_connections 2024-05-20T20:14:01.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.214+0000 7f756d62b740 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbc46e70 0x5596fbc52af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.214+0000 7f756d62b740 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc3c5b0 0x5596fbc43eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.214+0000 7f756d62b740 1 --2- 172.21.15.28:0/2134163779 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3bc90 0x5596fbc429d0 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.214+0000 7f756d62b740 1 -- 172.21.15.28:0/2134163779 >> 172.21.15.28:0/2134163779 conn(0x5596fbc381a0 msgr2=0x5596fbc37310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:01.215 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.215+0000 7f756d62b740 1 -- 172.21.15.28:0/2134163779 shutdown_connections 2024-05-20T20:14:01.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.215+0000 7f756d62b740 1 -- 172.21.15.28:0/2134163779 wait complete. 2024-05-20T20:14:01.216 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.216+0000 7f756d62b740 1 -- start start 2024-05-20T20:14:01.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.216+0000 7f756d62b740 1 Processor -- start 2024-05-20T20:14:01.220 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.217+0000 7f756d62b740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc4c150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.220 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.217+0000 7f756d62b740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc4c690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.220 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.217+0000 7f756d62b740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbcbd430 0x5596fbcbf820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.218+0000 7f756d62b740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f75641be8c0 con 0x5596fbc3c5b0 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.218+0000 7f756d62b740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f75641be1f0 con 0x5596fbcbd430 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.218+0000 7f756d62b740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f75641bb450 con 0x5596fbc46e70 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.218+0000 7f756d62b740 10 client.0 _ll_register_callbacks cb 0x7f755400a7d0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.218+0000 7f7563400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc4c150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.218+0000 7f7563400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc4c150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58520/0 (socket says 172.21.15.28:58520) 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.218+0000 7f7563400640 1 -- 172.21.15.28:0/675668708 learned_addr learned my addr 172.21.15.28:0/675668708 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.219+0000 7f7563400640 1 -- 172.21.15.28:0/675668708 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 msgr2=0x5596fbc4c690 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.219+0000 7f7563400640 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc4c690 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.219+0000 7f7563400640 1 -- 172.21.15.28:0/675668708 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbcbd430 msgr2=0x5596fbcbf820 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.219+0000 7f7563400640 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbcbd430 0x5596fbcbf820 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.219+0000 7f7563400640 1 -- 172.21.15.28:0/675668708 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5596fbcbfd60 con 0x5596fbc3c5b0 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.220+0000 7f754d600640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:14:01.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.220+0000 7f7563400640 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc4c150 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f755c006010 tx=0x7f755c003a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7553f92662c67939 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:01.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.220+0000 7f754d600640 1 -- 172.21.15.28:0/675668708 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f755c034040 con 0x5596fbc3c5b0 2024-05-20T20:14:01.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.221+0000 7f754d600640 1 -- 172.21.15.28:0/675668708 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f755c05b020 con 0x5596fbc3c5b0 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.221+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f755c052070 con 0x5596fbc3c5b0 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.221+0000 7f754d600640 1 -- 172.21.15.28:0/675668708 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f755c034040 con 0x5596fbc3c5b0 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.221+0000 7f756d62b740 10 client.16830 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.221+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5596fbcc0fc0 con 0x5596fbc3c5b0 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f754d600640 1 -- 172.21.15.28:0/675668708 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f755c030640 con 0x5596fbc3c5b0 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f754d600640 1 -- 172.21.15.28:0/675668708 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f755c042020 con 0x5596fbc3c5b0 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f756d62b740 10 client.16830 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f756d62b740 10 client.16830 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:01.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f756d62b740 20 client.16830 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:01.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f756d62b740 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbcc16d0 0x5596fbcc3ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:01.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f756d62b740 4 client.16830 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:14:01.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.222+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x7f755c052070 con 0x5596fbcc16d0 2024-05-20T20:14:01.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.224+0000 7f7562a00640 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbcc16d0 0x5596fbcc3ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:01.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.226+0000 7f754d600640 10 client.16830 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:01.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.227+0000 7f7562a00640 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbcc16d0 0x5596fbcc3ae0 crc :-1 s=READY pgs=192 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ca89576f53747353 server_cookie=b0e671770eaf740b in_seq=0 out_seq=0 2024-05-20T20:14:01.249 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.249+0000 7f754d600640 1 -- 172.21.15.28:0/675668708 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x7f755c052070 con 0x5596fbcc16d0 2024-05-20T20:14:01.250 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.249+0000 7f754d600640 10 client.16830 handle_command_reply: tid=0 2024-05-20T20:14:01.251 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.250+0000 7f756d62b740 1 client.16830 shutdown 2024-05-20T20:14:01.251 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.250+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 msgr2=0x5596fbc4c150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.251 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.250+0000 7f756d62b740 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc4c150 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f755c006010 tx=0x7f755c003a00 comp rx=0 tx=0).stop 2024-05-20T20:14:01.252 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.252+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 shutdown_connections 2024-05-20T20:14:01.252 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.252+0000 7f756d62b740 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5596fbcc16d0 0x5596fbcc3ae0 crc :-1 s=READY pgs=192 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.252 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.252+0000 7f756d62b740 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596fbcbd430 0x5596fbcbf820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.253 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.252+0000 7f756d62b740 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596fbc46e70 0x5596fbc4c690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.253 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.252+0000 7f754d600640 0 client.16830 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:01.253 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.252+0000 7f756d62b740 1 --2- 172.21.15.28:0/675668708 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596fbc3c5b0 0x5596fbc4c150 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.253 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.252+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 >> 172.21.15.28:0/675668708 conn(0x5596fbc381a0 msgr2=0x5596fbc51ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:01.253 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.253+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 shutdown_connections 2024-05-20T20:14:01.253 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.253+0000 7f756d62b740 1 -- 172.21.15.28:0/675668708 wait complete. 2024-05-20T20:14:01.254 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.254+0000 7f756d62b740 20 client.16830 trim_cache size 0 max 16384 2024-05-20T20:14:01.255 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:14:01.255 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:14:01.255 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:14:01.255 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75480b2ae0 msgr2=0x7f75480b4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.255 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75480b2ae0 0x7f75480b4fa0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f755400ab10 tx=0x7f755400a250 comp rx=0 tx=0).stop 2024-05-20T20:14:01.255 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 msgr2=0x7f756410fc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:01.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 0x7f756410fc80 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f755000d770 tx=0x7f755000dc30 comp rx=0 tx=0).stop 2024-05-20T20:14:01.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 shutdown_connections 2024-05-20T20:14:01.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7564102310 0x7f756410fc80 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f75480b2ae0 0x7f75480b4fa0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7564106df0 0x7f75641101c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 --2- 172.21.15.28:0/2570519635 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f75641019f0 0x7f7564115c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:01.256 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.255+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 >> 172.21.15.28:0/2570519635 conn(0x7f75640fd420 msgr2=0x7f75640fe350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:01.257 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.256+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 shutdown_connections 2024-05-20T20:14:01.257 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:01.256+0000 7f756a800640 1 -- 172.21.15.28:0/2570519635 wait complete. 2024-05-20T20:14:01.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:01 smithi028 ceph-mon[31178]: pgmap v258: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 368 MiB/s wr, 2.02k op/s 2024-05-20T20:14:01.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:01 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/4057293613' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:14:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:01 smithi154 ceph-mon[36314]: pgmap v258: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 368 MiB/s wr, 2.02k op/s 2024-05-20T20:14:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:01 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/4057293613' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:14:01.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:01 smithi033 ceph-mon[36179]: pgmap v258: 97 pgs: 97 active+clean; 53 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 368 MiB/s wr, 2.02k op/s 2024-05-20T20:14:01.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:01 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/4057293613' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:14:02.147 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:14:02.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.900+0000 7f9215200700 1 -- 172.21.15.28:0/3588104266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 msgr2=0x7f921011ae70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:02.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.900+0000 7f9215200700 1 --2- 172.21.15.28:0/3588104266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 0x7f921011ae70 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f9208009f90 tx=0x7f920800b330 comp rx=0 tx=0).stop 2024-05-20T20:14:02.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.902+0000 7f9215200700 1 -- 172.21.15.28:0/3588104266 shutdown_connections 2024-05-20T20:14:02.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.902+0000 7f9215200700 1 --2- 172.21.15.28:0/3588104266 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 0x7f921011ae70 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.902+0000 7f9215200700 1 --2- 172.21.15.28:0/3588104266 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f921010a540 0x7f921010a920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.902+0000 7f9215200700 1 --2- 172.21.15.28:0/3588104266 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f921010ae60 0x7f9210118540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.902+0000 7f9215200700 1 -- 172.21.15.28:0/3588104266 >> 172.21.15.28:0/3588104266 conn(0x7f921006e6b0 msgr2=0x7f921006eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:02.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.902+0000 7f9215200700 1 -- 172.21.15.28:0/3588104266 shutdown_connections 2024-05-20T20:14:02.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.903+0000 7f9215200700 1 -- 172.21.15.28:0/3588104266 wait complete. 2024-05-20T20:14:02.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.904+0000 7f9215200700 1 Processor -- start 2024-05-20T20:14:02.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.904+0000 7f9215200700 1 -- start start 2024-05-20T20:14:02.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.905+0000 7f9215200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f921010a540 0x7f921010e1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.905+0000 7f9215200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f921010ae60 0x7f921010e700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.905+0000 7f9215200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 0x7f921010ec40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.905+0000 7f9215200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f921011e1b0 con 0x7f9210118a80 2024-05-20T20:14:02.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.905+0000 7f9215200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f921011e030 con 0x7f921010ae60 2024-05-20T20:14:02.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.905+0000 7f9215200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f921011e330 con 0x7f921010a540 2024-05-20T20:14:02.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.906+0000 7f920de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f921010a540 0x7f921010e1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.907+0000 7f920d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f921010ae60 0x7f921010e700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.907+0000 7f920de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f921010a540 0x7f921010e1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33354/0 (socket says 172.21.15.28:33354) 2024-05-20T20:14:02.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.907+0000 7f920de00700 1 -- 172.21.15.28:0/1532202187 learned_addr learned my addr 172.21.15.28:0/1532202187 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:02.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.908+0000 7f920e800700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 0x7f921010ec40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.909+0000 7f920d400700 1 -- 172.21.15.28:0/1532202187 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f921010a540 msgr2=0x7f921010e1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:02.910 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.909+0000 7f920d400700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f921010a540 0x7f921010e1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.909+0000 7f920d400700 1 -- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 msgr2=0x7f921010ec40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:02.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.909+0000 7f920d400700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 0x7f921010ec40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.909+0000 7f920d400700 1 -- 172.21.15.28:0/1532202187 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f92101b4c90 con 0x7f921010ae60 2024-05-20T20:14:02.911 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.911+0000 7f920d400700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f921010ae60 0x7f921010e700 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f91fc00c9a0 tx=0x7f91fc00ccb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:02.912 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.911+0000 7f9206a00700 1 -- 172.21.15.28:0/1532202187 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f91fc014070 con 0x7f921010ae60 2024-05-20T20:14:02.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.912+0000 7f9206a00700 1 -- 172.21.15.28:0/1532202187 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f91fc004440 con 0x7f921010ae60 2024-05-20T20:14:02.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.912+0000 7f9206a00700 1 -- 172.21.15.28:0/1532202187 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f91fc00ea30 con 0x7f921010ae60 2024-05-20T20:14:02.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.914+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f92101b4f80 con 0x7f921010ae60 2024-05-20T20:14:02.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.914+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f92101b53e0 con 0x7f921010ae60 2024-05-20T20:14:02.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.925+0000 7f921faacb80 1 Processor -- start 2024-05-20T20:14:02.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.928+0000 7f9206a00700 1 -- 172.21.15.28:0/1532202187 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f91fc017070 con 0x7f921010ae60 2024-05-20T20:14:02.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.928+0000 7f921faacb80 1 -- start start 2024-05-20T20:14:02.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.929+0000 7f921faacb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799aeff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.930+0000 7f921faacb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799af530 0x5583799a8fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.930+0000 7f921faacb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a9a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.930+0000 7f921faacb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558379906f40 con 0x5583799aec10 2024-05-20T20:14:02.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.930+0000 7f921faacb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5583798f6ae0 con 0x5583799a95f0 2024-05-20T20:14:02.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.930+0000 7f921faacb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5583798fab20 con 0x5583799af530 2024-05-20T20:14:02.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.931+0000 7f920e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799aeff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.931+0000 7f920e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799aeff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58552/0 (socket says 172.21.15.28:58552) 2024-05-20T20:14:02.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.931+0000 7f920e800700 1 -- 172.21.15.28:0/3659537931 learned_addr learned my addr 172.21.15.28:0/3659537931 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:02.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.931+0000 7f9206a00700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f91f40b2c00 0x7f91f40b50c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.932+0000 7f920d400700 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a9a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.932+0000 7f920d400700 1 -- 172.21.15.28:0/3659537931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799af530 msgr2=0x5583799a8fd0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:02.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.932+0000 7f920d400700 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799af530 0x5583799a8fd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.932+0000 7f920d400700 1 -- 172.21.15.28:0/3659537931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 msgr2=0x5583799aeff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:02.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.932+0000 7f920d400700 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799aeff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.932+0000 7f920d400700 1 -- 172.21.15.28:0/3659537931 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5583798bf520 con 0x5583799a95f0 2024-05-20T20:14:02.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.932+0000 7f920d400700 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a9a50 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f91fc0df4f0 tx=0x7f91fc0a9950 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ebadd87c160bcbba server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:02.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.933+0000 7f9206a00700 1 -- 172.21.15.28:0/1532202187 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f91fc0e1e10 con 0x7f921010ae60 2024-05-20T20:14:02.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.933+0000 7f91fb400700 1 -- 172.21.15.28:0/3659537931 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f91fc0b1f60 con 0x5583799a95f0 2024-05-20T20:14:02.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.933+0000 7f91fb400700 1 -- 172.21.15.28:0/3659537931 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f91fc003c50 con 0x5583799a95f0 2024-05-20T20:14:02.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.933+0000 7f91fb400700 1 -- 172.21.15.28:0/3659537931 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f91fc0a95c0 con 0x5583799a95f0 2024-05-20T20:14:02.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.935+0000 7f921faacb80 1 -- 172.21.15.28:0/3659537931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 msgr2=0x5583799a9a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:02.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.935+0000 7f921faacb80 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a9a50 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7f91fc0df4f0 tx=0x7f91fc0a9950 comp rx=0 tx=0).stop 2024-05-20T20:14:02.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.935+0000 7f921faacb80 1 -- 172.21.15.28:0/3659537931 shutdown_connections 2024-05-20T20:14:02.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.935+0000 7f921faacb80 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a9a50 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.935+0000 7f921faacb80 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799aeff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.935+0000 7f921faacb80 1 --2- 172.21.15.28:0/3659537931 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799af530 0x5583799a8fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.935+0000 7f921faacb80 1 -- 172.21.15.28:0/3659537931 >> 172.21.15.28:0/3659537931 conn(0x5583798d6ce0 msgr2=0x5583799a3f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:02.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.936+0000 7f921faacb80 1 -- 172.21.15.28:0/3659537931 shutdown_connections 2024-05-20T20:14:02.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.936+0000 7f921faacb80 1 -- 172.21.15.28:0/3659537931 wait complete. 2024-05-20T20:14:02.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.937+0000 7f921faacb80 1 -- start start 2024-05-20T20:14:02.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.937+0000 7f921faacb80 1 Processor -- start 2024-05-20T20:14:02.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.938+0000 7f921faacb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799b5270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.938+0000 7f921faacb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799aec10 0x5583799b2610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.938+0000 7f921faacb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799b2b50 0x5583799b2fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f921faacb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5583797fd0a0 con 0x5583799b2b50 2024-05-20T20:14:02.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f921faacb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558379897860 con 0x5583799a95f0 2024-05-20T20:14:02.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f921faacb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5583799b5fa0 con 0x5583799aec10 2024-05-20T20:14:02.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f921faacb80 10 client.0 _ll_register_callbacks cb 0x5583798a0d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:02.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f920d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799aec10 0x5583799b2610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f920de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799b5270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f920d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799aec10 0x5583799b2610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33376/0 (socket says 172.21.15.28:33376) 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.939+0000 7f920d400700 1 -- 172.21.15.28:0/1529882348 learned_addr learned my addr 172.21.15.28:0/1529882348 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.940+0000 7f920de00700 1 -- 172.21.15.28:0/1529882348 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799aec10 msgr2=0x5583799b2610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.940+0000 7f920de00700 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799aec10 0x5583799b2610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.940+0000 7f920de00700 1 -- 172.21.15.28:0/1529882348 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799b2b50 msgr2=0x5583799b2fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.940+0000 7f920de00700 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799b2b50 0x5583799b2fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.940+0000 7f920de00700 1 -- 172.21.15.28:0/1529882348 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5583798469d0 con 0x5583799a95f0 2024-05-20T20:14:02.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.940+0000 7f920de00700 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799b5270 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x5583799a5870 tx=0x7f92000106b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b41811035fac7b6f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:02.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.940+0000 7f91efe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:02.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.941+0000 7f920e800700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f91f40b2c00 0x7f91f40b50c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.942+0000 7f91efe00700 1 -- 172.21.15.28:0/1529882348 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9200011440 con 0x5583799a95f0 2024-05-20T20:14:02.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.942+0000 7f91efe00700 1 -- 172.21.15.28:0/1529882348 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f92000115e0 con 0x5583799a95f0 2024-05-20T20:14:02.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.942+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f91fc003c50 con 0x5583799a95f0 2024-05-20T20:14:02.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.943+0000 7f920e800700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f91f40b2c00 0x7f91f40b50c0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9208009c10 tx=0x7f920800b730 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:02.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.943+0000 7f91efe00700 1 -- 172.21.15.28:0/1529882348 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f92000117e0 con 0x5583799a95f0 2024-05-20T20:14:02.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.943+0000 7f921faacb80 10 client.27233 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:02.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.943+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5583798c7160 con 0x5583799a95f0 2024-05-20T20:14:02.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.944+0000 7f91efe00700 1 -- 172.21.15.28:0/1529882348 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f91fc003c50 con 0x5583799a95f0 2024-05-20T20:14:02.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.945+0000 7f91efe00700 1 -- 172.21.15.28:0/1529882348 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f920001a8f0 con 0x5583799a95f0 2024-05-20T20:14:02.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.946+0000 7f921faacb80 10 client.27233 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:02.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.946+0000 7f921faacb80 10 client.27233 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:14:02.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.946+0000 7f921faacb80 20 client.27233 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:02.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.946+0000 7f921faacb80 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5583799c2fa0 0x5583799c53b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:02.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.946+0000 7f921faacb80 4 client.27233 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:02.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.946+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558379908e90 con 0x5583799c2fa0 2024-05-20T20:14:02.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.947+0000 7f920d400700 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5583799c2fa0 0x5583799c53b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:02.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.947+0000 7f920d400700 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5583799c2fa0 0x5583799c53b0 crc :-1 s=READY pgs=193 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7fef62dcfacb765b server_cookie=8c2c82b3cbc1fdb6 in_seq=0 out_seq=0 2024-05-20T20:14:02.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.947+0000 7f91efe00700 10 client.27233 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:02.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.951+0000 7f91efe00700 1 -- 172.21.15.28:0/1529882348 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558379908e90 con 0x5583799c2fa0 2024-05-20T20:14:02.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.951+0000 7f91efe00700 10 client.27233 handle_command_reply: tid=0 2024-05-20T20:14:02.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.951+0000 7f921faacb80 1 client.27233 shutdown 2024-05-20T20:14:02.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.952+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 msgr2=0x5583799b5270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:02.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.952+0000 7f921faacb80 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799b5270 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x5583799a5870 tx=0x7f92000106b0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.953+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 shutdown_connections 2024-05-20T20:14:02.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.953+0000 7f921faacb80 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5583799c2fa0 0x5583799c53b0 crc :-1 s=READY pgs=193 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.953+0000 7f921faacb80 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799b2b50 0x5583799b2fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.953+0000 7f921faacb80 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799b5270 unknown :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.953+0000 7f921faacb80 1 --2- 172.21.15.28:0/1529882348 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799aec10 0x5583799b2610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:02.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.953+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 >> 172.21.15.28:0/1529882348 conn(0x5583798d6ce0 msgr2=0x5583799a3430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:02.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.953+0000 7f91efe00700 0 client.27233 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:02.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.954+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 shutdown_connections 2024-05-20T20:14:02.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.954+0000 7f921faacb80 1 -- 172.21.15.28:0/1529882348 wait complete. 2024-05-20T20:14:02.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.955+0000 7f921faacb80 20 client.27233 trim_cache size 0 max 16384 2024-05-20T20:14:02.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.990+0000 7f921faacb80 1 Processor -- start 2024-05-20T20:14:02.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.995+0000 7f921faacb80 1 -- start start 2024-05-20T20:14:02.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.996+0000 7f921faacb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a8fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.996+0000 7f921faacb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799ac630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.996+0000 7f921faacb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799acb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.998+0000 7f921faacb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f92101bcd80 con 0x5583799aec10 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.998+0000 7f921faacb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f92101be8c0 con 0x5583799a95f0 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.998+0000 7f920e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799acb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.998+0000 7f920e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799acb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33388/0 (socket says 172.21.15.28:33388) 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.998+0000 7f920e800700 1 -- 172.21.15.28:0/2607513272 learned_addr learned my addr 172.21.15.28:0/2607513272 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.999+0000 7f921faacb80 1 -- 172.21.15.28:0/2607513272 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f92101bd450 con 0x5583799b2b50 2024-05-20T20:14:03.000 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:02.999+0000 7f920d400700 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799ac630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:03.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.000+0000 7f920e800700 1 -- 172.21.15.28:0/2607513272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 msgr2=0x5583799a8fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:03.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.000+0000 7f920de00700 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a8fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:03.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.001+0000 7f920e800700 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a8fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.001+0000 7f920e800700 1 -- 172.21.15.28:0/2607513272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 msgr2=0x5583799ac630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:03.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.001+0000 7f920e800700 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799ac630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.001+0000 7f920e800700 1 -- 172.21.15.28:0/2607513272 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5583799b55e0 con 0x5583799b2b50 2024-05-20T20:14:03.002 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.001+0000 7f920de00700 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a8fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:03.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.002+0000 7f920e800700 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799acb70 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f9208003ab0 tx=0x7f920801f000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=535f05c9b97173c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:03.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.003+0000 7f91faa00700 1 -- 172.21.15.28:0/2607513272 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f920802f070 con 0x5583799b2b50 2024-05-20T20:14:03.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.003+0000 7f91faa00700 1 -- 172.21.15.28:0/2607513272 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9208007b90 con 0x5583799b2b50 2024-05-20T20:14:03.003 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.003+0000 7f91faa00700 1 -- 172.21.15.28:0/2607513272 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f920802ab70 con 0x5583799b2b50 2024-05-20T20:14:03.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.005+0000 7f921faacb80 1 -- 172.21.15.28:0/2607513272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 msgr2=0x5583799acb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:03.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.005+0000 7f921faacb80 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799acb70 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f9208003ab0 tx=0x7f920801f000 comp rx=0 tx=0).stop 2024-05-20T20:14:03.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.005+0000 7f921faacb80 1 -- 172.21.15.28:0/2607513272 shutdown_connections 2024-05-20T20:14:03.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.005+0000 7f921faacb80 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799acb70 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.005+0000 7f921faacb80 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799a95f0 0x5583799a8fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.005+0000 7f921faacb80 1 --2- 172.21.15.28:0/2607513272 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799aec10 0x5583799ac630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.005+0000 7f921faacb80 1 -- 172.21.15.28:0/2607513272 >> 172.21.15.28:0/2607513272 conn(0x5583798d6ce0 msgr2=0x5583799c4fb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:03.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.007+0000 7f921faacb80 1 -- 172.21.15.28:0/2607513272 shutdown_connections 2024-05-20T20:14:03.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.007+0000 7f921faacb80 1 -- 172.21.15.28:0/2607513272 wait complete. 2024-05-20T20:14:03.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.008+0000 7f921faacb80 1 -- start start 2024-05-20T20:14:03.009 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.009+0000 7f921faacb80 1 Processor -- start 2024-05-20T20:14:03.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.010+0000 7f921faacb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799a95f0 0x5583799bb410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:03.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.010+0000 7f921faacb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799aec10 0x5583799bb950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:03.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.010+0000 7f921faacb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799b8200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:03.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.010+0000 7f921faacb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f92101be1f0 con 0x5583799a95f0 2024-05-20T20:14:03.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.010+0000 7f921faacb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f92101bdb20 con 0x5583799aec10 2024-05-20T20:14:03.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.010+0000 7f921faacb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f92101bad40 con 0x5583799b2b50 2024-05-20T20:14:03.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.010+0000 7f921faacb80 10 client.0 _ll_register_callbacks cb 0x5583798a0d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:03.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.011+0000 7f920e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799b8200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:03.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.011+0000 7f920e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799b8200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33390/0 (socket says 172.21.15.28:33390) 2024-05-20T20:14:03.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.011+0000 7f920e800700 1 -- 172.21.15.28:0/3090037427 learned_addr learned my addr 172.21.15.28:0/3090037427 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:03.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.011+0000 7f920d400700 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799aec10 0x5583799bb950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:03.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.012+0000 7f920e800700 1 -- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799aec10 msgr2=0x5583799bb950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:03.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.012+0000 7f920e800700 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799aec10 0x5583799bb950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.012+0000 7f920e800700 1 -- 172.21.15.28:0/3090037427 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799a95f0 msgr2=0x5583799bb410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:03.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.012+0000 7f920e800700 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799a95f0 0x5583799bb410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.012+0000 7f920e800700 1 -- 172.21.15.28:0/3090037427 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5583799b89f0 con 0x5583799b2b50 2024-05-20T20:14:03.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.014+0000 7f920e800700 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799b8200 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f920802a750 tx=0x7f920802aa90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d0250ea5213810b0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:03.015 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.014+0000 7f91efe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:14:03.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.015+0000 7f91efe00700 1 -- 172.21.15.28:0/3090037427 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f920803b060 con 0x5583799b2b50 2024-05-20T20:14:03.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.016+0000 7f91efe00700 1 -- 172.21.15.28:0/3090037427 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(85..85 src has 1..85) ==== 8542+0+0 (secure 0 0 0) 0x7f9208036680 con 0x5583799b2b50 2024-05-20T20:14:03.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.017+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f9208007b90 con 0x5583799b2b50 2024-05-20T20:14:03.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.022+0000 7f91efe00700 1 -- 172.21.15.28:0/3090037427 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f920802f070 con 0x5583799b2b50 2024-05-20T20:14:03.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.022+0000 7f921faacb80 10 client.27130 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:03.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.022+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558379a2e5f0 con 0x5583799b2b50 2024-05-20T20:14:03.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.023+0000 7f91efe00700 1 -- 172.21.15.28:0/3090037427 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f9208007b90 con 0x5583799b2b50 2024-05-20T20:14:03.024 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.023+0000 7f91efe00700 1 -- 172.21.15.28:0/3090037427 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f9208011070 con 0x5583799b2b50 2024-05-20T20:14:03.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.024+0000 7f921faacb80 10 client.27130 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:03.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.024+0000 7f921faacb80 10 client.27130 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:14:03.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.024+0000 7f921faacb80 20 client.27130 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:03.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.024+0000 7f921faacb80 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558379a2ea50 0x558379a30e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:03.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.025+0000 7f921faacb80 4 client.27130 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "c71942f8", "release": true, "await": true}] 2024-05-20T20:14:03.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.025+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "c71942f8", "release": true, "await": true}) -- 0x558379902670 con 0x558379a2ea50 2024-05-20T20:14:03.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.030+0000 7f920de00700 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558379a2ea50 0x558379a30e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:03.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.033+0000 7f920de00700 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558379a2ea50 0x558379a30e40 crc :-1 s=READY pgs=194 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=184873bb982ac142 server_cookie=744948121d11caba in_seq=0 out_seq=0 2024-05-20T20:14:03.034 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.033+0000 7f91efe00700 10 client.27130 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:03.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.039+0000 7f91efe00700 1 -- 172.21.15.28:0/3090037427 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x558379902670 con 0x558379a2ea50 2024-05-20T20:14:03.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.039+0000 7f91efe00700 10 client.27130 handle_command_reply: tid=0 2024-05-20T20:14:03.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.039+0000 7f921faacb80 1 client.27130 shutdown 2024-05-20T20:14:03.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.044+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 msgr2=0x5583799b8200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:03.046 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.044+0000 7f921faacb80 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799b8200 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f920802a750 tx=0x7f920802aa90 comp rx=0 tx=0).stop 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 shutdown_connections 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x558379a2ea50 0x558379a30e40 crc :-1 s=READY pgs=194 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5583799b2b50 0x5583799b8200 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5583799a95f0 0x5583799bb410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 --2- 172.21.15.28:0/3090037427 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5583799aec10 0x5583799bb950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 >> 172.21.15.28:0/3090037427 conn(0x5583798d6ce0 msgr2=0x5583799ac630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f91efe00700 0 client.27130 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 shutdown_connections 2024-05-20T20:14:03.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.046+0000 7f921faacb80 1 -- 172.21.15.28:0/3090037427 wait complete. 2024-05-20T20:14:03.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.047+0000 7f921faacb80 20 client.27130 trim_cache size 0 max 16384 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 24, 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "c71942f8": { 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 24, 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:14:03.049 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 143.3, 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:14:03.050 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f91f40b2c00 msgr2=0x7f91f40b50c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f91f40b2c00 0x7f91f40b50c0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9208009c10 tx=0x7f920800b730 comp rx=0 tx=0).stop 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f921010ae60 msgr2=0x7f921010e700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f921010ae60 0x7f921010e700 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f91fc00c9a0 tx=0x7f91fc00ccb0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 shutdown_connections 2024-05-20T20:14:03.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f9210118a80 0x7f921010ec40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f921010a540 0x7f921010e1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f91f40b2c00 0x7f91f40b50c0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 --2- 172.21.15.28:0/1532202187 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f921010ae60 0x7f921010e700 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:03.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.048+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 >> 172.21.15.28:0/1532202187 conn(0x7f921006e6b0 msgr2=0x7f9210118f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:03.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.051+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 shutdown_connections 2024-05-20T20:14:03.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:03.051+0000 7f9215200700 1 -- 172.21.15.28:0/1532202187 wait complete. 2024-05-20T20:14:03.091 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:03.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:03 smithi028 ceph-mon[31178]: pgmap v259: 97 pgs: 97 active+clean; 54 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 284 MiB/s wr, 1.78k op/s 2024-05-20T20:14:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:03 smithi154 ceph-mon[36314]: pgmap v259: 97 pgs: 97 active+clean; 54 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 284 MiB/s wr, 1.78k op/s 2024-05-20T20:14:03.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:03 smithi033 ceph-mon[36179]: pgmap v259: 97 pgs: 97 active+clean; 54 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 284 MiB/s wr, 1.78k op/s 2024-05-20T20:14:05.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:05 smithi028 ceph-mon[31178]: pgmap v260: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 349 MiB/s wr, 2.54k op/s 2024-05-20T20:14:05.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:05 smithi028 ceph-mon[31178]: osdmap e86: 12 total, 12 up, 12 in 2024-05-20T20:14:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:05 smithi154 ceph-mon[36314]: pgmap v260: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 349 MiB/s wr, 2.54k op/s 2024-05-20T20:14:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:05 smithi154 ceph-mon[36314]: osdmap e86: 12 total, 12 up, 12 in 2024-05-20T20:14:05.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:05 smithi033 ceph-mon[36179]: pgmap v260: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 349 MiB/s wr, 2.54k op/s 2024-05-20T20:14:05.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:05 smithi033 ceph-mon[36179]: osdmap e86: 12 total, 12 up, 12 in 2024-05-20T20:14:05.605 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set 'c71942f8', total seconds elapsed: 36.6 2024-05-20T20:14:07.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:07 smithi028 ceph-mon[31178]: pgmap v262: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 280 MiB/s wr, 2.38k op/s 2024-05-20T20:14:07.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:07 smithi154 ceph-mon[36314]: pgmap v262: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 280 MiB/s wr, 2.38k op/s 2024-05-20T20:14:07.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:07 smithi033 ceph-mon[36179]: pgmap v262: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 280 MiB/s wr, 2.38k op/s 2024-05-20T20:14:09.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:09 smithi028 ceph-mon[31178]: pgmap v263: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 270 MiB/s wr, 2.42k op/s 2024-05-20T20:14:09.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:09 smithi154 ceph-mon[36314]: pgmap v263: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 270 MiB/s wr, 2.42k op/s 2024-05-20T20:14:09.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:09 smithi033 ceph-mon[36179]: pgmap v263: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 270 MiB/s wr, 2.42k op/s 2024-05-20T20:14:11.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:11 smithi154 ceph-mon[36314]: pgmap v264: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 225 MiB/s wr, 2.24k op/s 2024-05-20T20:14:11.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:11 smithi154 ceph-mon[36314]: osdmap e87: 12 total, 12 up, 12 in 2024-05-20T20:14:11.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:11 smithi033 ceph-mon[36179]: pgmap v264: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 225 MiB/s wr, 2.24k op/s 2024-05-20T20:14:11.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:11 smithi033 ceph-mon[36179]: osdmap e87: 12 total, 12 up, 12 in 2024-05-20T20:14:11.559 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:11 smithi028 ceph-mon[31178]: pgmap v264: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 225 MiB/s wr, 2.24k op/s 2024-05-20T20:14:11.560 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:11 smithi028 ceph-mon[31178]: osdmap e87: 12 total, 12 up, 12 in 2024-05-20T20:14:12.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:12 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:14:12.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:12 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:14:12.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:12 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 -- 172.21.15.28:0/709361722 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 msgr2=0x7fc67806b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 --2- 172.21.15.28:0/709361722 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67806b6a0 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7fc668009f90 tx=0x7fc66800b330 comp rx=0 tx=0).stop 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 -- 172.21.15.28:0/709361722 shutdown_connections 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 --2- 172.21.15.28:0/709361722 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc67806bbe0 0x7fc6781103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 --2- 172.21.15.28:0/709361722 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc6781095b0 0x7fc678109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 --2- 172.21.15.28:0/709361722 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67806b6a0 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 -- 172.21.15.28:0/709361722 >> 172.21.15.28:0/709361722 conn(0x7fc6780793f0 msgr2=0x7fc678079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 -- 172.21.15.28:0/709361722 shutdown_connections 2024-05-20T20:14:12.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.689+0000 7fc67fc00700 1 -- 172.21.15.28:0/709361722 wait complete. 2024-05-20T20:14:12.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 Processor -- start 2024-05-20T20:14:12.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 -- start start 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67819f910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc67806bbe0 0x7fc67819fe50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc6781095b0 0x7fc678199ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc678113730 con 0x7fc6781095b0 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc6781135b0 con 0x7fc67806bbe0 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67fc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc6781138b0 con 0x7fc67806b220 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.691+0000 7fc67d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67819f910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67819f910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49020/0 (socket says 172.21.15.28:49020) 2024-05-20T20:14:12.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 -- 172.21.15.28:0/345689144 learned_addr learned my addr 172.21.15.28:0/345689144 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:12.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 -- 172.21.15.28:0/345689144 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc67806bbe0 msgr2=0x7fc67819fe50 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:14:12.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc67806bbe0 0x7fc67819fe50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 -- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc6781095b0 msgr2=0x7fc678199ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc6781095b0 0x7fc678199ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 -- 172.21.15.28:0/345689144 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc67819a230 con 0x7fc67806b220 2024-05-20T20:14:12.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.692+0000 7fc67d200700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67819f910 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fc66000e9c0 tx=0x7fc66000ed80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:12.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.693+0000 7fc675600700 1 -- 172.21.15.28:0/345689144 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc66000cbd0 con 0x7fc67806b220 2024-05-20T20:14:12.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.693+0000 7fc675600700 1 -- 172.21.15.28:0/345689144 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc66000cd70 con 0x7fc67806b220 2024-05-20T20:14:12.695 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.693+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc67819a520 con 0x7fc67806b220 2024-05-20T20:14:12.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.693+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc6781a67e0 con 0x7fc67806b220 2024-05-20T20:14:12.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.694+0000 7fc675600700 1 -- 172.21.15.28:0/345689144 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc660005660 con 0x7fc67806b220 2024-05-20T20:14:12.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.701+0000 7fc675600700 1 -- 172.21.15.28:0/345689144 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc660021440 con 0x7fc67806b220 2024-05-20T20:14:12.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.717+0000 7fc68a63db80 1 Processor -- start 2024-05-20T20:14:12.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.717+0000 7fc675600700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc65c0b2a70 0x7fc65c0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.717+0000 7fc675600700 1 -- 172.21.15.28:0/345689144 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fc6600e2730 con 0x7fc67806b220 2024-05-20T20:14:12.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.718+0000 7fc68a63db80 1 -- start start 2024-05-20T20:14:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ec8200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ec2330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ec2dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc68a63db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562556e17e40 con 0x562556ec8740 2024-05-20T20:14:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc68a63db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562556e079e0 con 0x562556ec7e20 2024-05-20T20:14:12.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc68a63db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562556e0ba20 con 0x562556ec2950 2024-05-20T20:14:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc67d200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ec8200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc67d200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ec8200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53342/0 (socket says 172.21.15.28:53342) 2024-05-20T20:14:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc67d200700 1 -- 172.21.15.28:0/1353148857 learned_addr learned my addr 172.21.15.28:0/1353148857 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.721+0000 7fc67dc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc65c0b2a70 0x7fc65c0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.722+0000 7fc67d200700 1 -- 172.21.15.28:0/1353148857 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 msgr2=0x562556ec2dd0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.722+0000 7fc67d200700 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ec2dd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.722+0000 7fc67d200700 1 -- 172.21.15.28:0/1353148857 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 msgr2=0x562556ec2330 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:14:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.724+0000 7fc677e00700 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ec2330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.724+0000 7fc67d200700 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ec2330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.724+0000 7fc67d200700 1 -- 172.21.15.28:0/1353148857 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562556dd0420 con 0x562556ec7e20 2024-05-20T20:14:12.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.724+0000 7fc677e00700 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ec2330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:12.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.724+0000 7fc67d200700 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ec8200 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7fc6600a5a80 tx=0x7fc6600a9480 comp rx=0 tx=0).ready entity=mon.2 client_cookie=565725e58d96c400 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:12.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.728+0000 7fc66e000700 1 -- 172.21.15.28:0/1353148857 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc6600b21b0 con 0x562556ec7e20 2024-05-20T20:14:12.729 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.728+0000 7fc66e000700 1 -- 172.21.15.28:0/1353148857 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc6600bee50 con 0x562556ec7e20 2024-05-20T20:14:12.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.729+0000 7fc68a63db80 1 -- 172.21.15.28:0/1353148857 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 msgr2=0x562556ec8200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.729+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ec8200 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7fc6600a5a80 tx=0x7fc6600a9480 comp rx=0 tx=0).stop 2024-05-20T20:14:12.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.731+0000 7fc68a63db80 1 -- 172.21.15.28:0/1353148857 shutdown_connections 2024-05-20T20:14:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.731+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ec2dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.731+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ec8200 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.731+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1353148857 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ec2330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.731+0000 7fc68a63db80 1 -- 172.21.15.28:0/1353148857 >> 172.21.15.28:0/1353148857 conn(0x562556de7be0 msgr2=0x562556eb4690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:12.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.732+0000 7fc67dc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc65c0b2a70 0x7fc65c0b4f30 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x562556ec3310 tx=0x7fc6700058e0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.735+0000 7fc68a63db80 1 -- 172.21.15.28:0/1353148857 shutdown_connections 2024-05-20T20:14:12.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.735+0000 7fc68a63db80 1 -- 172.21.15.28:0/1353148857 wait complete. 2024-05-20T20:14:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.738+0000 7fc68a63db80 1 -- start start 2024-05-20T20:14:12.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.739+0000 7fc68a63db80 1 Processor -- start 2024-05-20T20:14:12.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.740+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.741+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ecb7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.741+0000 7fc67d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.741+0000 7fc67d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49044/0 (socket says 172.21.15.28:49044) 2024-05-20T20:14:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.741+0000 7fc67d200700 1 -- 172.21.15.28:0/1154689098 learned_addr learned my addr 172.21.15.28:0/1154689098 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.742+0000 7fc677e00700 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ecb7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.742+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ecbd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.743+0000 7fc67dc00700 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ecbd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562556d0dfa0 con 0x562556ec8740 2024-05-20T20:14:12.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562556da8760 con 0x562556ec7e20 2024-05-20T20:14:12.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562556ecf240 con 0x562556ec2950 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc68a63db80 10 client.0 _ll_register_callbacks cb 0x562556db1c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc67dc00700 1 -- 172.21.15.28:0/1154689098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 msgr2=0x562556ece430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc67dc00700 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc67dc00700 1 -- 172.21.15.28:0/1154689098 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 msgr2=0x562556ecb7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.744+0000 7fc67dc00700 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ecb7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.745+0000 7fc67dc00700 1 -- 172.21.15.28:0/1154689098 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562556d578d0 con 0x562556ec8740 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.745+0000 7fc67dc00700 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ecbd10 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7fc67000ef60 tx=0x7fc670004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c7237f06ba466822 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.745+0000 7fc666a00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.745+0000 7fc666a00700 1 -- 172.21.15.28:0/1154689098 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc670020070 con 0x562556ec8740 2024-05-20T20:14:12.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.746+0000 7fc666a00700 1 -- 172.21.15.28:0/1154689098 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fc670004a30 con 0x562556ec8740 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.746+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc6600bee50 con 0x562556ec8740 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.746+0000 7fc666a00700 1 -- 172.21.15.28:0/1154689098 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc670010e60 con 0x562556ec8740 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.746+0000 7fc68a63db80 10 client.16848 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.746+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562556dd8060 con 0x562556ec8740 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc666a00700 1 -- 172.21.15.28:0/1154689098 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc6600bee50 con 0x562556ec8740 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc666a00700 1 -- 172.21.15.28:0/1154689098 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc67001c670 con 0x562556ec8740 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc68a63db80 10 client.16848 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc68a63db80 10 client.16848 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:14:12.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc68a63db80 20 client.16848 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:12.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556edbe30 0x562556ede260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc68a63db80 4 client.16848 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:12.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.747+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562556e19d90 con 0x562556edbe30 2024-05-20T20:14:12.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.750+0000 7fc67d200700 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556edbe30 0x562556ede260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.751+0000 7fc67d200700 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556edbe30 0x562556ede260 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=d1a911509ebe9a51 server_cookie=63c14394bb7434fe in_seq=0 out_seq=0 2024-05-20T20:14:12.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.751+0000 7fc666a00700 10 client.16848 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:14:12.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.752+0000 7fc666a00700 1 -- 172.21.15.28:0/1154689098 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562556e19d90 con 0x562556edbe30 2024-05-20T20:14:12.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.752+0000 7fc666a00700 10 client.16848 handle_command_reply: tid=0 2024-05-20T20:14:12.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.754+0000 7fc68a63db80 1 client.16848 shutdown 2024-05-20T20:14:12.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.756+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 msgr2=0x562556ecbd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.756+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ecbd10 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7fc67000ef60 tx=0x7fc670004750 comp rx=0 tx=0).stop 2024-05-20T20:14:12.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.756+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 shutdown_connections 2024-05-20T20:14:12.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.756+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec8740 0x562556ecbd10 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.757+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556edbe30 0x562556ede260 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.757+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.757+0000 7fc68a63db80 1 --2- 172.21.15.28:0/1154689098 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ec7e20 0x562556ecb7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.757+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 >> 172.21.15.28:0/1154689098 conn(0x562556de7be0 msgr2=0x562556eb3d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:12.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.757+0000 7fc666a00700 0 client.16848 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:14:12.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.758+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 shutdown_connections 2024-05-20T20:14:12.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.758+0000 7fc68a63db80 1 -- 172.21.15.28:0/1154689098 wait complete. 2024-05-20T20:14:12.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.759+0000 7fc68a63db80 20 client.16848 trim_cache size 0 max 16384 2024-05-20T20:14:12.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.793+0000 7fc68a63db80 1 Processor -- start 2024-05-20T20:14:12.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.793+0000 7fc68a63db80 1 -- start start 2024-05-20T20:14:12.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.794+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.794+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec8740 0x562556eb44f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.794+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ed8390 0x562556ecb5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.794+0000 7fc68a63db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc65400d670 con 0x562556ec2950 2024-05-20T20:14:12.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.794+0000 7fc68a63db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc65400f1b0 con 0x562556ed8390 2024-05-20T20:14:12.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.794+0000 7fc68a63db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc65400dd40 con 0x562556ec8740 2024-05-20T20:14:12.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.796+0000 7fc67dc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ed8390 0x562556ecb5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.796+0000 7fc67dc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ed8390 0x562556ecb5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53360/0 (socket says 172.21.15.28:53360) 2024-05-20T20:14:12.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.796+0000 7fc67d200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.796+0000 7fc67dc00700 1 -- 172.21.15.28:0/386962519 learned_addr learned my addr 172.21.15.28:0/386962519 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.796+0000 7fc677e00700 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec8740 0x562556eb44f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.797+0000 7fc677e00700 1 -- 172.21.15.28:0/386962519 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ed8390 msgr2=0x562556ecb5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.797+0000 7fc677e00700 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ed8390 0x562556ecb5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.797+0000 7fc677e00700 1 -- 172.21.15.28:0/386962519 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec2950 msgr2=0x562556ece430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.797+0000 7fc677e00700 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.797+0000 7fc677e00700 1 -- 172.21.15.28:0/386962519 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562556ede4b0 con 0x562556ec8740 2024-05-20T20:14:12.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.798+0000 7fc677e00700 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec8740 0x562556eb44f0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fc668000f80 tx=0x7fc668009470 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c26ed29fe9d3d888 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:12.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.799+0000 7fc66d600700 1 -- 172.21.15.28:0/386962519 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc668003a00 con 0x562556ec8740 2024-05-20T20:14:12.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.799+0000 7fc66d600700 1 -- 172.21.15.28:0/386962519 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc668003ba0 con 0x562556ec8740 2024-05-20T20:14:12.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.799+0000 7fc66d600700 1 -- 172.21.15.28:0/386962519 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc6680056b0 con 0x562556ec8740 2024-05-20T20:14:12.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 -- 172.21.15.28:0/386962519 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec8740 msgr2=0x562556eb44f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec8740 0x562556eb44f0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fc668000f80 tx=0x7fc668009470 comp rx=0 tx=0).stop 2024-05-20T20:14:12.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 -- 172.21.15.28:0/386962519 shutdown_connections 2024-05-20T20:14:12.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556ed8390 0x562556ecb5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ec2950 0x562556ece430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 --2- 172.21.15.28:0/386962519 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec8740 0x562556eb44f0 secure :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fc668000f80 tx=0x7fc668009470 comp rx=0 tx=0).stop 2024-05-20T20:14:12.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 -- 172.21.15.28:0/386962519 >> 172.21.15.28:0/386962519 conn(0x562556de7be0 msgr2=0x562556eca0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:12.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 -- 172.21.15.28:0/386962519 shutdown_connections 2024-05-20T20:14:12.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.800+0000 7fc68a63db80 1 -- 172.21.15.28:0/386962519 wait complete. 2024-05-20T20:14:12.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.801+0000 7fc68a63db80 1 -- start start 2024-05-20T20:14:12.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.801+0000 7fc68a63db80 1 Processor -- start 2024-05-20T20:14:12.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ed4580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ed8390 0x562556ed4ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc68a63db80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556f41a20 0x562556f41ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc68a63db80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc65400eae0 con 0x562556ed8390 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc68a63db80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc65400e410 con 0x562556f41a20 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc68a63db80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc65400b630 con 0x562556ec2950 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc68a63db80 10 client.0 _ll_register_callbacks cb 0x562556db1c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc67d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ed4580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc67d200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ed4580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49056/0 (socket says 172.21.15.28:49056) 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc67d200700 1 -- 172.21.15.28:0/671657160 learned_addr learned my addr 172.21.15.28:0/671657160 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.802+0000 7fc67dc00700 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556f41a20 0x562556f41ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.804 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.803+0000 7fc677e00700 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ed8390 0x562556ed4ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc67d200700 1 -- 172.21.15.28:0/671657160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556f41a20 msgr2=0x562556f41ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:12.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc67d200700 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556f41a20 0x562556f41ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc67d200700 1 -- 172.21.15.28:0/671657160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ed8390 msgr2=0x562556ed4ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:12.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc67d200700 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ed8390 0x562556ed4ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc67d200700 1 -- 172.21.15.28:0/671657160 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562556f423e0 con 0x562556ec2950 2024-05-20T20:14:12.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc67d200700 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ed4580 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fc6600a5a80 tx=0x7fc6600a9670 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2502a7e68eb02564 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:12.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc666a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:14:12.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc666a00700 1 -- 172.21.15.28:0/671657160 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc6600a9a00 con 0x562556ec2950 2024-05-20T20:14:12.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.805+0000 7fc666a00700 1 -- 172.21.15.28:0/671657160 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fc6600a9e70 con 0x562556ec2950 2024-05-20T20:14:12.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.806+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc668003ba0 con 0x562556ec2950 2024-05-20T20:14:12.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.806+0000 7fc666a00700 1 -- 172.21.15.28:0/671657160 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc6600b23d0 con 0x562556ec2950 2024-05-20T20:14:12.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.808+0000 7fc68a63db80 10 client.27145 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:12.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.808+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562556ee3760 con 0x562556ec2950 2024-05-20T20:14:12.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.808+0000 7fc666a00700 1 -- 172.21.15.28:0/671657160 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc668003ba0 con 0x562556ec2950 2024-05-20T20:14:12.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.808+0000 7fc666a00700 1 -- 172.21.15.28:0/671657160 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc6600c5020 con 0x562556ec2950 2024-05-20T20:14:12.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.809+0000 7fc68a63db80 10 client.27145 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:12.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.809+0000 7fc68a63db80 10 client.27145 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:14:12.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.809+0000 7fc68a63db80 20 client.27145 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:12.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.809+0000 7fc68a63db80 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556ebab20 0x562556ebcf40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:12.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.809+0000 7fc68a63db80 4 client.27145 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:14:12.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.809+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x562556e13570 con 0x562556ebab20 2024-05-20T20:14:12.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.810+0000 7fc677e00700 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556ebab20 0x562556ebcf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:12.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.812+0000 7fc677e00700 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556ebab20 0x562556ebcf40 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=9bf8b267d9313598 server_cookie=e48cd6ad22945194 in_seq=0 out_seq=0 2024-05-20T20:14:12.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.812+0000 7fc666a00700 10 client.27145 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:14:12.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.813+0000 7fc666a00700 1 -- 172.21.15.28:0/671657160 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x562556e13570 con 0x562556ebab20 2024-05-20T20:14:12.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.813+0000 7fc666a00700 10 client.27145 handle_command_reply: tid=0 2024-05-20T20:14:12.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 client.27145 shutdown 2024-05-20T20:14:12.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 msgr2=0x562556ed4580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ed4580 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fc6600a5a80 tx=0x7fc6600a9670 comp rx=0 tx=0).stop 2024-05-20T20:14:12.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 shutdown_connections 2024-05-20T20:14:12.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562556f41a20 0x562556f41ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x562556ebab20 0x562556ebcf40 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562556ec2950 0x562556ed4580 secure :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7fc6600a5a80 tx=0x7fc6600a9670 comp rx=0 tx=0).stop 2024-05-20T20:14:12.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 --2- 172.21.15.28:0/671657160 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562556ed8390 0x562556ed4ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 >> 172.21.15.28:0/671657160 conn(0x562556de7be0 msgr2=0x562556ec9e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:12.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.814+0000 7fc666a00700 0 client.27145 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:14:12.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.815+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 shutdown_connections 2024-05-20T20:14:12.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.815+0000 7fc68a63db80 1 -- 172.21.15.28:0/671657160 wait complete. 2024-05-20T20:14:12.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.816+0000 7fc68a63db80 20 client.27145 trim_cache size 0 max 16384 2024-05-20T20:14:12.817 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:14:12.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc65c0b2a70 msgr2=0x7fc65c0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc65c0b2a70 0x7fc65c0b4f30 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x562556ec3310 tx=0x7fc6700058e0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 msgr2=0x7fc67819f910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:12.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67819f910 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fc66000e9c0 tx=0x7fc66000ed80 comp rx=0 tx=0).stop 2024-05-20T20:14:12.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 shutdown_connections 2024-05-20T20:14:12.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc6781095b0 0x7fc678199ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc67806b220 0x7fc67819f910 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc65c0b2a70 0x7fc65c0b4f30 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 --2- 172.21.15.28:0/345689144 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc67806bbe0 0x7fc67819fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:12.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 >> 172.21.15.28:0/345689144 conn(0x7fc6780793f0 msgr2=0x7fc6781011a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:12.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 shutdown_connections 2024-05-20T20:14:12.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:12.817+0000 7fc67fc00700 1 -- 172.21.15.28:0/345689144 wait complete. 2024-05-20T20:14:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:13 smithi154 ceph-mon[36314]: pgmap v266: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 131 MiB/s wr, 1.35k op/s 2024-05-20T20:14:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:13 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:13.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:13 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:13 smithi033 ceph-mon[36179]: pgmap v266: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 131 MiB/s wr, 1.35k op/s 2024-05-20T20:14:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:13 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:13.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:13 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:13.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:13 smithi028 ceph-mon[31178]: pgmap v266: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 131 MiB/s wr, 1.35k op/s 2024-05-20T20:14:13.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:13 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:13.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:13 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:14.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.504+0000 7fd851800640 1 -- 172.21.15.28:0/579832280 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c109c60 msgr2=0x7fd84c10dca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.504+0000 7fd851800640 1 --2- 172.21.15.28:0/579832280 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c109c60 0x7fd84c10dca0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fd83c009900 tx=0x7fd83c02a440 comp rx=0 tx=0).stop 2024-05-20T20:14:14.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.505+0000 7fd84b400640 1 --2- 172.21.15.28:0/579832280 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c115f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:14.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.505+0000 7fd851800640 1 -- 172.21.15.28:0/579832280 shutdown_connections 2024-05-20T20:14:14.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.505+0000 7fd851800640 1 --2- 172.21.15.28:0/579832280 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c115f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.505+0000 7fd851800640 1 --2- 172.21.15.28:0/579832280 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c109c60 0x7fd84c10dca0 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.505+0000 7fd851800640 1 --2- 172.21.15.28:0/579832280 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd84c1092b0 0x7fd84c109690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.505+0000 7fd851800640 1 -- 172.21.15.28:0/579832280 >> 172.21.15.28:0/579832280 conn(0x7fd84c076fe0 msgr2=0x7fd84c079400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:14.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.514+0000 7fd851800640 1 -- 172.21.15.28:0/579832280 shutdown_connections 2024-05-20T20:14:14.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.514+0000 7fd851800640 1 -- 172.21.15.28:0/579832280 wait complete. 2024-05-20T20:14:14.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.516+0000 7fd851800640 1 Processor -- start 2024-05-20T20:14:14.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.516+0000 7fd851800640 1 -- start start 2024-05-20T20:14:14.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.517+0000 7fd851800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c082f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c082f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c082f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53384/0 (socket says 172.21.15.28:53384) 2024-05-20T20:14:14.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd851800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd84c083480 0x7fd84c07e060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd84aa00640 1 -- 172.21.15.28:0/907839763 learned_addr learned my addr 172.21.15.28:0/907839763 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:14.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd851800640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c07e5a0 0x7fd84c07ea50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd84c07c9d0 con 0x7fd84c083480 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd84c07c850 con 0x7fd84c10e380 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.518+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd84c07cb50 con 0x7fd84c07e5a0 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84a000640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd84c083480 0x7fd84c07e060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84b400640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c07e5a0 0x7fd84c07ea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84aa00640 1 -- 172.21.15.28:0/907839763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c07e5a0 msgr2=0x7fd84c07ea50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84aa00640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c07e5a0 0x7fd84c07ea50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84aa00640 1 -- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd84c083480 msgr2=0x7fd84c07e060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84aa00640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd84c083480 0x7fd84c07e060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84aa00640 1 -- 172.21.15.28:0/907839763 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd84c1b4b60 con 0x7fd84c10e380 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.519+0000 7fd84a000640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd84c083480 0x7fd84c07e060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:14:14.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.520+0000 7fd84aa00640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c082f40 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7fd83400b400 tx=0x7fd83400b8c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:14.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.520+0000 7fd843400640 1 -- 172.21.15.28:0/907839763 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd834015020 con 0x7fd84c10e380 2024-05-20T20:14:14.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.520+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd84c1b4e50 con 0x7fd84c10e380 2024-05-20T20:14:14.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.520+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd84c1b52b0 con 0x7fd84c10e380 2024-05-20T20:14:14.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.521+0000 7fd843400640 1 -- 172.21.15.28:0/907839763 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd834011ce0 con 0x7fd84c10e380 2024-05-20T20:14:14.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.521+0000 7fd843400640 1 -- 172.21.15.28:0/907839763 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd834010de0 con 0x7fd84c10e380 2024-05-20T20:14:14.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.540+0000 7fd843400640 1 -- 172.21.15.28:0/907839763 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd834018020 con 0x7fd84c10e380 2024-05-20T20:14:14.544 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.544+0000 7fd854779740 1 Processor -- start 2024-05-20T20:14:14.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.545+0000 7fd854779740 1 -- start start 2024-05-20T20:14:14.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.546+0000 7fd843400640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8300b2ae0 0x7fd8300b4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.546+0000 7fd843400640 1 -- 172.21.15.28:0/907839763 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fd8340e1c20 con 0x7fd84c10e380 2024-05-20T20:14:14.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.547+0000 7fd84b400640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8300b2ae0 0x7fd8300b4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.547+0000 7fd854779740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743875c90 0x55f743876070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.548 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.547+0000 7fd854779740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f7438765b0 0x55f743880930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.547+0000 7fd854779740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f743883260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.549 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.547+0000 7fd854779740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55f7437e6b90 con 0x55f743880e70 2024-05-20T20:14:14.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.547+0000 7fd854779740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55f7437d6730 con 0x55f7438765b0 2024-05-20T20:14:14.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.547+0000 7fd854779740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55f7437da770 con 0x55f743875c90 2024-05-20T20:14:14.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743875c90 0x55f743876070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743875c90 0x55f743876070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49076/0 (socket says 172.21.15.28:49076) 2024-05-20T20:14:14.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 -- 172.21.15.28:0/2123856883 learned_addr learned my addr 172.21.15.28:0/2123856883 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:14.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 -- 172.21.15.28:0/2123856883 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f7438765b0 msgr2=0x55f743880930 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 --2- 172.21.15.28:0/2123856883 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f7438765b0 0x55f743880930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 -- 172.21.15.28:0/2123856883 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 msgr2=0x55f743883260 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 --2- 172.21.15.28:0/2123856883 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f743883260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.548+0000 7fd84aa00640 1 -- 172.21.15.28:0/2123856883 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f74376f8b0 con 0x55f743875c90 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.549+0000 7fd84aa00640 1 --2- 172.21.15.28:0/2123856883 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743875c90 0x55f743876070 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fd8340a8f30 tx=0x7fd8340a8bf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=39aeb347123de5fd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.549+0000 7fd83be00640 1 -- 172.21.15.28:0/2123856883 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8340ad350 con 0x55f743875c90 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.549+0000 7fd83be00640 1 -- 172.21.15.28:0/2123856883 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd8340b1e00 con 0x55f743875c90 2024-05-20T20:14:14.551 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.549+0000 7fd83be00640 1 -- 172.21.15.28:0/2123856883 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8340a4b70 con 0x55f743875c90 2024-05-20T20:14:14.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.551+0000 7fd854779740 1 -- 172.21.15.28:0/2123856883 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743875c90 msgr2=0x55f743876070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.551+0000 7fd854779740 1 --2- 172.21.15.28:0/2123856883 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743875c90 0x55f743876070 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fd8340a8f30 tx=0x7fd8340a8bf0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.552+0000 7fd84b400640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8300b2ae0 0x7fd8300b4fa0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fd844007e30 tx=0x7fd844007520 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:14.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.552+0000 7fd854779740 1 -- 172.21.15.28:0/2123856883 shutdown_connections 2024-05-20T20:14:14.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.552+0000 7fd854779740 1 --2- 172.21.15.28:0/2123856883 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f7438765b0 0x55f743880930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.552+0000 7fd854779740 1 --2- 172.21.15.28:0/2123856883 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f743883260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.554 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.553+0000 7fd854779740 1 --2- 172.21.15.28:0/2123856883 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743875c90 0x55f743876070 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.554 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.553+0000 7fd854779740 1 -- 172.21.15.28:0/2123856883 >> 172.21.15.28:0/2123856883 conn(0x55f7438721a0 msgr2=0x55f743872580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:14.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.557+0000 7fd854779740 1 -- 172.21.15.28:0/2123856883 shutdown_connections 2024-05-20T20:14:14.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.558+0000 7fd854779740 1 -- 172.21.15.28:0/2123856883 wait complete. 2024-05-20T20:14:14.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.558+0000 7fd854779740 1 -- start start 2024-05-20T20:14:14.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.559+0000 7fd854779740 1 Processor -- start 2024-05-20T20:14:14.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.562+0000 7fd854779740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f743875c90 0x55f7438808a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.562+0000 7fd854779740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f74387db90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd854779740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f74387e230 0x55f74388cdc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f743875c90 0x55f7438808a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd84a000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f74387db90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f743875c90 0x55f7438808a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53406/0 (socket says 172.21.15.28:53406) 2024-05-20T20:14:14.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd84aa00640 1 -- 172.21.15.28:0/2284268578 learned_addr learned my addr 172.21.15.28:0/2284268578 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:14.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd84b400640 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f74387e230 0x55f74388cdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55f7434bc790 con 0x55f743880e70 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55f74377ebc0 con 0x55f743875c90 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55f743883aa0 con 0x55f74387e230 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.563+0000 7fd854779740 10 client.0 _ll_register_callbacks cb 0x55f74375c400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.564+0000 7fd84aa00640 1 -- 172.21.15.28:0/2284268578 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f74387e230 msgr2=0x55f74388cdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.564+0000 7fd84aa00640 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f74387e230 0x55f74388cdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.564+0000 7fd84aa00640 1 -- 172.21.15.28:0/2284268578 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 msgr2=0x55f74387db90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.564+0000 7fd84aa00640 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f74387db90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.564+0000 7fd84aa00640 1 -- 172.21.15.28:0/2284268578 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f7434eb210 con 0x55f743875c90 2024-05-20T20:14:14.566 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.564+0000 7fd84a000640 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f74387db90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:14.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.566+0000 7fd84aa00640 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f743875c90 0x55f7438808a0 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7fd8340a6210 tx=0x7fd8340a5460 comp rx=0 tx=0).ready entity=mon.2 client_cookie=163400f1878432bc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:14.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.567+0000 7fd838c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:14.571 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.570+0000 7fd838c00640 1 -- 172.21.15.28:0/2284268578 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8340c7070 con 0x55f743875c90 2024-05-20T20:14:14.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.571+0000 7fd838c00640 1 -- 172.21.15.28:0/2284268578 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fd834011830 con 0x55f743875c90 2024-05-20T20:14:14.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.571+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd8340b1e00 con 0x55f743875c90 2024-05-20T20:14:14.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.572+0000 7fd838c00640 1 -- 172.21.15.28:0/2284268578 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd834011a30 con 0x55f743875c90 2024-05-20T20:14:14.572 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.572+0000 7fd854779740 10 client.27257 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:14.573 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.572+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55f7435b7620 con 0x55f743875c90 2024-05-20T20:14:14.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.572+0000 7fd838c00640 1 -- 172.21.15.28:0/2284268578 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd8340b1e00 con 0x55f743875c90 2024-05-20T20:14:14.574 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.573+0000 7fd838c00640 1 -- 172.21.15.28:0/2284268578 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd8340ae4e0 con 0x55f743875c90 2024-05-20T20:14:14.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.574+0000 7fd854779740 10 client.27257 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:14.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.574+0000 7fd854779740 10 client.27257 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:14.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.574+0000 7fd854779740 20 client.27257 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:14.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.574+0000 7fd854779740 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f743892f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.574+0000 7fd854779740 4 client.27257 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:14.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.574+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55f7437e8ae0 con 0x55f743890b40 2024-05-20T20:14:14.575 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.574+0000 7fd84a000640 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f743892f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.575+0000 7fd84a000640 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f743892f30 crc :-1 s=READY pgs=195 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8d9ced62bcd862c6 server_cookie=dd11333634312481 in_seq=0 out_seq=0 2024-05-20T20:14:14.576 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.575+0000 7fd838c00640 10 client.27257 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:14.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.576+0000 7fd838c00640 1 -- 172.21.15.28:0/2284268578 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55f7437e8ae0 con 0x55f743890b40 2024-05-20T20:14:14.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.576+0000 7fd838c00640 10 client.27257 handle_command_reply: tid=0 2024-05-20T20:14:14.577 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.577+0000 7fd854779740 1 client.27257 shutdown 2024-05-20T20:14:14.578 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f743875c90 msgr2=0x55f7438808a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd854779740 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f743875c90 0x55f7438808a0 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7fd8340a6210 tx=0x7fd8340a5460 comp rx=0 tx=0).stop 2024-05-20T20:14:14.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 shutdown_connections 2024-05-20T20:14:14.579 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd854779740 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f743892f30 crc :-1 s=READY pgs=195 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd838c00640 0 client.27257 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd854779740 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f74387e230 0x55f74388cdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd854779740 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743880e70 0x55f74387db90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.578+0000 7fd854779740 1 --2- 172.21.15.28:0/2284268578 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f743875c90 0x55f7438808a0 unknown :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.579+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 >> 172.21.15.28:0/2284268578 conn(0x55f7438721a0 msgr2=0x55f743877ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.579+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 shutdown_connections 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.579+0000 7fd854779740 1 -- 172.21.15.28:0/2284268578 wait complete. 2024-05-20T20:14:14.580 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.579+0000 7fd854779740 20 client.27257 trim_cache size 0 max 16384 2024-05-20T20:14:14.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 Processor -- start 2024-05-20T20:14:14.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 -- start start 2024-05-20T20:14:14.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438803d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743880910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f74387adf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.631 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd82800e730 con 0x55f743875c90 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd828010270 con 0x55f74387e230 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.622+0000 7fd854779740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd82800ee00 con 0x55f743880e70 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f74387adf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f74387adf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49090/0 (socket says 172.21.15.28:49090) 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 -- 172.21.15.28:0/3372500958 learned_addr learned my addr 172.21.15.28:0/3372500958 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 -- 172.21.15.28:0/3372500958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 msgr2=0x55f743880910 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 --2- 172.21.15.28:0/3372500958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743880910 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.632 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 -- 172.21.15.28:0/3372500958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 msgr2=0x55f7438803d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:14.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 --2- 172.21.15.28:0/3372500958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438803d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.623+0000 7fd84b400640 1 -- 172.21.15.28:0/3372500958 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f7438706b0 con 0x55f743880e70 2024-05-20T20:14:14.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.624+0000 7fd84b400640 1 --2- 172.21.15.28:0/3372500958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f74387adf0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fd84401e780 tx=0x7fd844012000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b68c4a8f8eb7c46b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:14.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.625+0000 7fd83b400640 1 -- 172.21.15.28:0/3372500958 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd844010dc0 con 0x55f743880e70 2024-05-20T20:14:14.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.625+0000 7fd83b400640 1 -- 172.21.15.28:0/3372500958 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd844026890 con 0x55f743880e70 2024-05-20T20:14:14.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.625+0000 7fd83b400640 1 -- 172.21.15.28:0/3372500958 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd844019800 con 0x55f743880e70 2024-05-20T20:14:14.633 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.626+0000 7fd854779740 1 -- 172.21.15.28:0/3372500958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 msgr2=0x55f74387adf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.626+0000 7fd854779740 1 --2- 172.21.15.28:0/3372500958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f74387adf0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fd84401e780 tx=0x7fd844012000 comp rx=0 tx=0).stop 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.627+0000 7fd854779740 1 -- 172.21.15.28:0/3372500958 shutdown_connections 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.627+0000 7fd854779740 1 --2- 172.21.15.28:0/3372500958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743880910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.627+0000 7fd854779740 1 --2- 172.21.15.28:0/3372500958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f74387adf0 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.627+0000 7fd854779740 1 --2- 172.21.15.28:0/3372500958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438803d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.627+0000 7fd854779740 1 -- 172.21.15.28:0/3372500958 >> 172.21.15.28:0/3372500958 conn(0x55f7438721a0 msgr2=0x55f743871310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.628+0000 7fd854779740 1 -- 172.21.15.28:0/3372500958 shutdown_connections 2024-05-20T20:14:14.634 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.628+0000 7fd854779740 1 -- 172.21.15.28:0/3372500958 wait complete. 2024-05-20T20:14:14.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.628+0000 7fd854779740 1 -- start start 2024-05-20T20:14:14.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.628+0000 7fd854779740 1 Processor -- start 2024-05-20T20:14:14.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd854779740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438963a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd854779740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743893730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd854779740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f743893c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd854779740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd82800fba0 con 0x55f743875c90 2024-05-20T20:14:14.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd854779740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd82800f4d0 con 0x55f74387e230 2024-05-20T20:14:14.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd854779740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd82800c7c0 con 0x55f743880e70 2024-05-20T20:14:14.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd854779740 10 client.0 _ll_register_callbacks cb 0x7fd8340a87a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:14.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438963a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd84aa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438963a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43768/0 (socket says 172.21.15.28:43768) 2024-05-20T20:14:14.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.629+0000 7fd84aa00640 1 -- 172.21.15.28:0/3566948182 learned_addr learned my addr 172.21.15.28:0/3566948182 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:14.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.630+0000 7fd84a000640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743893730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.636 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.630+0000 7fd84b400640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f743893c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.639 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.630+0000 7fd84aa00640 1 -- 172.21.15.28:0/3566948182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 msgr2=0x55f743893c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:14.639 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.630+0000 7fd84aa00640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f743893c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.631+0000 7fd84aa00640 1 -- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 msgr2=0x55f743893730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.631+0000 7fd84aa00640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743893730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.631+0000 7fd84aa00640 1 -- 172.21.15.28:0/3566948182 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f7438941b0 con 0x55f743875c90 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.631+0000 7fd84a000640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743893730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.631+0000 7fd84aa00640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438963a0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7fd83400bfd0 tx=0x7fd8340a8bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f2e1d837c6f23bc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.631+0000 7fd838c00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.632+0000 7fd838c00640 1 -- 172.21.15.28:0/3566948182 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8340c7070 con 0x55f743875c90 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.632+0000 7fd838c00640 1 -- 172.21.15.28:0/3566948182 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fd8340a5590 con 0x55f743875c90 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.632+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd844026890 con 0x55f743875c90 2024-05-20T20:14:14.640 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.634+0000 7fd838c00640 1 -- 172.21.15.28:0/3566948182 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd8340df790 con 0x55f743875c90 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.634+0000 7fd854779740 10 client.16860 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.634+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55f7438941b0 con 0x55f743875c90 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.634+0000 7fd838c00640 1 -- 172.21.15.28:0/3566948182 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd844026890 con 0x55f743875c90 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd838c00640 1 -- 172.21.15.28:0/3566948182 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd834010380 con 0x55f743875c90 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd854779740 10 client.16860 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd854779740 10 client.16860 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd854779740 20 client.16860 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd854779740 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f7438fd4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd854779740 4 client.16860 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55f7437e22c0 con 0x55f743890b40 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.635+0000 7fd84a000640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f7438fd4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.638+0000 7fd84a000640 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f7438fd4b0 crc :-1 s=READY pgs=196 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=dc283cff3adbd2ac server_cookie=c89306f437123cc2 in_seq=0 out_seq=0 2024-05-20T20:14:14.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.638+0000 7fd838c00640 10 client.16860 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:14.676 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.676+0000 7fd838c00640 1 -- 172.21.15.28:0/3566948182 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55f7437e22c0 con 0x55f743890b40 2024-05-20T20:14:14.677 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.676+0000 7fd838c00640 10 client.16860 handle_command_reply: tid=0 2024-05-20T20:14:14.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.676+0000 7fd854779740 1 client.16860 shutdown 2024-05-20T20:14:14.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.678+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 msgr2=0x55f7438963a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.678 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.678+0000 7fd854779740 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438963a0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7fd83400bfd0 tx=0x7fd8340a8bf0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.678+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 shutdown_connections 2024-05-20T20:14:14.679 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.679+0000 7fd854779740 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f743890b40 0x55f7438fd4b0 crc :-1 s=READY pgs=196 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.679+0000 7fd854779740 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f74387e230 0x55f743893730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.679+0000 7fd838c00640 0 client.16860 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:14.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.679+0000 7fd854779740 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f743880e70 0x55f743893c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.679+0000 7fd854779740 1 --2- 172.21.15.28:0/3566948182 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f743875c90 0x55f7438963a0 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.679+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 >> 172.21.15.28:0/3566948182 conn(0x55f7438721a0 msgr2=0x55f743870a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:14.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.680+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 shutdown_connections 2024-05-20T20:14:14.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.680+0000 7fd854779740 1 -- 172.21.15.28:0/3566948182 wait complete. 2024-05-20T20:14:14.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.680+0000 7fd854779740 20 client.16860 trim_cache size 0 max 16384 2024-05-20T20:14:14.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:14:14.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:14:14.682 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:14:14.684 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8300b2ae0 msgr2=0x7fd8300b4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8300b2ae0 0x7fd8300b4fa0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fd844007e30 tx=0x7fd844007520 comp rx=0 tx=0).stop 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 msgr2=0x7fd84c082f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c082f40 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7fd83400b400 tx=0x7fd83400b8c0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 shutdown_connections 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd8300b2ae0 0x7fd8300b4fa0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd84c07e5a0 0x7fd84c07ea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd84c083480 0x7fd84c07e060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.685 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 --2- 172.21.15.28:0/907839763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd84c10e380 0x7fd84c082f40 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:14.686 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.684+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 >> 172.21.15.28:0/907839763 conn(0x7fd84c076fe0 msgr2=0x7fd84c079340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:14.687 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.686+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 shutdown_connections 2024-05-20T20:14:14.687 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:14.686+0000 7fd851800640 1 -- 172.21.15.28:0/907839763 wait complete. 2024-05-20T20:14:14.834 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:14:15.558 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:15 smithi028 ceph-mon[31178]: pgmap v267: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 231 MiB/s wr, 1.97k op/s 2024-05-20T20:14:15.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:15 smithi154 ceph-mon[36314]: pgmap v267: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 231 MiB/s wr, 1.97k op/s 2024-05-20T20:14:15.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:15 smithi033 ceph-mon[36179]: pgmap v267: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 231 MiB/s wr, 1.97k op/s 2024-05-20T20:14:15.900 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:16.457 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:16 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:14:16.457 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:16 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:14:16.457 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:16 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:16 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:14:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:16 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:14:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:16 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:16.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:16 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:14:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:16 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:14:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:16 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:14:17.553 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:17 smithi028 ceph-mon[31178]: pgmap v268: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 233 MiB/s wr, 1.95k op/s 2024-05-20T20:14:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:17 smithi154 ceph-mon[36314]: pgmap v268: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 233 MiB/s wr, 1.95k op/s 2024-05-20T20:14:17.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:17 smithi033 ceph-mon[36179]: pgmap v268: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 233 MiB/s wr, 1.95k op/s 2024-05-20T20:14:18.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:18 smithi028 ceph-mon[31178]: pgmap v269: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 238 MiB/s wr, 1.81k op/s 2024-05-20T20:14:18.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:18 smithi154 ceph-mon[36314]: pgmap v269: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 238 MiB/s wr, 1.81k op/s 2024-05-20T20:14:18.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:18 smithi033 ceph-mon[36179]: pgmap v269: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 238 MiB/s wr, 1.81k op/s 2024-05-20T20:14:20.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.631+0000 7f4d27c00700 1 -- 172.21.15.28:0/3507108723 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4d20109dd0 msgr2=0x7f4d2010a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.631+0000 7f4d27c00700 1 --2- 172.21.15.28:0/3507108723 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4d20109dd0 0x7f4d2010a1b0 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f4d0c0099c0 tx=0x7f4d0c009cd0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.632+0000 7f4d27c00700 1 -- 172.21.15.28:0/3507108723 shutdown_connections 2024-05-20T20:14:20.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.632+0000 7f4d27c00700 1 --2- 172.21.15.28:0/3507108723 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4d2010eeb0 0x7f4d20116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.632+0000 7f4d27c00700 1 --2- 172.21.15.28:0/3507108723 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4d20109dd0 0x7f4d2010a1b0 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.632+0000 7f4d27c00700 1 --2- 172.21.15.28:0/3507108723 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 0x7f4d2010e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.632+0000 7f4d27c00700 1 -- 172.21.15.28:0/3507108723 >> 172.21.15.28:0/3507108723 conn(0x7f4d20076d10 msgr2=0x7f4d20108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:20.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.633+0000 7f4d27c00700 1 -- 172.21.15.28:0/3507108723 shutdown_connections 2024-05-20T20:14:20.638 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.635+0000 7f4d27c00700 1 -- 172.21.15.28:0/3507108723 wait complete. 2024-05-20T20:14:20.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.639+0000 7f4d27c00700 1 Processor -- start 2024-05-20T20:14:20.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.640+0000 7f4d27c00700 1 -- start start 2024-05-20T20:14:20.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.643+0000 7f4d27c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4d20109dd0 0x7f4d201a0250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d27c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 0x7f4d201a0790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d27c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4d2010eeb0 0x7f4d201a4b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d27c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4d20119e00 con 0x7f4d20109dd0 2024-05-20T20:14:20.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d27c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4d20119c80 con 0x7f4d2010a780 2024-05-20T20:14:20.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d27c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4d20119f80 con 0x7f4d2010eeb0 2024-05-20T20:14:20.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 0x7f4d201a0790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 0x7f4d201a0790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42324/0 (socket says 172.21.15.28:42324) 2024-05-20T20:14:20.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d1fe00700 1 -- 172.21.15.28:0/700486161 learned_addr learned my addr 172.21.15.28:0/700486161 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:20.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.644+0000 7f4d25c00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4d2010eeb0 0x7f4d201a4b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.645+0000 7f4d1fe00700 1 -- 172.21.15.28:0/700486161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4d2010eeb0 msgr2=0x7f4d201a4b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.645+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4d2010eeb0 0x7f4d201a4b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.645+0000 7f4d1fe00700 1 -- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4d20109dd0 msgr2=0x7f4d201a0250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.645+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4d20109dd0 0x7f4d201a0250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.645+0000 7f4d1fe00700 1 -- 172.21.15.28:0/700486161 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4d201a5260 con 0x7f4d2010a780 2024-05-20T20:14:20.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.646+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 0x7f4d201a0790 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f4d1400c8f0 tx=0x7f4d1400cc00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:20.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.648+0000 7f4d1d600700 1 -- 172.21.15.28:0/700486161 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d14007940 con 0x7f4d2010a780 2024-05-20T20:14:20.648 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.648+0000 7f4d1d600700 1 -- 172.21.15.28:0/700486161 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4d1400f450 con 0x7f4d2010a780 2024-05-20T20:14:20.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.649+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4d201a5550 con 0x7f4d2010a780 2024-05-20T20:14:20.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.649+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4d201acdd0 con 0x7f4d2010a780 2024-05-20T20:14:20.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.649+0000 7f4d1d600700 1 -- 172.21.15.28:0/700486161 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d14018660 con 0x7f4d2010a780 2024-05-20T20:14:20.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.661+0000 7f4d1d600700 1 -- 172.21.15.28:0/700486161 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4d1400f870 con 0x7f4d2010a780 2024-05-20T20:14:20.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.664+0000 7f4d1d600700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d080b2aa0 0x7f4d080b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.664+0000 7f4d1d600700 1 -- 172.21.15.28:0/700486161 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f4d140e2460 con 0x7f4d2010a780 2024-05-20T20:14:20.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.666+0000 7f4d325fcb80 1 Processor -- start 2024-05-20T20:14:20.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.666+0000 7f4d325fcb80 1 -- start start 2024-05-20T20:14:20.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.666+0000 7f4d25200700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d080b2aa0 0x7f4d080b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.666+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e4290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.666+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313de260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.666+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 0x55e4313dece0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.667+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e43133ce40 con 0x55e4313de880 2024-05-20T20:14:20.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.667+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e43132c9e0 con 0x55e4313e47d0 2024-05-20T20:14:20.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.667+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e431330a20 con 0x55e4313e3eb0 2024-05-20T20:14:20.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.668+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313de260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.668+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313de260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42332/0 (socket says 172.21.15.28:42332) 2024-05-20T20:14:20.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.668+0000 7f4d1fe00700 1 -- 172.21.15.28:0/632803123 learned_addr learned my addr 172.21.15.28:0/632803123 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:20.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.669+0000 7f4d1fe00700 1 -- 172.21.15.28:0/632803123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 msgr2=0x55e4313e4290 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:14:20.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.669+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/632803123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e4290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.669+0000 7f4d1fe00700 1 -- 172.21.15.28:0/632803123 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 msgr2=0x55e4313dece0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:20.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.669+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/632803123 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 0x55e4313dece0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.669+0000 7f4d1fe00700 1 -- 172.21.15.28:0/632803123 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e4312f5420 con 0x55e4313e47d0 2024-05-20T20:14:20.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.669+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/632803123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313de260 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f4d140a5750 tx=0x7f4d140a9150 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5dc47602232ecb01 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:20.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.670+0000 7f4d12000700 1 -- 172.21.15.28:0/632803123 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d140c3070 con 0x55e4313e47d0 2024-05-20T20:14:20.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.670+0000 7f4d12000700 1 -- 172.21.15.28:0/632803123 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4d140ada80 con 0x55e4313e47d0 2024-05-20T20:14:20.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.671+0000 7f4d325fcb80 1 -- 172.21.15.28:0/632803123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 msgr2=0x55e4313de260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.671+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/632803123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313de260 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f4d140a5750 tx=0x7f4d140a9150 comp rx=0 tx=0).stop 2024-05-20T20:14:20.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.673+0000 7f4d25200700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d080b2aa0 0x7f4d080b4f60 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f4d0c003800 tx=0x7f4d0c000bc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:20.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.674+0000 7f4d325fcb80 1 -- 172.21.15.28:0/632803123 shutdown_connections 2024-05-20T20:14:20.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.674+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/632803123 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 0x55e4313dece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.674+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/632803123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e4290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.674+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/632803123 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313de260 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.674+0000 7f4d325fcb80 1 -- 172.21.15.28:0/632803123 >> 172.21.15.28:0/632803123 conn(0x55e43130cbe0 msgr2=0x55e4313d9010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:20.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.675+0000 7f4d325fcb80 1 -- 172.21.15.28:0/632803123 shutdown_connections 2024-05-20T20:14:20.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.675+0000 7f4d325fcb80 1 -- 172.21.15.28:0/632803123 wait complete. 2024-05-20T20:14:20.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.675+0000 7f4d325fcb80 1 -- start start 2024-05-20T20:14:20.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.676+0000 7f4d325fcb80 1 Processor -- start 2024-05-20T20:14:20.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.678+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313ea4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.679+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e77d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.679+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e4313e7d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.679+0000 7f4d25200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313ea4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.679+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e77d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.680+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e77d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47050/0 (socket says 172.21.15.28:47050) 2024-05-20T20:14:20.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.680+0000 7f4d25200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313ea4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42340/0 (socket says 172.21.15.28:42340) 2024-05-20T20:14:20.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.680+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e431232fa0 con 0x55e4313e47d0 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.680+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e4312cd760 con 0x55e4313de880 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.680+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e4313eb2e0 con 0x55e4313e3eb0 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.680+0000 7f4d25200700 1 -- 172.21.15.28:0/181215451 learned_addr learned my addr 172.21.15.28:0/181215451 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.681+0000 7f4d325fcb80 10 client.0 _ll_register_callbacks cb 0x55e4312d6c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.681+0000 7f4d25c00700 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e4313e7d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.682+0000 7f4d1fe00700 1 -- 172.21.15.28:0/181215451 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 msgr2=0x55e4313ea4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.682+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313ea4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.682+0000 7f4d1fe00700 1 -- 172.21.15.28:0/181215451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 msgr2=0x55e4313e7d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.682+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e4313e7d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.682+0000 7f4d1fe00700 1 -- 172.21.15.28:0/181215451 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e43127c8d0 con 0x55e4313e3eb0 2024-05-20T20:14:20.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.683+0000 7f4d25c00700 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e4313e7d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:14:20.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.684+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e77d0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f4d14005eb0 tx=0x7f4d140a9f10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=249f07ce76f7960e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:20.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.685+0000 7f4d06a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:14:20.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.685+0000 7f4d06a00700 1 -- 172.21.15.28:0/181215451 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d140c3070 con 0x55e4313e3eb0 2024-05-20T20:14:20.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.685+0000 7f4d06a00700 1 -- 172.21.15.28:0/181215451 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f4d140a5cc0 con 0x55e4313e3eb0 2024-05-20T20:14:20.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.685+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4d140ada80 con 0x55e4313e3eb0 2024-05-20T20:14:20.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.687+0000 7f4d06a00700 1 -- 172.21.15.28:0/181215451 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4d140b2530 con 0x55e4313e3eb0 2024-05-20T20:14:20.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.687+0000 7f4d325fcb80 10 client.27172 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:20.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.687+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e4312fd060 con 0x55e4313e3eb0 2024-05-20T20:14:20.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d06a00700 1 -- 172.21.15.28:0/181215451 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4d140ada80 con 0x55e4313e3eb0 2024-05-20T20:14:20.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d06a00700 1 -- 172.21.15.28:0/181215451 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4d14020020 con 0x55e4313e3eb0 2024-05-20T20:14:20.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d325fcb80 10 client.27172 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:20.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d325fcb80 10 client.27172 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d325fcb80 20 client.27172 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e4313f7cb0 0x55e4313fa0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d325fcb80 4 client.27172 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.688+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e43133ed90 con 0x55e4313f7cb0 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.689+0000 7f4d25200700 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e4313f7cb0 0x55e4313fa0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.689+0000 7f4d06a00700 10 client.27172 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.691+0000 7f4d25200700 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e4313f7cb0 0x55e4313fa0a0 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=b78ea5a68e02df1a server_cookie=aa96af61eb509480 in_seq=0 out_seq=0 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.692+0000 7f4d06a00700 1 -- 172.21.15.28:0/181215451 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55e43133ed90 con 0x55e4313f7cb0 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.692+0000 7f4d06a00700 10 client.27172 handle_command_reply: tid=0 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.692+0000 7f4d325fcb80 1 client.27172 shutdown 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 msgr2=0x55e4313e77d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e77d0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f4d14005eb0 tx=0x7f4d140a9f10 comp rx=0 tx=0).stop 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 shutdown_connections 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e4313f7cb0 0x55e4313fa0a0 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e4313e7d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d06a00700 0 client.27172 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313ea4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/181215451 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313e77d0 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.694+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 >> 172.21.15.28:0/181215451 conn(0x55e43130cbe0 msgr2=0x55e4313dd250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.695+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 shutdown_connections 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.698+0000 7f4d325fcb80 1 -- 172.21.15.28:0/181215451 wait complete. 2024-05-20T20:14:20.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.698+0000 7f4d325fcb80 20 client.27172 trim_cache size 0 max 16384 2024-05-20T20:14:20.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.766+0000 7f4d325fcb80 1 Processor -- start 2024-05-20T20:14:20.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.766+0000 7f4d325fcb80 1 -- start start 2024-05-20T20:14:20.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.766+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 0x55e4313de260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.766+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313dec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.766+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313e18f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.766+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4d201b4500 con 0x55e4313de880 2024-05-20T20:14:20.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.766+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4d201b6040 con 0x55e4313e47d0 2024-05-20T20:14:20.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.767+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313dec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.767+0000 7f4d1fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313dec60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47060/0 (socket says 172.21.15.28:47060) 2024-05-20T20:14:20.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.767+0000 7f4d1fe00700 1 -- 172.21.15.28:0/1564146800 learned_addr learned my addr 172.21.15.28:0/1564146800 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:20.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.767+0000 7f4d25c00700 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313e18f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.770+0000 7f4d325fcb80 1 -- 172.21.15.28:0/1564146800 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4d201b4bd0 con 0x55e4313e3eb0 2024-05-20T20:14:20.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.770+0000 7f4d25200700 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 0x55e4313de260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.773+0000 7f4d25c00700 1 -- 172.21.15.28:0/1564146800 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 msgr2=0x55e4313dec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:20.774 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.774+0000 7f4d25c00700 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313dec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.774+0000 7f4d25c00700 1 -- 172.21.15.28:0/1564146800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 msgr2=0x55e4313de260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:20.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.774+0000 7f4d25c00700 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 0x55e4313de260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.775+0000 7f4d25c00700 1 -- 172.21.15.28:0/1564146800 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e4313e5d60 con 0x55e4313e47d0 2024-05-20T20:14:20.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.777+0000 7f4d25c00700 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313e18f0 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f4d18003eb0 tx=0x7f4d18009ac0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=44893a61088bcff0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:20.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.779+0000 7f4d11600700 1 -- 172.21.15.28:0/1564146800 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d1801a020 con 0x55e4313e47d0 2024-05-20T20:14:20.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.779+0000 7f4d11600700 1 -- 172.21.15.28:0/1564146800 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4d18007500 con 0x55e4313e47d0 2024-05-20T20:14:20.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.779+0000 7f4d11600700 1 -- 172.21.15.28:0/1564146800 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d180166c0 con 0x55e4313e47d0 2024-05-20T20:14:20.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.781+0000 7f4d325fcb80 1 -- 172.21.15.28:0/1564146800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 msgr2=0x55e4313e18f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.781+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313e18f0 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f4d18003eb0 tx=0x7f4d18009ac0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.783+0000 7f4d325fcb80 1 -- 172.21.15.28:0/1564146800 shutdown_connections 2024-05-20T20:14:20.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.783+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313e47d0 0x55e4313e18f0 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.783+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313de880 0x55e4313de260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.783+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/1564146800 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e4313dec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.783+0000 7f4d325fcb80 1 -- 172.21.15.28:0/1564146800 >> 172.21.15.28:0/1564146800 conn(0x55e43130cbe0 msgr2=0x55e4313e60e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:20.786 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.786+0000 7f4d325fcb80 1 -- 172.21.15.28:0/1564146800 shutdown_connections 2024-05-20T20:14:20.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.786+0000 7f4d325fcb80 1 -- 172.21.15.28:0/1564146800 wait complete. 2024-05-20T20:14:20.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.787+0000 7f4d325fcb80 1 -- start start 2024-05-20T20:14:20.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.788+0000 7f4d325fcb80 1 Processor -- start 2024-05-20T20:14:20.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.791+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313fd820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.791+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e43145d4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.791+0000 7f4d325fcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e43145da50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4d201b5970 con 0x55e4313e47d0 2024-05-20T20:14:20.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4d201b52a0 con 0x55e4313de880 2024-05-20T20:14:20.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d325fcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4d201b24a0 con 0x55e4313e3eb0 2024-05-20T20:14:20.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d25200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313fd820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d25200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313fd820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42348/0 (socket says 172.21.15.28:42348) 2024-05-20T20:14:20.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d25200700 1 -- 172.21.15.28:0/565044941 learned_addr learned my addr 172.21.15.28:0/565044941 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:20.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e43145d4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.792+0000 7f4d325fcb80 10 client.0 _ll_register_callbacks cb 0x55e4312d6c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:20.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.796+0000 7f4d25200700 1 -- 172.21.15.28:0/565044941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 msgr2=0x55e43145d4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:20.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.796+0000 7f4d25200700 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e43145d4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.796+0000 7f4d25200700 1 -- 172.21.15.28:0/565044941 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 msgr2=0x55e43145da50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:20.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.796+0000 7f4d25200700 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e43145da50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.796+0000 7f4d25200700 1 -- 172.21.15.28:0/565044941 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e43145e290 con 0x55e4313de880 2024-05-20T20:14:20.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.797+0000 7f4d06a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:20.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.797+0000 7f4d25200700 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313fd820 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x55e43132cb30 tx=0x7f4d0c00b540 comp rx=0 tx=0).ready entity=mon.2 client_cookie=50b4a6dc459c6a9f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:20.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.799+0000 7f4d06a00700 1 -- 172.21.15.28:0/565044941 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4d0c034070 con 0x55e4313de880 2024-05-20T20:14:20.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.800+0000 7f4d06a00700 1 -- 172.21.15.28:0/565044941 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f4d0c00b790 con 0x55e4313de880 2024-05-20T20:14:20.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.800+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4d18007500 con 0x55e4313de880 2024-05-20T20:14:20.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.802+0000 7f4d06a00700 1 -- 172.21.15.28:0/565044941 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4d0c040050 con 0x55e4313de880 2024-05-20T20:14:20.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.802+0000 7f4d325fcb80 10 client.27275 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:20.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.802+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e4314735b0 con 0x55e4313de880 2024-05-20T20:14:20.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.803+0000 7f4d06a00700 1 -- 172.21.15.28:0/565044941 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4d18007500 con 0x55e4313de880 2024-05-20T20:14:20.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.805+0000 7f4d06a00700 1 -- 172.21.15.28:0/565044941 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4d0c030840 con 0x55e4313de880 2024-05-20T20:14:20.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.807+0000 7f4d325fcb80 10 client.27275 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:20.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.807+0000 7f4d325fcb80 10 client.27275 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:14:20.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.807+0000 7f4d325fcb80 20 client.27275 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:20.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.807+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e431473c60 0x55e4314760a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:20.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.810+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e431473c60 0x55e4314760a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:20.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.812+0000 7f4d1fe00700 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e431473c60 0x55e4314760a0 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=5c702f2c2092cdc8 server_cookie=7a6302ca8b0e33d1 in_seq=0 out_seq=0 2024-05-20T20:14:20.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.813+0000 7f4d325fcb80 4 client.27275 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:14:20.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.813+0000 7f4d06a00700 10 client.27275 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:14:20.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.813+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55e431338570 con 0x55e431473c60 2024-05-20T20:14:20.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.816+0000 7f4d06a00700 1 -- 172.21.15.28:0/565044941 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55e431338570 con 0x55e431473c60 2024-05-20T20:14:20.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.816+0000 7f4d06a00700 10 client.27275 handle_command_reply: tid=0 2024-05-20T20:14:20.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.816+0000 7f4d325fcb80 1 client.27275 shutdown 2024-05-20T20:14:20.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.816+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 msgr2=0x55e4313fd820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.816+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313fd820 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x55e43132cb30 tx=0x7f4d0c00b540 comp rx=0 tx=0).stop 2024-05-20T20:14:20.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.817+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 shutdown_connections 2024-05-20T20:14:20.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.817+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e431473c60 0x55e4314760a0 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.817+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4313e47d0 0x55e43145da50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.817+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4313de880 0x55e4313fd820 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.817+0000 7f4d06a00700 0 client.27275 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:14:20.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.817+0000 7f4d325fcb80 1 --2- 172.21.15.28:0/565044941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4313e3eb0 0x55e43145d4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.817+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 >> 172.21.15.28:0/565044941 conn(0x55e43130cbe0 msgr2=0x55e4313e6050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:20.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.820+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 shutdown_connections 2024-05-20T20:14:20.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.820+0000 7f4d325fcb80 1 -- 172.21.15.28:0/565044941 wait complete. 2024-05-20T20:14:20.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.821+0000 7f4d325fcb80 20 client.27275 trim_cache size 0 max 16384 2024-05-20T20:14:20.822 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:14:20.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.825+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d080b2aa0 msgr2=0x7f4d080b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.825+0000 7f4d27c00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d080b2aa0 0x7f4d080b4f60 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f4d0c003800 tx=0x7f4d0c000bc0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.830+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 msgr2=0x7f4d201a0790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:20.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.830+0000 7f4d27c00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 0x7f4d201a0790 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f4d1400c8f0 tx=0x7f4d1400cc00 comp rx=0 tx=0).stop 2024-05-20T20:14:20.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.832+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 shutdown_connections 2024-05-20T20:14:20.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.832+0000 7f4d27c00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4d2010eeb0 0x7f4d201a4b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.832+0000 7f4d27c00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4d20109dd0 0x7f4d201a0250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.832+0000 7f4d27c00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4d080b2aa0 0x7f4d080b4f60 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.832+0000 7f4d27c00700 1 --2- 172.21.15.28:0/700486161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4d2010a780 0x7f4d201a0790 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:20.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.832+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 >> 172.21.15.28:0/700486161 conn(0x7f4d20076d10 msgr2=0x7f4d20078320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:20.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.836+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 shutdown_connections 2024-05-20T20:14:20.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:20.837+0000 7f4d27c00700 1 -- 172.21.15.28:0/700486161 wait complete. 2024-05-20T20:14:21.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:21 smithi028 ceph-mon[31178]: pgmap v270: 97 pgs: 97 active+clean; 58 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 279 MiB/s wr, 1.46k op/s 2024-05-20T20:14:21.327 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:21 smithi154 ceph-mon[36314]: pgmap v270: 97 pgs: 97 active+clean; 58 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 279 MiB/s wr, 1.46k op/s 2024-05-20T20:14:21.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:21 smithi033 ceph-mon[36179]: pgmap v270: 97 pgs: 97 active+clean; 58 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 279 MiB/s wr, 1.46k op/s 2024-05-20T20:14:22.502 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:14:23.262 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:23.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:23 smithi154 ceph-mon[36314]: pgmap v271: 97 pgs: 97 active+clean; 58 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 245 MiB/s wr, 1.28k op/s 2024-05-20T20:14:23.399 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:23 smithi033 ceph-mon[36179]: pgmap v271: 97 pgs: 97 active+clean; 58 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 245 MiB/s wr, 1.28k op/s 2024-05-20T20:14:23.515 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:23 smithi028 ceph-mon[31178]: pgmap v271: 97 pgs: 97 active+clean; 58 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 245 MiB/s wr, 1.28k op/s 2024-05-20T20:14:25.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:25 smithi028 ceph-mon[31178]: pgmap v272: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 358 MiB/s wr, 1.96k op/s 2024-05-20T20:14:25.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:25 smithi154 ceph-mon[36314]: pgmap v272: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 358 MiB/s wr, 1.96k op/s 2024-05-20T20:14:25.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:25 smithi033 ceph-mon[36179]: pgmap v272: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 358 MiB/s wr, 1.96k op/s 2024-05-20T20:14:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.051+0000 7ff96d400700 1 -- 172.21.15.28:0/1796734835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff96806b790 msgr2=0x7ff96806bbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.051+0000 7ff96d400700 1 --2- 172.21.15.28:0/1796734835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff96806b790 0x7ff96806bbf0 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7ff954009990 tx=0x7ff954009ca0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.052+0000 7ff96d400700 1 -- 172.21.15.28:0/1796734835 shutdown_connections 2024-05-20T20:14:26.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.052+0000 7ff96d400700 1 --2- 172.21.15.28:0/1796734835 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff968105f80 0x7ff968106410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.052+0000 7ff96d400700 1 --2- 172.21.15.28:0/1796734835 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff968108470 0x7ff96806b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.052+0000 7ff96d400700 1 --2- 172.21.15.28:0/1796734835 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff96806b790 0x7ff96806bbf0 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.052+0000 7ff96d400700 1 -- 172.21.15.28:0/1796734835 >> 172.21.15.28:0/1796734835 conn(0x7ff9680793f0 msgr2=0x7ff968079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:26.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.052+0000 7ff96d400700 1 -- 172.21.15.28:0/1796734835 shutdown_connections 2024-05-20T20:14:26.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.052+0000 7ff96d400700 1 -- 172.21.15.28:0/1796734835 wait complete. 2024-05-20T20:14:26.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 Processor -- start 2024-05-20T20:14:26.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 -- start start 2024-05-20T20:14:26.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff96806b790 0x7ff9681a01c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff968105f80 0x7ff9681a0700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff968108470 0x7ff9681a4a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff968119cd0 con 0x7ff968105f80 2024-05-20T20:14:26.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff968119b50 con 0x7ff968108470 2024-05-20T20:14:26.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.054+0000 7ff96d400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff968119e50 con 0x7ff96806b790 2024-05-20T20:14:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff96806b790 0x7ff9681a01c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff96806b790 0x7ff9681a01c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47088/0 (socket says 172.21.15.28:47088) 2024-05-20T20:14:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 -- 172.21.15.28:0/3457400592 learned_addr learned my addr 172.21.15.28:0/3457400592 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff965e00700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff968105f80 0x7ff9681a0700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff967200700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff968108470 0x7ff9681a4a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 -- 172.21.15.28:0/3457400592 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff968108470 msgr2=0x7ff9681a4a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff968108470 0x7ff9681a4a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 -- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff968105f80 msgr2=0x7ff9681a0700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff968105f80 0x7ff9681a0700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 -- 172.21.15.28:0/3457400592 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff9681a51d0 con 0x7ff96806b790 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.055+0000 7ff966800700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff96806b790 0x7ff9681a01c0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7ff95800eb30 tx=0x7ff95800ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.056+0000 7ff95f400700 1 -- 172.21.15.28:0/3457400592 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff95800cbd0 con 0x7ff96806b790 2024-05-20T20:14:26.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.056+0000 7ff95f400700 1 -- 172.21.15.28:0/3457400592 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff9580044d0 con 0x7ff96806b790 2024-05-20T20:14:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.056+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff9681a54c0 con 0x7ff96806b790 2024-05-20T20:14:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.056+0000 7ff95f400700 1 -- 172.21.15.28:0/3457400592 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff958003e30 con 0x7ff96806b790 2024-05-20T20:14:26.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.056+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff9681acd40 con 0x7ff96806b790 2024-05-20T20:14:26.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.059+0000 7ff95f400700 1 -- 172.21.15.28:0/3457400592 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ff958004020 con 0x7ff96806b790 2024-05-20T20:14:26.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.061+0000 7ff977dcdb80 1 Processor -- start 2024-05-20T20:14:26.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.061+0000 7ff977dcdb80 1 -- start start 2024-05-20T20:14:26.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff95f400700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff94c0b2a70 0x7ff94c0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988610 0x55ba639889f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba63982b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba639835c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ba638e0e40 con 0x55ba63983140 2024-05-20T20:14:26.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ba638d09e0 con 0x55ba63988610 2024-05-20T20:14:26.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ba638d4a20 con 0x55ba63988f30 2024-05-20T20:14:26.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff95f400700 1 -- 172.21.15.28:0/3457400592 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7ff9580e2910 con 0x7ff96806b790 2024-05-20T20:14:26.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff966800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988610 0x55ba639889f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff966800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988610 0x55ba639889f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42364/0 (socket says 172.21.15.28:42364) 2024-05-20T20:14:26.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff966800700 1 -- 172.21.15.28:0/3284392113 learned_addr learned my addr 172.21.15.28:0/3284392113 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:26.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.062+0000 7ff967200700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff94c0b2a70 0x7ff94c0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff967200700 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba639835c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff966800700 1 -- 172.21.15.28:0/3284392113 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 msgr2=0x55ba63982b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:26.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff966800700 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba63982b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff966800700 1 -- 172.21.15.28:0/3284392113 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 msgr2=0x55ba639835c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:26.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff966800700 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba639835c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.064 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff966800700 1 -- 172.21.15.28:0/3284392113 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ba63899420 con 0x55ba63988610 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff966800700 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988610 0x55ba639889f0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7ff9580a5c00 tx=0x7ff9580098f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d26916e01918a511 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff953e00700 1 -- 172.21.15.28:0/3284392113 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9580a5310 con 0x55ba63988610 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff953e00700 1 -- 172.21.15.28:0/3284392113 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff9580b2330 con 0x55ba63988610 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.063+0000 7ff953e00700 1 -- 172.21.15.28:0/3284392113 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9580e0480 con 0x55ba63988610 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 -- 172.21.15.28:0/3284392113 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988610 msgr2=0x55ba639889f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff967200700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff94c0b2a70 0x7ff94c0b4f30 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x55ba63983b00 tx=0x7ff960009920 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988610 0x55ba639889f0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7ff9580a5c00 tx=0x7ff9580098f0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 -- 172.21.15.28:0/3284392113 shutdown_connections 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba639835c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988610 0x55ba639889f0 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/3284392113 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba63982b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 -- 172.21.15.28:0/3284392113 >> 172.21.15.28:0/3284392113 conn(0x55ba638b0be0 msgr2=0x55ba6397d690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:26.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.064+0000 7ff977dcdb80 1 -- 172.21.15.28:0/3284392113 shutdown_connections 2024-05-20T20:14:26.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.065+0000 7ff977dcdb80 1 -- 172.21.15.28:0/3284392113 wait complete. 2024-05-20T20:14:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.065+0000 7ff977dcdb80 1 -- start start 2024-05-20T20:14:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.065+0000 7ff977dcdb80 1 Processor -- start 2024-05-20T20:14:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba6398ed00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988610 0x55ba6398c0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba6398c5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ba637d6fa0 con 0x55ba63988610 2024-05-20T20:14:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ba63871760 con 0x55ba63983140 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ba6398f9e0 con 0x55ba63988f30 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff977dcdb80 10 client.0 _ll_register_callbacks cb 0x55ba6387ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff966800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba6398ed00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff966800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba6398ed00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42366/0 (socket says 172.21.15.28:42366) 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff966800700 1 -- 172.21.15.28:0/1007070467 learned_addr learned my addr 172.21.15.28:0/1007070467 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff965e00700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988610 0x55ba6398c0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff967200700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba6398c5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.066+0000 7ff966800700 1 -- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 msgr2=0x55ba6398c5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff966800700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba6398c5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff966800700 1 -- 172.21.15.28:0/1007070467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988610 msgr2=0x55ba6398c0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff966800700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988610 0x55ba6398c0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff966800700 1 -- 172.21.15.28:0/1007070467 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ba638208d0 con 0x55ba63983140 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff967200700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba6398c5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:14:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff966800700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba6398ed00 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7ff9580a9b10 tx=0x7ff9580aa090 comp rx=0 tx=0).ready entity=mon.2 client_cookie=edc2e00172f94af4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff950c00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff950c00700 1 -- 172.21.15.28:0/1007070467 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9580e0480 con 0x55ba63983140 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff950c00700 1 -- 172.21.15.28:0/1007070467 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7ff9580b2760 con 0x55ba63983140 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff9580b2330 con 0x55ba63983140 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff950c00700 1 -- 172.21.15.28:0/1007070467 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff9580a5310 con 0x55ba63983140 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff977dcdb80 10 client.27287 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.067+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55ba638a1060 con 0x55ba63983140 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff950c00700 1 -- 172.21.15.28:0/1007070467 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff9580b2330 con 0x55ba63983140 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff950c00700 1 -- 172.21.15.28:0/1007070467 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff958022a60 con 0x55ba63983140 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff977dcdb80 10 client.27287 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff977dcdb80 10 client.27287 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff977dcdb80 20 client.27287 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:26.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba6399c440 0x55ba6399e830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff977dcdb80 4 client.27287 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:26.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ba638e2d90 con 0x55ba6399c440 2024-05-20T20:14:26.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.068+0000 7ff965e00700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba6399c440 0x55ba6399e830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.069+0000 7ff965e00700 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba6399c440 0x55ba6399e830 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=640dab27ea8d72a server_cookie=6bf2e64a80106802 in_seq=0 out_seq=0 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.069+0000 7ff950c00700 10 client.27287 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.069+0000 7ff950c00700 1 -- 172.21.15.28:0/1007070467 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55ba638e2d90 con 0x55ba6399c440 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.069+0000 7ff950c00700 10 client.27287 handle_command_reply: tid=0 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 client.27287 shutdown 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 msgr2=0x55ba6398ed00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba6398ed00 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7ff9580a9b10 tx=0x7ff9580aa090 comp rx=0 tx=0).stop 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 shutdown_connections 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988f30 0x55ba6398c5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba6398ed00 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba6399c440 0x55ba6399e830 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/1007070467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988610 0x55ba6398c0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 >> 172.21.15.28:0/1007070467 conn(0x55ba638b0be0 msgr2=0x55ba6397cd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:26.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff950c00700 0 client.27287 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:14:26.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 shutdown_connections 2024-05-20T20:14:26.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 1 -- 172.21.15.28:0/1007070467 wait complete. 2024-05-20T20:14:26.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.070+0000 7ff977dcdb80 20 client.27287 trim_cache size 0 max 16384 2024-05-20T20:14:26.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.087+0000 7ff977dcdb80 1 Processor -- start 2024-05-20T20:14:26.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.087+0000 7ff977dcdb80 1 -- start start 2024-05-20T20:14:26.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba63989c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.087 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6397d4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988f30 0x55ba6398a860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff9681b4520 con 0x55ba63983140 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff9681b6060 con 0x55ba63988f30 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff9681b4bf0 con 0x55ba63988610 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff965e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6397d4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff966800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba63989c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff965e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6397d4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:47108/0 (socket says 172.21.15.28:47108) 2024-05-20T20:14:26.088 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.088+0000 7ff965e00700 1 -- 172.21.15.28:0/240868356 learned_addr learned my addr 172.21.15.28:0/240868356 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.089+0000 7ff965e00700 1 -- 172.21.15.28:0/240868356 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988f30 msgr2=0x55ba6398a860 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:14:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.089+0000 7ff967200700 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988f30 0x55ba6398a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.089+0000 7ff965e00700 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988f30 0x55ba6398a860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.089+0000 7ff965e00700 1 -- 172.21.15.28:0/240868356 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 msgr2=0x55ba63989c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.089+0000 7ff965e00700 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba63989c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.089+0000 7ff965e00700 1 -- 172.21.15.28:0/240868356 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ba6398a610 con 0x55ba63988610 2024-05-20T20:14:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.089+0000 7ff966800700 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba63989c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:26.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.090+0000 7ff965e00700 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6397d4f0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7ff9540098e0 tx=0x7ff954005e80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3a70e275a884975b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:26.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.090+0000 7ff953400700 1 -- 172.21.15.28:0/240868356 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff95402a020 con 0x55ba63988610 2024-05-20T20:14:26.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.090+0000 7ff953400700 1 -- 172.21.15.28:0/240868356 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff95400bbd0 con 0x55ba63988610 2024-05-20T20:14:26.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.090+0000 7ff953400700 1 -- 172.21.15.28:0/240868356 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff954003d70 con 0x55ba63988610 2024-05-20T20:14:26.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 -- 172.21.15.28:0/240868356 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 msgr2=0x55ba6397d4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6397d4f0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7ff9540098e0 tx=0x7ff954005e80 comp rx=0 tx=0).stop 2024-05-20T20:14:26.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 -- 172.21.15.28:0/240868356 shutdown_connections 2024-05-20T20:14:26.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63988f30 0x55ba6398a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63983140 0x55ba63989c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/240868356 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6397d4f0 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 -- 172.21.15.28:0/240868356 >> 172.21.15.28:0/240868356 conn(0x55ba638b0be0 msgr2=0x55ba6398c4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:26.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.091+0000 7ff977dcdb80 1 -- 172.21.15.28:0/240868356 shutdown_connections 2024-05-20T20:14:26.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.093+0000 7ff977dcdb80 1 -- 172.21.15.28:0/240868356 wait complete. 2024-05-20T20:14:26.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.093+0000 7ff977dcdb80 1 -- start start 2024-05-20T20:14:26.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 1 Processor -- start 2024-05-20T20:14:26.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba639a1e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6399f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988f30 0x55ba6399f710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff9681b5990 con 0x55ba63988f30 2024-05-20T20:14:26.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff9681b52c0 con 0x55ba63983140 2024-05-20T20:14:26.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff9681b24e0 con 0x55ba63988610 2024-05-20T20:14:26.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff977dcdb80 10 client.0 _ll_register_callbacks cb 0x55ba6387ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:26.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff966800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba639a1e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff966800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba639a1e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:42392/0 (socket says 172.21.15.28:42392) 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.094+0000 7ff966800700 1 -- 172.21.15.28:0/2023562486 learned_addr learned my addr 172.21.15.28:0/2023562486 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff967200700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988f30 0x55ba6399f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff965e00700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6399f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff966800700 1 -- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 msgr2=0x55ba6399f1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff966800700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6399f1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff966800700 1 -- 172.21.15.28:0/2023562486 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988f30 msgr2=0x55ba6399f710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff966800700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988f30 0x55ba6399f710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff966800700 1 -- 172.21.15.28:0/2023562486 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ba6399fe20 con 0x55ba63983140 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff965e00700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6399f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff966800700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba639a1e30 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7ff9580a9880 tx=0x7ff95800c6e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a3499115588463c0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:26.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff950c00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.095+0000 7ff950c00700 1 -- 172.21.15.28:0/2023562486 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff9580c4260 con 0x55ba63983140 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff95400bbd0 con 0x55ba63983140 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff950c00700 1 -- 172.21.15.28:0/2023562486 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7ff9580a8ff0 con 0x55ba63983140 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff950c00700 1 -- 172.21.15.28:0/2023562486 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff958004310 con 0x55ba63983140 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 10 client.27290 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55ba6399fe20 con 0x55ba63983140 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff950c00700 1 -- 172.21.15.28:0/2023562486 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff95400bbd0 con 0x55ba63983140 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff950c00700 1 -- 172.21.15.28:0/2023562486 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff9580a8850 con 0x55ba63983140 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 10 client.27290 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 10 client.27290 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 20 client.27290 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba63a06f10 0x55ba63a09330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 4 client.27290 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:14:26.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.096+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55ba638dc570 con 0x55ba63a06f10 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.097+0000 7ff965e00700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba63a06f10 0x55ba63a09330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff950c00700 10 client.27290 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff965e00700 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba63a06f10 0x55ba63a09330 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=911171025be15292 server_cookie=63d0709883e2d4c in_seq=0 out_seq=0 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff950c00700 1 -- 172.21.15.28:0/2023562486 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55ba638dc570 con 0x55ba63a06f10 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff950c00700 10 client.27290 handle_command_reply: tid=0 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 client.27290 shutdown 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 msgr2=0x55ba639a1e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba639a1e30 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7ff9580a9880 tx=0x7ff95800c6e0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 shutdown_connections 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55ba63a06f10 0x55ba63a09330 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ba63988f30 0x55ba6399f710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff950c00700 0 client.27290 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:14:26.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ba63983140 0x55ba639a1e30 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 --2- 172.21.15.28:0/2023562486 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ba63988610 0x55ba6399f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.099+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 >> 172.21.15.28:0/2023562486 conn(0x55ba638b0be0 msgr2=0x55ba639871c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:26.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.100+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 shutdown_connections 2024-05-20T20:14:26.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.100+0000 7ff977dcdb80 1 -- 172.21.15.28:0/2023562486 wait complete. 2024-05-20T20:14:26.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.100+0000 7ff977dcdb80 20 client.27290 trim_cache size 0 max 16384 2024-05-20T20:14:26.104 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:14:26.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff94c0b2a70 msgr2=0x7ff94c0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff94c0b2a70 0x7ff94c0b4f30 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x55ba63983b00 tx=0x7ff960009920 comp rx=0 tx=0).stop 2024-05-20T20:14:26.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff96806b790 msgr2=0x7ff9681a01c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:26.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff96806b790 0x7ff9681a01c0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7ff95800eb30 tx=0x7ff95800ee40 comp rx=0 tx=0).stop 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 shutdown_connections 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff94c0b2a70 0x7ff94c0b4f30 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff968108470 0x7ff9681a4a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff96806b790 0x7ff9681a01c0 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 --2- 172.21.15.28:0/3457400592 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff968105f80 0x7ff9681a0700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 >> 172.21.15.28:0/3457400592 conn(0x7ff9680793f0 msgr2=0x7ff968104f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 shutdown_connections 2024-05-20T20:14:26.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:26.101+0000 7ff96d400700 1 -- 172.21.15.28:0/3457400592 wait complete. 2024-05-20T20:14:27.275 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:14:27.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:27 smithi154 ceph-mon[36314]: pgmap v273: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 282 MiB/s wr, 1.38k op/s 2024-05-20T20:14:27.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:27 smithi033 ceph-mon[36179]: pgmap v273: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 282 MiB/s wr, 1.38k op/s 2024-05-20T20:14:27.432 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:27 smithi028 ceph-mon[31178]: pgmap v273: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 282 MiB/s wr, 1.38k op/s 2024-05-20T20:14:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.941+0000 7f1a45000640 1 -- 172.21.15.28:0/1840152647 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 msgr2=0x7f1a4010f940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:27.940 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.941+0000 7f1a45000640 1 --2- 172.21.15.28:0/1840152647 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a4010f940 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f1a38009be0 tx=0x7f1a3802a6c0 comp rx=0 tx=0).stop 2024-05-20T20:14:27.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.945+0000 7f1a45000640 1 -- 172.21.15.28:0/1840152647 shutdown_connections 2024-05-20T20:14:27.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.946+0000 7f1a45000640 1 --2- 172.21.15.28:0/1840152647 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a4010f940 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:27.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.946+0000 7f1a45000640 1 --2- 172.21.15.28:0/1840152647 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a40102a70 0x7f1a40102ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:27.945 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.946+0000 7f1a45000640 1 --2- 172.21.15.28:0/1840152647 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a40108a80 0x7f1a40108e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:27.946 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.946+0000 7f1a45000640 1 -- 172.21.15.28:0/1840152647 >> 172.21.15.28:0/1840152647 conn(0x7f1a400fe500 msgr2=0x7f1a40100920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:27.947 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.947+0000 7f1a45000640 1 -- 172.21.15.28:0/1840152647 shutdown_connections 2024-05-20T20:14:27.950 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.947+0000 7f1a45000640 1 -- 172.21.15.28:0/1840152647 wait complete. 2024-05-20T20:14:27.950 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.948+0000 7f1a45000640 1 Processor -- start 2024-05-20T20:14:27.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.952+0000 7f1a45000640 1 -- start start 2024-05-20T20:14:27.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.955+0000 7f1a45000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a40102a70 0x7f1a4019fca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:27.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.955+0000 7f1a45000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a401a01e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:27.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.955+0000 7f1a3de00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a40102a70 0x7f1a4019fca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:27.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.956+0000 7f1a3d400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a401a01e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:27.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.956+0000 7f1a3d400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a401a01e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60684/0 (socket says 172.21.15.28:60684) 2024-05-20T20:14:27.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.956+0000 7f1a45000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a40108a80 0x7f1a4010ea60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:27.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.956+0000 7f1a3d400640 1 -- 172.21.15.28:0/457988684 learned_addr learned my addr 172.21.15.28:0/457988684 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:27.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.956+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1a40112d40 con 0x7f1a40108a80 2024-05-20T20:14:27.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.957+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1a40112bc0 con 0x7f1a40103430 2024-05-20T20:14:27.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.957+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1a40112ec0 con 0x7f1a40102a70 2024-05-20T20:14:27.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.958+0000 7f1a3e800640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a40108a80 0x7f1a4010ea60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:27.959 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.959+0000 7f1a3de00640 1 -- 172.21.15.28:0/457988684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 msgr2=0x7f1a401a01e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:27.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.960+0000 7f1a3de00640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a401a01e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:27.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.960+0000 7f1a3de00640 1 -- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a40108a80 msgr2=0x7f1a4010ea60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:27.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.960+0000 7f1a3de00640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a40108a80 0x7f1a4010ea60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:27.962 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.960+0000 7f1a3de00640 1 -- 172.21.15.28:0/457988684 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1a4010f230 con 0x7f1a40102a70 2024-05-20T20:14:27.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.961+0000 7f1a3d400640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a401a01e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:27.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.963+0000 7f1a3e800640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a40108a80 0x7f1a4010ea60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:14:27.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.963+0000 7f1a3de00640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a40102a70 0x7f1a4019fca0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f1a3000e790 tx=0x7f1a3000ec50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:27.963 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.964+0000 7f1a36a00640 1 -- 172.21.15.28:0/457988684 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a3000eea0 con 0x7f1a40102a70 2024-05-20T20:14:27.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.964+0000 7f1a36a00640 1 -- 172.21.15.28:0/457988684 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1a3000ccc0 con 0x7f1a40102a70 2024-05-20T20:14:27.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.964+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1a4010f520 con 0x7f1a40102a70 2024-05-20T20:14:27.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.965+0000 7f1a36a00640 1 -- 172.21.15.28:0/457988684 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a30018680 con 0x7f1a40102a70 2024-05-20T20:14:27.964 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.965+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1a4019b7d0 con 0x7f1a40102a70 2024-05-20T20:14:27.978 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.976+0000 7f1a47e58740 1 Processor -- start 2024-05-20T20:14:27.979 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.979+0000 7f1a47e58740 1 -- start start 2024-05-20T20:14:27.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.980+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51ad070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:27.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.985+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:27.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.987+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51ba260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:27.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.987+0000 7f1a47e58740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562be511db90 con 0x562be51acc90 2024-05-20T20:14:27.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.987+0000 7f1a47e58740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562be510d730 con 0x562be51b7e70 2024-05-20T20:14:27.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.987+0000 7f1a47e58740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562be5111770 con 0x562be51ad5b0 2024-05-20T20:14:27.989 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.989+0000 7f1a36a00640 1 -- 172.21.15.28:0/457988684 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1a30018840 con 0x7f1a40102a70 2024-05-20T20:14:27.992 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.991+0000 7f1a3d400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51ba260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:27.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.993+0000 7f1a3de00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:27.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.993+0000 7f1a3e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51ad070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:27.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.993+0000 7f1a3e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51ad070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60570/0 (socket says 172.21.15.28:60570) 2024-05-20T20:14:27.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.993+0000 7f1a3e800640 1 -- 172.21.15.28:0/2182746784 learned_addr learned my addr 172.21.15.28:0/2182746784 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:28.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.995+0000 7f1a3e800640 1 -- 172.21.15.28:0/2182746784 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 msgr2=0x562be51b7930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:28.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.995+0000 7f1a3e800640 1 --2- 172.21.15.28:0/2182746784 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.995+0000 7f1a3e800640 1 -- 172.21.15.28:0/2182746784 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 msgr2=0x562be51ba260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:28.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.995+0000 7f1a3e800640 1 --2- 172.21.15.28:0/2182746784 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51ba260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.002 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.995+0000 7f1a3e800640 1 -- 172.21.15.28:0/2182746784 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562be50a68b0 con 0x562be51acc90 2024-05-20T20:14:28.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:27.997+0000 7f1a36a00640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1a240b2ba0 0x7f1a240b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.003 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.003+0000 7f1a36a00640 1 -- 172.21.15.28:0/457988684 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1a300e2700 con 0x7f1a40102a70 2024-05-20T20:14:28.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.005+0000 7f1a3de00640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1a240b2ba0 0x7f1a240b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.006+0000 7f1a3e800640 1 --2- 172.21.15.28:0/2182746784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51ad070 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7f1a38009be0 tx=0x7f1a380305c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=72f6cd8a0686c60f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:28.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.007+0000 7f1a3de00640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1a240b2ba0 0x7f1a240b5060 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f1a300dfef0 tx=0x7f1a300e8550 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:28.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.007+0000 7f1a2b400640 1 -- 172.21.15.28:0/2182746784 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a38040030 con 0x562be51acc90 2024-05-20T20:14:28.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.008+0000 7f1a2b400640 1 -- 172.21.15.28:0/2182746784 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1a38003e20 con 0x562be51acc90 2024-05-20T20:14:28.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.011+0000 7f1a47e58740 1 -- 172.21.15.28:0/2182746784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 msgr2=0x562be51ad070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:28.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.011+0000 7f1a47e58740 1 --2- 172.21.15.28:0/2182746784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51ad070 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7f1a38009be0 tx=0x7f1a380305c0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.013+0000 7f1a47e58740 1 -- 172.21.15.28:0/2182746784 shutdown_connections 2024-05-20T20:14:28.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.013+0000 7f1a47e58740 1 --2- 172.21.15.28:0/2182746784 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51ba260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.013+0000 7f1a47e58740 1 --2- 172.21.15.28:0/2182746784 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.013+0000 7f1a47e58740 1 --2- 172.21.15.28:0/2182746784 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51ad070 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.013+0000 7f1a47e58740 1 -- 172.21.15.28:0/2182746784 >> 172.21.15.28:0/2182746784 conn(0x562be51a91a0 msgr2=0x562be51a9580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:28.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.014+0000 7f1a47e58740 1 -- 172.21.15.28:0/2182746784 shutdown_connections 2024-05-20T20:14:28.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.014+0000 7f1a47e58740 1 -- 172.21.15.28:0/2182746784 wait complete. 2024-05-20T20:14:28.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.015+0000 7f1a47e58740 1 -- start start 2024-05-20T20:14:28.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.015+0000 7f1a47e58740 1 Processor -- start 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.017+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51b7730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.018+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b4a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.018+0000 7f1a3e800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51b7730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.021+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.021+0000 7f1a3d400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b4a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.021+0000 7f1a3d400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b4a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33392/0 (socket says 172.21.15.28:33392) 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.021+0000 7f1a3d400640 1 -- 172.21.15.28:0/1418225626 learned_addr learned my addr 172.21.15.28:0/1418225626 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.021+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562be4df3790 con 0x562be51acc90 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.023+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562be50b5bc0 con 0x562be51b7e70 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.023+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562be51baa40 con 0x562be51ad5b0 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.023+0000 7f1a47e58740 10 client.0 _ll_register_callbacks cb 0x562be5093400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:28.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.027+0000 7f1a3d400640 1 -- 172.21.15.28:0/1418225626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 msgr2=0x562be51b4f60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.028+0000 7f1a3d400640 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51b4f60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.028+0000 7f1a3d400640 1 -- 172.21.15.28:0/1418225626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 msgr2=0x562be51b7730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.028+0000 7f1a3d400640 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51b7730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.028+0000 7f1a3d400640 1 -- 172.21.15.28:0/1418225626 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562be4e22210 con 0x562be51ad5b0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.028+0000 7f1a3d400640 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b4a20 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f1a2c00e720 tx=0x7f1a2c00ebe0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1dbf871d9b81c8d7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.028+0000 7f1a1fe00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.028+0000 7f1a1fe00640 1 -- 172.21.15.28:0/1418225626 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a2c009a80 con 0x562be51ad5b0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.029+0000 7f1a1fe00640 1 -- 172.21.15.28:0/1418225626 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1a2c009c20 con 0x562be51ad5b0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.029+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a38003e20 con 0x562be51ad5b0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.029+0000 7f1a1fe00640 1 -- 172.21.15.28:0/1418225626 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1a2c009e20 con 0x562be51ad5b0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.029+0000 7f1a47e58740 10 client.27202 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.029+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562be4eee620 con 0x562be51ad5b0 2024-05-20T20:14:28.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.029+0000 7f1a1fe00640 1 -- 172.21.15.28:0/1418225626 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1a38003e20 con 0x562be51ad5b0 2024-05-20T20:14:28.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.031+0000 7f1a1fe00640 1 -- 172.21.15.28:0/1418225626 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1a2c017440 con 0x562be51ad5b0 2024-05-20T20:14:28.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.031+0000 7f1a47e58740 10 client.27202 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:28.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.031+0000 7f1a47e58740 10 client.27202 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:28.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.031+0000 7f1a47e58740 20 client.27202 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:28.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.031+0000 7f1a47e58740 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be51c76c0 0x562be51c9b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.031+0000 7f1a47e58740 4 client.27202 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:28.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.031+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562be511fae0 con 0x562be51c76c0 2024-05-20T20:14:28.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.033+0000 7f1a3e800640 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be51c76c0 0x562be51c9b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.035+0000 7f1a1fe00640 10 client.27202 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:28.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.035+0000 7f1a3e800640 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be51c76c0 0x562be51c9b80 crc :-1 s=READY pgs=197 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7b9c345729f1aed5 server_cookie=80dd4b55fd871896 in_seq=0 out_seq=0 2024-05-20T20:14:28.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.036+0000 7f1a1fe00640 1 -- 172.21.15.28:0/1418225626 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562be511fae0 con 0x562be51c76c0 2024-05-20T20:14:28.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.036+0000 7f1a1fe00640 10 client.27202 handle_command_reply: tid=0 2024-05-20T20:14:28.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.036+0000 7f1a47e58740 1 client.27202 shutdown 2024-05-20T20:14:28.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.038+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 msgr2=0x562be51b4a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:28.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.038+0000 7f1a47e58740 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b4a20 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f1a2c00e720 tx=0x7f1a2c00ebe0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 shutdown_connections 2024-05-20T20:14:28.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a47e58740 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be51c76c0 0x562be51c9b80 crc :-1 s=READY pgs=197 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a47e58740 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51b4f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a47e58740 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b4a20 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a47e58740 1 --2- 172.21.15.28:0/1418225626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51b7730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 >> 172.21.15.28:0/1418225626 conn(0x562be51a91a0 msgr2=0x562be51aeed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:28.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a1fe00640 0 client.27202 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:28.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.040+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 shutdown_connections 2024-05-20T20:14:28.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.041+0000 7f1a47e58740 1 -- 172.21.15.28:0/1418225626 wait complete. 2024-05-20T20:14:28.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.042+0000 7f1a47e58740 20 client.27202 trim_cache size 0 max 16384 2024-05-20T20:14:28.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.107+0000 7f1a47e58740 1 Processor -- start 2024-05-20T20:14:28.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.109+0000 7f1a47e58740 1 -- start start 2024-05-20T20:14:28.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.109+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51affe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.109+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.109+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51c3b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.109+0000 7f1a47e58740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1a2000e940 con 0x562be51acc90 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.109+0000 7f1a47e58740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1a20010480 con 0x562be51b7e70 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.109+0000 7f1a47e58740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1a2000f010 con 0x562be51ad5b0 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.110+0000 7f1a3e800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51c3b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.110+0000 7f1a3e800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51c3b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60712/0 (socket says 172.21.15.28:60712) 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.110+0000 7f1a3e800640 1 -- 172.21.15.28:0/713872942 learned_addr learned my addr 172.21.15.28:0/713872942 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:28.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.113+0000 7f1a3d400640 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.114+0000 7f1a3d400640 1 -- 172.21.15.28:0/713872942 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 msgr2=0x562be51c3b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:28.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.114+0000 7f1a3d400640 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51c3b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.114+0000 7f1a3d400640 1 -- 172.21.15.28:0/713872942 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 msgr2=0x562be51affe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:28.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.114+0000 7f1a3e800640 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51affe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.115+0000 7f1a3d400640 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51affe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.115+0000 7f1a3d400640 1 -- 172.21.15.28:0/713872942 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562be51a3350 con 0x562be51ad5b0 2024-05-20T20:14:28.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.115+0000 7f1a3d400640 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7260 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f1a2c017800 tx=0x7f1a2c011070 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1dec0634ed674aaa server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:28.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.116+0000 7f1a2aa00640 1 -- 172.21.15.28:0/713872942 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a2c009e20 con 0x562be51ad5b0 2024-05-20T20:14:28.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.116+0000 7f1a2aa00640 1 -- 172.21.15.28:0/713872942 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1a2c017e50 con 0x562be51ad5b0 2024-05-20T20:14:28.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.116+0000 7f1a2aa00640 1 -- 172.21.15.28:0/713872942 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a2c0218d0 con 0x562be51ad5b0 2024-05-20T20:14:28.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.118+0000 7f1a47e58740 1 -- 172.21.15.28:0/713872942 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 msgr2=0x562be51b7260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:28.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.118+0000 7f1a47e58740 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7260 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f1a2c017800 tx=0x7f1a2c011070 comp rx=0 tx=0).stop 2024-05-20T20:14:28.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.120+0000 7f1a47e58740 1 -- 172.21.15.28:0/713872942 shutdown_connections 2024-05-20T20:14:28.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.120+0000 7f1a47e58740 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51b7e70 0x562be51c3b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.120+0000 7f1a47e58740 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51ad5b0 0x562be51b7260 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.120+0000 7f1a47e58740 1 --2- 172.21.15.28:0/713872942 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51acc90 0x562be51affe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.120+0000 7f1a47e58740 1 -- 172.21.15.28:0/713872942 >> 172.21.15.28:0/713872942 conn(0x562be51a91a0 msgr2=0x562be51af8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:28.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.120+0000 7f1a47e58740 1 -- 172.21.15.28:0/713872942 shutdown_connections 2024-05-20T20:14:28.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.123+0000 7f1a47e58740 1 -- 172.21.15.28:0/713872942 wait complete. 2024-05-20T20:14:28.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.123+0000 7f1a47e58740 1 -- start start 2024-05-20T20:14:28.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.124+0000 7f1a47e58740 1 Processor -- start 2024-05-20T20:14:28.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.127+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51acc90 0x562be51c01d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.127+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51ad5b0 0x562be51bb540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.127+0000 7f1a47e58740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51b7e70 0x562be51bba80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.127+0000 7f1a47e58740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1a2000fdb0 con 0x562be51ad5b0 2024-05-20T20:14:28.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.127+0000 7f1a47e58740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1a2000f6e0 con 0x562be51acc90 2024-05-20T20:14:28.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.127+0000 7f1a47e58740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1a2000c900 con 0x562be51b7e70 2024-05-20T20:14:28.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.127+0000 7f1a47e58740 10 client.0 _ll_register_callbacks cb 0x7f1a2c00c3b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:28.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.129+0000 7f1a3e800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51acc90 0x562be51c01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.129+0000 7f1a3e800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51acc90 0x562be51c01d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60714/0 (socket says 172.21.15.28:60714) 2024-05-20T20:14:28.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.129+0000 7f1a3e800640 1 -- 172.21.15.28:0/3319533773 learned_addr learned my addr 172.21.15.28:0/3319533773 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:28.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.130+0000 7f1a3e800640 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51b7e70 0x562be51bba80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.135+0000 7f1a3e800640 1 -- 172.21.15.28:0/3319533773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51b7e70 msgr2=0x562be51bba80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:28.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.135+0000 7f1a3e800640 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51b7e70 0x562be51bba80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.135+0000 7f1a3e800640 1 -- 172.21.15.28:0/3319533773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51ad5b0 msgr2=0x562be51bb540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:28.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.135+0000 7f1a3e800640 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51ad5b0 0x562be51bb540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.135+0000 7f1a3e800640 1 -- 172.21.15.28:0/3319533773 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562be51bc040 con 0x562be51acc90 2024-05-20T20:14:28.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.136+0000 7f1a3e800640 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51acc90 0x562be51c01d0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f1a3802ac30 tx=0x7f1a3802a6c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=23c4e32897280b32 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:28.136 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.136+0000 7f1a1fe00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:28.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.137+0000 7f1a1fe00640 1 -- 172.21.15.28:0/3319533773 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a38040030 con 0x562be51acc90 2024-05-20T20:14:28.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.140+0000 7f1a1fe00640 1 -- 172.21.15.28:0/3319533773 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1a38003970 con 0x562be51acc90 2024-05-20T20:14:28.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.140+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1a2c017e50 con 0x562be51acc90 2024-05-20T20:14:28.140 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.141+0000 7f1a1fe00640 1 -- 172.21.15.28:0/3319533773 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1a38036040 con 0x562be51acc90 2024-05-20T20:14:28.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.141+0000 7f1a47e58740 10 client.27299 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:28.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.141+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562be52320e0 con 0x562be51acc90 2024-05-20T20:14:28.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.144+0000 7f1a1fe00640 1 -- 172.21.15.28:0/3319533773 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1a2c017e50 con 0x562be51acc90 2024-05-20T20:14:28.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.144+0000 7f1a1fe00640 1 -- 172.21.15.28:0/3319533773 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1a38063410 con 0x562be51acc90 2024-05-20T20:14:28.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.144+0000 7f1a47e58740 10 client.27299 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:28.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.144+0000 7f1a47e58740 10 client.27299 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:28.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.144+0000 7f1a47e58740 20 client.27299 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:28.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.144+0000 7f1a47e58740 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be52326c0 0x562be5234ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:28.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.145+0000 7f1a47e58740 4 client.27299 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:14:28.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.145+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x562be51b52f0 con 0x562be52326c0 2024-05-20T20:14:28.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.157+0000 7f1a3d400640 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be52326c0 0x562be5234ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:28.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.172+0000 7f1a3d400640 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be52326c0 0x562be5234ae0 crc :-1 s=READY pgs=198 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c817a9d3f2c1f104 server_cookie=788e63c33637a4eb in_seq=0 out_seq=0 2024-05-20T20:14:28.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.183+0000 7f1a1fe00640 10 client.27299 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:28.639 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:28.698 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.698+0000 7f1a1fe00640 1 -- 172.21.15.28:0/3319533773 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x562be51b52f0 con 0x562be52326c0 2024-05-20T20:14:28.698 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.698+0000 7f1a1fe00640 10 client.27299 handle_command_reply: tid=0 2024-05-20T20:14:28.698 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.699+0000 7f1a47e58740 1 client.27299 shutdown 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.699+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51acc90 msgr2=0x562be51c01d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.699+0000 7f1a47e58740 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51acc90 0x562be51c01d0 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f1a3802ac30 tx=0x7f1a3802a6c0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.699+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 shutdown_connections 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.699+0000 7f1a47e58740 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be52326c0 0x562be5234ae0 crc :-1 s=READY pgs=198 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.700+0000 7f1a47e58740 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be51b7e70 0x562be51bba80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.700+0000 7f1a47e58740 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be51ad5b0 0x562be51bb540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.700+0000 7f1a47e58740 1 --2- 172.21.15.28:0/3319533773 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be51acc90 0x562be51c01d0 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.700+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 >> 172.21.15.28:0/3319533773 conn(0x562be51a91a0 msgr2=0x562be51a7a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:28.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.700+0000 7f1a1fe00640 0 client.27299 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:28.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.701+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 shutdown_connections 2024-05-20T20:14:28.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.701+0000 7f1a47e58740 1 -- 172.21.15.28:0/3319533773 wait complete. 2024-05-20T20:14:28.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.701+0000 7f1a47e58740 20 client.27299 trim_cache size 0 max 16384 2024-05-20T20:14:28.701 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:14:28.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:14:28.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:14:28.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.702+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1a240b2ba0 msgr2=0x7f1a240b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:28.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.702+0000 7f1a45000640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1a240b2ba0 0x7f1a240b5060 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f1a300dfef0 tx=0x7f1a300e8550 comp rx=0 tx=0).stop 2024-05-20T20:14:28.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.702+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a40102a70 msgr2=0x7f1a4019fca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:28.702 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.702+0000 7f1a45000640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a40102a70 0x7f1a4019fca0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f1a3000e790 tx=0x7f1a3000ec50 comp rx=0 tx=0).stop 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 shutdown_connections 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1a40103430 0x7f1a401a01e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1a240b2ba0 0x7f1a240b5060 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1a40108a80 0x7f1a4010ea60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 --2- 172.21.15.28:0/457988684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1a40102a70 0x7f1a4019fca0 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 >> 172.21.15.28:0/457988684 conn(0x7f1a400fe500 msgr2=0x7f1a40074c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 shutdown_connections 2024-05-20T20:14:28.703 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:28.703+0000 7f1a45000640 1 -- 172.21.15.28:0/457988684 wait complete. 2024-05-20T20:14:29.459 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:29 smithi028 ceph-mon[31178]: pgmap v274: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 308 MiB/s wr, 1.58k op/s 2024-05-20T20:14:29.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:29 smithi154 ceph-mon[36314]: pgmap v274: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 308 MiB/s wr, 1.58k op/s 2024-05-20T20:14:29.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:29 smithi033 ceph-mon[36179]: pgmap v274: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 308 MiB/s wr, 1.58k op/s 2024-05-20T20:14:31.593 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:31 smithi028 ceph-mon[31178]: pgmap v275: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 389 MiB/s wr, 2.21k op/s 2024-05-20T20:14:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:31 smithi154 ceph-mon[36314]: pgmap v275: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 389 MiB/s wr, 2.21k op/s 2024-05-20T20:14:31.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:31 smithi033 ceph-mon[36179]: pgmap v275: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 389 MiB/s wr, 2.21k op/s 2024-05-20T20:14:32.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.051+0000 7f1af8c00700 1 -- 172.21.15.28:0/397271997 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 msgr2=0x7f1af406bae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.051+0000 7f1af8c00700 1 --2- 172.21.15.28:0/397271997 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406bae0 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4009990 tx=0x7f1ae4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.057+0000 7f1af8c00700 1 -- 172.21.15.28:0/397271997 shutdown_connections 2024-05-20T20:14:32.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.057+0000 7f1af8c00700 1 --2- 172.21.15.28:0/397271997 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1af4114290 0x7f1af4116680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.057+0000 7f1af8c00700 1 --2- 172.21.15.28:0/397271997 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406bae0 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.057+0000 7f1af8c00700 1 --2- 172.21.15.28:0/397271997 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1af406c0b0 0x7f1af406c510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.057+0000 7f1af8c00700 1 -- 172.21.15.28:0/397271997 >> 172.21.15.28:0/397271997 conn(0x7f1af40793b0 msgr2=0x7f1af40797c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:32.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.060+0000 7f1af8c00700 1 -- 172.21.15.28:0/397271997 shutdown_connections 2024-05-20T20:14:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.060+0000 7f1af8c00700 1 -- 172.21.15.28:0/397271997 wait complete. 2024-05-20T20:14:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.061+0000 7f1af8c00700 1 Processor -- start 2024-05-20T20:14:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.063+0000 7f1af8c00700 1 -- start start 2024-05-20T20:14:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.063+0000 7f1af8c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1af406c0b0 0x7f1af4074d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.063+0000 7f1af8c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406ddf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.063+0000 7f1af8c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1af4114290 0x7f1af406e330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.065+0000 7f1af1400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406ddf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.065+0000 7f1af1400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406ddf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60742/0 (socket says 172.21.15.28:60742) 2024-05-20T20:14:32.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.065+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1af406c0b0 0x7f1af4074d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.065+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1af406c0b0 0x7f1af4074d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33424/0 (socket says 172.21.15.28:33424) 2024-05-20T20:14:32.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.068+0000 7f1af1400700 1 -- 172.21.15.28:0/1580968988 learned_addr learned my addr 172.21.15.28:0/1580968988 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.068+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1af4119d70 con 0x7f1af4114290 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.068+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1af4119bf0 con 0x7f1af4104420 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.068+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1af4119ef0 con 0x7f1af406c0b0 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.070+0000 7f1af1400700 1 -- 172.21.15.28:0/1580968988 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1af406c0b0 msgr2=0x7f1af4074d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.070+0000 7f1af1400700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1af406c0b0 0x7f1af4074d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.070+0000 7f1af1400700 1 -- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1af4114290 msgr2=0x7f1af406e330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.070+0000 7f1af1400700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1af4114290 0x7f1af406e330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.070+0000 7f1af1400700 1 -- 172.21.15.28:0/1580968988 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1af406e970 con 0x7f1af4104420 2024-05-20T20:14:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.071+0000 7f1af1400700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406ddf0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f1ae000d980 tx=0x7f1ae000dd40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:32.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.072+0000 7f1aeaa00700 1 -- 172.21.15.28:0/1580968988 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae0016070 con 0x7f1af4104420 2024-05-20T20:14:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.072+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1af4072890 con 0x7f1af4104420 2024-05-20T20:14:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.072+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1af4072e20 con 0x7f1af4104420 2024-05-20T20:14:32.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.079+0000 7f1aeaa00700 1 -- 172.21.15.28:0/1580968988 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1ae0010460 con 0x7f1af4104420 2024-05-20T20:14:32.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.079+0000 7f1aeaa00700 1 -- 172.21.15.28:0/1580968988 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae0016070 con 0x7f1af4104420 2024-05-20T20:14:32.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.090+0000 7f1aeaa00700 1 -- 172.21.15.28:0/1580968988 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1ae0019070 con 0x7f1af4104420 2024-05-20T20:14:32.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.095+0000 7f1b034dfb80 1 Processor -- start 2024-05-20T20:14:32.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.096+0000 7f1b034dfb80 1 -- start start 2024-05-20T20:14:32.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.097+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634feb0 0x563566350290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.101+0000 7f1af2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634feb0 0x563566350290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.101+0000 7f1af2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634feb0 0x563566350290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60750/0 (socket says 172.21.15.28:60750) 2024-05-20T20:14:32.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.101+0000 7f1af2800700 1 -- 172.21.15.28:0/2840194374 learned_addr learned my addr 172.21.15.28:0/2840194374 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:32.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.101+0000 7f1aeaa00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1ad80b2cf0 0x7f1ad80b51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.101+0000 7f1aeaa00700 1 -- 172.21.15.28:0/1580968988 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1ae00e1d50 con 0x7f1af4104420 2024-05-20T20:14:32.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.102+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x56356634a260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.102+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634a880 0x56356634ace0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.102+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5635662a8e40 con 0x56356634a880 2024-05-20T20:14:32.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.102+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5635662989e0 con 0x56356634feb0 2024-05-20T20:14:32.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.102+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56356629ca20 con 0x5635663507d0 2024-05-20T20:14:32.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.104+0000 7f1af1400700 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x56356634a260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.106+0000 7f1af1400700 1 -- 172.21.15.28:0/2840194374 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634feb0 msgr2=0x563566350290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:32.105 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.106+0000 7f1af1400700 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634feb0 0x563566350290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.106+0000 7f1af1400700 1 -- 172.21.15.28:0/2840194374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634a880 msgr2=0x56356634ace0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:32.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.106+0000 7f1af1400700 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634a880 0x56356634ace0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.106+0000 7f1af1400700 1 -- 172.21.15.28:0/2840194374 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563566261420 con 0x5635663507d0 2024-05-20T20:14:32.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.107+0000 7f1af1400700 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x56356634a260 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00df430 tx=0x7f1ae00a8bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f1ba3e422db58f2a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:32.107 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.107+0000 7f1adf400700 1 -- 172.21.15.28:0/2840194374 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae0020040 con 0x5635663507d0 2024-05-20T20:14:32.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.107+0000 7f1adf400700 1 -- 172.21.15.28:0/2840194374 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1ae00ad370 con 0x5635663507d0 2024-05-20T20:14:32.108 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.107+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1ad80b2cf0 0x7f1ad80b51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.112+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 msgr2=0x56356634a260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.112 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.112+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1ad80b2cf0 0x7f1ad80b51b0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f1ae400b5c0 tx=0x7f1ae401a040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:32.113 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.112+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x56356634a260 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f1ae00df430 tx=0x7f1ae00a8bc0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.116+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 shutdown_connections 2024-05-20T20:14:32.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.116+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634a880 0x56356634ace0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.116+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634feb0 0x563566350290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.116+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/2840194374 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x56356634a260 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.116 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.116+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 >> 172.21.15.28:0/2840194374 conn(0x563566278be0 msgr2=0x563566345010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:32.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.118+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 shutdown_connections 2024-05-20T20:14:32.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.118+0000 7f1b034dfb80 1 -- 172.21.15.28:0/2840194374 wait complete. 2024-05-20T20:14:32.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.120+0000 7f1b034dfb80 1 -- start start 2024-05-20T20:14:32.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.121+0000 7f1b034dfb80 1 Processor -- start 2024-05-20T20:14:32.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.123+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x563566356330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.123 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.124+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634feb0 0x563566356870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.124+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635663507d0 0x5635663538d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.124+0000 7f1b034dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56356619efa0 con 0x5635663507d0 2024-05-20T20:14:32.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.124+0000 7f1b034dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563566239760 con 0x56356634a880 2024-05-20T20:14:32.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.124+0000 7f1b034dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5635663572e0 con 0x56356634feb0 2024-05-20T20:14:32.124 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.124+0000 7f1b034dfb80 10 client.0 _ll_register_callbacks cb 0x563566242c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:32.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.125+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x563566356330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.125+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x563566356330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60764/0 (socket says 172.21.15.28:60764) 2024-05-20T20:14:32.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.125+0000 7f1af1e00700 1 -- 172.21.15.28:0/1136359811 learned_addr learned my addr 172.21.15.28:0/1136359811 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:32.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.126+0000 7f1af1400700 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634feb0 0x563566356870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.126+0000 7f1af1e00700 1 -- 172.21.15.28:0/1136359811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634feb0 msgr2=0x563566356870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:32.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.126+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634feb0 0x563566356870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.126+0000 7f1af1e00700 1 -- 172.21.15.28:0/1136359811 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635663507d0 msgr2=0x5635663538d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:32.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.126+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635663507d0 0x5635663538d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.126+0000 7f1af1e00700 1 -- 172.21.15.28:0/1136359811 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5635661e88d0 con 0x56356634a880 2024-05-20T20:14:32.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.128+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x563566356330 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f1ae400bf80 tx=0x7f1ae4009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6ff036b947727930 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:32.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.128+0000 7f1ad3e00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:32.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.128+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1136359811 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae4023610 con 0x56356634a880 2024-05-20T20:14:32.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.129+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1136359811 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1ae40237b0 con 0x56356634a880 2024-05-20T20:14:32.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.129+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1ae00ad370 con 0x56356634a880 2024-05-20T20:14:32.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.129+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1136359811 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1ae40239b0 con 0x56356634a880 2024-05-20T20:14:32.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.129+0000 7f1b034dfb80 10 client.27311 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:32.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.129+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563566269060 con 0x56356634a880 2024-05-20T20:14:32.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.131+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1136359811 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1ae00ad370 con 0x56356634a880 2024-05-20T20:14:32.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.131+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1136359811 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1ae402dbf0 con 0x56356634a880 2024-05-20T20:14:32.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.132+0000 7f1b034dfb80 10 client.27311 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:32.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.132+0000 7f1b034dfb80 10 client.27311 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:14:32.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.132+0000 7f1b034dfb80 20 client.27311 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:32.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.132+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563566363d90 0x5635663661f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.132+0000 7f1b034dfb80 4 client.27311 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:32.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.132+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5635662aad90 con 0x563566363d90 2024-05-20T20:14:32.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.132+0000 7f1af1400700 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563566363d90 0x5635663661f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.134+0000 7f1af1400700 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563566363d90 0x5635663661f0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=6f243a5eaa58f510 server_cookie=d41a457ec12b7dc4 in_seq=0 out_seq=0 2024-05-20T20:14:32.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.134+0000 7f1ad3e00700 10 client.27311 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:14:32.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1136359811 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5635662aad90 con 0x563566363d90 2024-05-20T20:14:32.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1ad3e00700 10 client.27311 handle_command_reply: tid=0 2024-05-20T20:14:32.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 client.27311 shutdown 2024-05-20T20:14:32.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 msgr2=0x563566356330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x563566356330 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f1ae400bf80 tx=0x7f1ae4009310 comp rx=0 tx=0).stop 2024-05-20T20:14:32.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 shutdown_connections 2024-05-20T20:14:32.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563566363d90 0x5635663661f0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5635663507d0 0x5635663538d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x563566356330 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1136359811 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634feb0 0x563566356870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.135+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 >> 172.21.15.28:0/1136359811 conn(0x563566278be0 msgr2=0x563566349250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:32.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.136+0000 7f1ad3e00700 0 client.27311 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:14:32.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.137+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 shutdown_connections 2024-05-20T20:14:32.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.137+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1136359811 wait complete. 2024-05-20T20:14:32.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.137+0000 7f1b034dfb80 20 client.27311 trim_cache size 0 max 16384 2024-05-20T20:14:32.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.192+0000 7f1b034dfb80 1 Processor -- start 2024-05-20T20:14:32.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1b034dfb80 1 -- start start 2024-05-20T20:14:32.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x56356634a260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 0x56356634ac60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x563566355b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1b034dfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad400d6b0 con 0x56356634feb0 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1b034dfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad400f1f0 con 0x56356634a880 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1b034dfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad400dd80 con 0x5635663507d0 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.201+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x56356634a260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.202+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x56356634a260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60766/0 (socket says 172.21.15.28:60766) 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.202+0000 7f1af1e00700 1 -- 172.21.15.28:0/1784354497 learned_addr learned my addr 172.21.15.28:0/1784354497 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.202+0000 7f1af1e00700 1 -- 172.21.15.28:0/1784354497 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 msgr2=0x563566355b40 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.203+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1784354497 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x563566355b40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.203+0000 7f1af1e00700 1 -- 172.21.15.28:0/1784354497 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 msgr2=0x56356634ac60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.203+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1784354497 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 0x56356634ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.203+0000 7f1af1e00700 1 -- 172.21.15.28:0/1784354497 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563566357120 con 0x56356634a880 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.203+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1784354497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x56356634a260 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4000c00 tx=0x7f1ae4009510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ec6c9bc66b1f5f87 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:32.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.206+0000 7f1adea00700 1 -- 172.21.15.28:0/1784354497 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae4023d50 con 0x56356634a880 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.207+0000 7f1adea00700 1 -- 172.21.15.28:0/1784354497 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1ae4032070 con 0x56356634a880 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.207+0000 7f1adea00700 1 -- 172.21.15.28:0/1784354497 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1ae402d3f0 con 0x56356634a880 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1784354497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 msgr2=0x56356634a260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1784354497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x56356634a260 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4000c00 tx=0x7f1ae4009510 comp rx=0 tx=0).stop 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1784354497 shutdown_connections 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1784354497 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5635663507d0 0x563566355b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1784354497 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56356634a880 0x56356634a260 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1784354497 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 0x56356634ac60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1784354497 >> 172.21.15.28:0/1784354497 conn(0x563566278be0 msgr2=0x56356633ee50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1784354497 shutdown_connections 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.208+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1784354497 wait complete. 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.209+0000 7f1b034dfb80 1 -- start start 2024-05-20T20:14:32.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.209+0000 7f1b034dfb80 1 Processor -- start 2024-05-20T20:14:32.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.210+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634a880 0x56356635c640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.210+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 0x56356635cb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.210+0000 7f1b034dfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635663507d0 0x56356636abf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.214+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634a880 0x56356635c640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.214+0000 7f1af2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635663507d0 0x56356636abf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.215+0000 7f1af2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635663507d0 0x56356636abf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60772/0 (socket says 172.21.15.28:60772) 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.215+0000 7f1af1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634a880 0x56356635c640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33470/0 (socket says 172.21.15.28:33470) 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.215+0000 7f1af2800700 1 -- 172.21.15.28:0/1595263438 learned_addr learned my addr 172.21.15.28:0/1595263438 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.215+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad400eb20 con 0x56356634feb0 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.215+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad400e450 con 0x5635663507d0 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.215+0000 7f1af1400700 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 0x56356635cb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.215+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1ad400b670 con 0x56356634a880 2024-05-20T20:14:32.219 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.219+0000 7f1af2800700 1 -- 172.21.15.28:0/1595263438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634a880 msgr2=0x56356635c640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:32.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.219+0000 7f1b034dfb80 10 client.0 _ll_register_callbacks cb 0x563566242c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:32.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.219+0000 7f1af2800700 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634a880 0x56356635c640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.220+0000 7f1af2800700 1 -- 172.21.15.28:0/1595263438 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 msgr2=0x56356635cb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:32.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.220+0000 7f1af2800700 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 0x56356635cb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.220+0000 7f1af2800700 1 -- 172.21.15.28:0/1595263438 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56356636b270 con 0x5635663507d0 2024-05-20T20:14:32.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.220+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1ae4032070 con 0x5635663507d0 2024-05-20T20:14:32.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.221+0000 7f1af2800700 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635663507d0 0x56356636abf0 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f1aec007710 tx=0x7f1aec007a20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b3eba8a8996fbff3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:32.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.221+0000 7f1ad3e00700 10 client.27317 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:32.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.224+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1595263438 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1aec01a060 con 0x5635663507d0 2024-05-20T20:14:32.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.224+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1595263438 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1aec012730 con 0x5635663507d0 2024-05-20T20:14:32.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.224+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1595263438 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1aec008040 con 0x5635663507d0 2024-05-20T20:14:32.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.225+0000 7f1b034dfb80 10 client.27317 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:32.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.225+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x5635663ce270 con 0x5635663507d0 2024-05-20T20:14:32.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.226+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1595263438 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1aec012970 con 0x5635663507d0 2024-05-20T20:14:32.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.226+0000 7f1b034dfb80 10 client.27317 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:32.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.226+0000 7f1b034dfb80 10 client.27317 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:14:32.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.226+0000 7f1b034dfb80 20 client.27317 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:32.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.226+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5635663ce8f0 0x5635663d0d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:32.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.226+0000 7f1b034dfb80 4 client.27317 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:14:32.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.226+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5635662a4570 con 0x5635663ce8f0 2024-05-20T20:14:32.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.227+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5635663ce8f0 0x5635663d0d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:32.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.228+0000 7f1ad3e00700 10 client.27317 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:14:32.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.228+0000 7f1af1e00700 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5635663ce8f0 0x5635663d0d10 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=dc11da9fc00c847a server_cookie=4299ad0063f44ce9 in_seq=0 out_seq=0 2024-05-20T20:14:32.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.228+0000 7f1ad3e00700 1 -- 172.21.15.28:0/1595263438 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5635662a4570 con 0x5635663ce8f0 2024-05-20T20:14:32.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.228+0000 7f1ad3e00700 10 client.27317 handle_command_reply: tid=0 2024-05-20T20:14:32.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.229+0000 7f1b034dfb80 1 client.27317 shutdown 2024-05-20T20:14:32.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.229+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635663507d0 msgr2=0x56356636abf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.229+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635663507d0 0x56356636abf0 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f1aec007710 tx=0x7f1aec007a20 comp rx=0 tx=0).stop 2024-05-20T20:14:32.236 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 shutdown_connections 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5635663ce8f0 0x5635663d0d10 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1ad3e00700 0 client.27317 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5635663507d0 0x56356636abf0 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56356634a880 0x56356635c640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1b034dfb80 1 --2- 172.21.15.28:0/1595263438 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56356634feb0 0x56356635cb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 >> 172.21.15.28:0/1595263438 conn(0x563566278be0 msgr2=0x563566356330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.230+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 shutdown_connections 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.231+0000 7f1b034dfb80 1 -- 172.21.15.28:0/1595263438 wait complete. 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.231+0000 7f1b034dfb80 20 client.27317 trim_cache size 0 max 16384 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.232+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1ad80b2cf0 msgr2=0x7f1ad80b51b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.232+0000 7f1af8c00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1ad80b2cf0 0x7f1ad80b51b0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f1ae400b5c0 tx=0x7f1ae401a040 comp rx=0 tx=0).stop 2024-05-20T20:14:32.237 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.232+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 msgr2=0x7f1af406ddf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.232+0000 7f1af8c00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406ddf0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f1ae000d980 tx=0x7f1ae000dd40 comp rx=0 tx=0).stop 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.233+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 shutdown_connections 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.233+0000 7f1af8c00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1af4114290 0x7f1af406e330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.233+0000 7f1af8c00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1af406c0b0 0x7f1af4074d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.233+0000 7f1af8c00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1ad80b2cf0 0x7f1ad80b51b0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.233+0000 7f1af8c00700 1 --2- 172.21.15.28:0/1580968988 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1af4104420 0x7f1af406ddf0 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.233+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 >> 172.21.15.28:0/1580968988 conn(0x7f1af40793b0 msgr2=0x7f1af4101420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.235+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 shutdown_connections 2024-05-20T20:14:32.238 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:32.236+0000 7f1af8c00700 1 -- 172.21.15.28:0/1580968988 wait complete. 2024-05-20T20:14:33.560 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:33 smithi028 ceph-mon[31178]: pgmap v276: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 309 MiB/s wr, 2.00k op/s 2024-05-20T20:14:33.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:33 smithi154 ceph-mon[36314]: pgmap v276: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 309 MiB/s wr, 2.00k op/s 2024-05-20T20:14:33.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:33 smithi033 ceph-mon[36179]: pgmap v276: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 309 MiB/s wr, 2.00k op/s 2024-05-20T20:14:34.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:34 smithi028 ceph-mon[31178]: pgmap v277: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 404 MiB/s wr, 2.89k op/s 2024-05-20T20:14:34.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:34 smithi154 ceph-mon[36314]: pgmap v277: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 404 MiB/s wr, 2.89k op/s 2024-05-20T20:14:34.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:34 smithi033 ceph-mon[36179]: pgmap v277: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 404 MiB/s wr, 2.89k op/s 2024-05-20T20:14:35.668 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:14:36.478 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:37.295 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:37 smithi028 ceph-mon[31178]: pgmap v278: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 295 MiB/s wr, 2.31k op/s 2024-05-20T20:14:37.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:37 smithi154 ceph-mon[36314]: pgmap v278: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 295 MiB/s wr, 2.31k op/s 2024-05-20T20:14:37.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:37 smithi033 ceph-mon[36179]: pgmap v278: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 295 MiB/s wr, 2.31k op/s 2024-05-20T20:14:39.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:39 smithi154 ceph-mon[36314]: pgmap v279: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 289 MiB/s wr, 2.34k op/s 2024-05-20T20:14:39.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:39 smithi033 ceph-mon[36179]: pgmap v279: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 289 MiB/s wr, 2.34k op/s 2024-05-20T20:14:39.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:39 smithi028 ceph-mon[31178]: pgmap v279: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 289 MiB/s wr, 2.34k op/s 2024-05-20T20:14:41.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:41 smithi154 ceph-mon[36314]: pgmap v280: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 329 MiB/s wr, 2.68k op/s 2024-05-20T20:14:41.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:41 smithi033 ceph-mon[36179]: pgmap v280: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 329 MiB/s wr, 2.68k op/s 2024-05-20T20:14:41.496 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:41 smithi028 ceph-mon[31178]: pgmap v280: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 329 MiB/s wr, 2.68k op/s 2024-05-20T20:14:41.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.792+0000 7fc4fae00700 1 -- 172.21.15.28:0/1311086244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec009a00 msgr2=0x7fc4ec009de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.792+0000 7fc4fae00700 1 --2- 172.21.15.28:0/1311086244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec009a00 0x7fc4ec009de0 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7fc4e8009990 tx=0x7fc4e8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.792+0000 7fc4fae00700 1 -- 172.21.15.28:0/1311086244 shutdown_connections 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.792+0000 7fc4fae00700 1 --2- 172.21.15.28:0/1311086244 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec0a6be0 0x7fc4ec0a8fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.792+0000 7fc4fae00700 1 --2- 172.21.15.28:0/1311086244 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec009a00 0x7fc4ec009de0 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.792+0000 7fc4fae00700 1 --2- 172.21.15.28:0/1311086244 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4ec00a320 0x7fc4ec0a66a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.792+0000 7fc4fae00700 1 -- 172.21.15.28:0/1311086244 >> 172.21.15.28:0/1311086244 conn(0x7fc4ec093a90 msgr2=0x7fc4ec095eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.793+0000 7fc4fae00700 1 -- 172.21.15.28:0/1311086244 shutdown_connections 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.793+0000 7fc4fae00700 1 -- 172.21.15.28:0/1311086244 wait complete. 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.794+0000 7fc4fae00700 1 Processor -- start 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4fae00700 1 -- start start 2024-05-20T20:14:41.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4fae00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec009a00 0x7fc4ec1386c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4fae00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec00a320 0x7fc4ec138c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4fae00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4ec0a6be0 0x7fc4ec132880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4fae00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc4ec0ac540 con 0x7fc4ec00a320 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4fae00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc4ec0ac3c0 con 0x7fc4ec0a6be0 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4fae00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc4ec0ac6c0 con 0x7fc4ec009a00 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4f9a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec009a00 0x7fc4ec1386c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.795+0000 7fc4f9a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec009a00 0x7fc4ec1386c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50590/0 (socket says 172.21.15.28:50590) 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9a00700 1 -- 172.21.15.28:0/923572788 learned_addr learned my addr 172.21.15.28:0/923572788 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9000700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec00a320 0x7fc4ec138c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9a00700 1 -- 172.21.15.28:0/923572788 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4ec0a6be0 msgr2=0x7fc4ec132880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4ec0a6be0 0x7fc4ec132880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9a00700 1 -- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec00a320 msgr2=0x7fc4ec138c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec00a320 0x7fc4ec138c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9a00700 1 -- 172.21.15.28:0/923572788 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc4ec132fe0 con 0x7fc4ec009a00 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.796+0000 7fc4f9000700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec00a320 0x7fc4ec138c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.797+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec009a00 0x7fc4ec1386c0 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7fc4e8005910 tx=0x7fc4e800b670 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.797+0000 7fc4f2000700 1 -- 172.21.15.28:0/923572788 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4e8027070 con 0x7fc4ec009a00 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.797+0000 7fc4f2000700 1 -- 172.21.15.28:0/923572788 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4e8022470 con 0x7fc4ec009a00 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.797+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc4ec133270 con 0x7fc4ec009a00 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.798+0000 7fc4f2000700 1 -- 172.21.15.28:0/923572788 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4e8017650 con 0x7fc4ec009a00 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.798+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc4ec13f590 con 0x7fc4ec009a00 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.804+0000 7fc506df7b80 1 Processor -- start 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc4f2000700 1 -- 172.21.15.28:0/923572788 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc4e802a070 con 0x7fc4ec009a00 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc506df7b80 1 -- start start 2024-05-20T20:14:41.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c1ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611bbd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611bc7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc506df7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562161119e40 con 0x5621611c20e0 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc506df7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5621611099e0 con 0x5621611bc390 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.805+0000 7fc506df7b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56216110da20 con 0x5621611c17c0 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611bc7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611bc7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34142/0 (socket says 172.21.15.28:34142) 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 -- 172.21.15.28:0/1150206775 learned_addr learned my addr 172.21.15.28:0/1150206775 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4fa400700 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c1ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611bbd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 -- 172.21.15.28:0/1150206775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 msgr2=0x5621611c1ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c1ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 -- 172.21.15.28:0/1150206775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 msgr2=0x5621611bbd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611bbd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.806+0000 7fc4f9000700 1 -- 172.21.15.28:0/1150206775 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5621610d2420 con 0x5621611bc390 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.807+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611bbd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.807+0000 7fc4f9000700 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611bc7f0 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7fc4e0010500 tx=0x7fc4e0010810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cd42627c4d98cba server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.807+0000 7fc4e6a00700 1 -- 172.21.15.28:0/1150206775 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4e001c480 con 0x5621611bc390 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.807+0000 7fc4e6a00700 1 -- 172.21.15.28:0/1150206775 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4e001cad0 con 0x5621611bc390 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.807+0000 7fc4e6a00700 1 -- 172.21.15.28:0/1150206775 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4e0011c70 con 0x5621611bc390 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.807+0000 7fc4f2000700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4dc0b2a70 0x7fc4dc0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.807+0000 7fc4f2000700 1 -- 172.21.15.28:0/923572788 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fc4e80e3470 con 0x7fc4ec009a00 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.808+0000 7fc4fa400700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4dc0b2a70 0x7fc4dc0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.808+0000 7fc506df7b80 1 -- 172.21.15.28:0/1150206775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 msgr2=0x5621611bc7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.808+0000 7fc506df7b80 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611bc7f0 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7fc4e0010500 tx=0x7fc4e0010810 comp rx=0 tx=0).stop 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc506df7b80 1 -- 172.21.15.28:0/1150206775 shutdown_connections 2024-05-20T20:14:41.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc506df7b80 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611bc7f0 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc506df7b80 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c1ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc506df7b80 1 --2- 172.21.15.28:0/1150206775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611bbd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc506df7b80 1 -- 172.21.15.28:0/1150206775 >> 172.21.15.28:0/1150206775 conn(0x5621610e9be0 msgr2=0x5621611b6810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc4fa400700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4dc0b2a70 0x7fc4dc0b4f30 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fc4f4052820 tx=0x7fc4f4051a80 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc506df7b80 1 -- 172.21.15.28:0/1150206775 shutdown_connections 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.809+0000 7fc506df7b80 1 -- 172.21.15.28:0/1150206775 wait complete. 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.810+0000 7fc506df7b80 1 -- start start 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.810+0000 7fc506df7b80 1 Processor -- start 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611c7dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c5170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611c56b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc506df7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56216100ffa0 con 0x5621611c20e0 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc506df7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5621610aa760 con 0x5621611bc390 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc506df7b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5621611c8c00 con 0x5621611c17c0 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc506df7b80 10 client.0 _ll_register_callbacks cb 0x5621610b3c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.811+0000 7fc4fa400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611c56b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4f9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4fa400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611c56b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:55096/0 (socket says 172.21.15.28:55096) 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4fa400700 1 -- 172.21.15.28:0/2751255458 learned_addr learned my addr 172.21.15.28:0/2751255458 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611c7dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4f9a00700 1 -- 172.21.15.28:0/2751255458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 msgr2=0x5621611c5170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c5170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4f9a00700 1 -- 172.21.15.28:0/2751255458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 msgr2=0x5621611c56b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611c56b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.812+0000 7fc4f9a00700 1 -- 172.21.15.28:0/2751255458 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5621610598d0 con 0x5621611bc390 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.813+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611c7dd0 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fc4e80e0bb0 tx=0x7fc4e8022c00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5913161703d3899d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.813+0000 7fc4db400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.813+0000 7fc4f9000700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.813+0000 7fc4fa400700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611c56b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.813+0000 7fc4db400700 1 -- 172.21.15.28:0/2751255458 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4e80a6690 con 0x5621611bc390 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.813+0000 7fc4db400700 1 -- 172.21.15.28:0/2751255458 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fc4e8092320 con 0x5621611bc390 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.814+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc4e001cad0 con 0x5621611bc390 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.814+0000 7fc4db400700 1 -- 172.21.15.28:0/2751255458 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc4e80b10e0 con 0x5621611bc390 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.814+0000 7fc506df7b80 10 client.27326 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.814+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5621610da060 con 0x5621611bc390 2024-05-20T20:14:41.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.814+0000 7fc4db400700 1 -- 172.21.15.28:0/2751255458 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4e001cad0 con 0x5621611bc390 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.814+0000 7fc4db400700 1 -- 172.21.15.28:0/2751255458 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc4e80b3110 con 0x5621611bc390 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.815+0000 7fc506df7b80 10 client.27326 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.815+0000 7fc506df7b80 10 client.27326 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.815+0000 7fc506df7b80 20 client.27326 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.815+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5621611d5880 0x5621611d7ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.815+0000 7fc506df7b80 4 client.27326 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.815+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56216111bd90 con 0x5621611d5880 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.815+0000 7fc4f9000700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5621611d5880 0x5621611d7ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.816+0000 7fc4f9000700 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5621611d5880 0x5621611d7ca0 crc :-1 s=READY pgs=199 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=39c0bd25d281dd86 server_cookie=495d6406ff2fd0e0 in_seq=0 out_seq=0 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.816+0000 7fc4db400700 10 client.27326 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.817+0000 7fc4db400700 1 -- 172.21.15.28:0/2751255458 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56216111bd90 con 0x5621611d5880 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.817+0000 7fc4db400700 10 client.27326 handle_command_reply: tid=0 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.818+0000 7fc506df7b80 1 client.27326 shutdown 2024-05-20T20:14:41.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.818+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 msgr2=0x5621611c7dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.818+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611c7dd0 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fc4e80e0bb0 tx=0x7fc4e8022c00 comp rx=0 tx=0).stop 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 shutdown_connections 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5621611d5880 0x5621611d7ca0 crc :-1 s=READY pgs=199 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c20e0 0x5621611c56b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611bc390 0x5621611c7dd0 unknown :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2751255458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611c5170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc4db400700 0 client.27326 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 >> 172.21.15.28:0/2751255458 conn(0x5621610e9be0 msgr2=0x5621611b5fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:41.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 shutdown_connections 2024-05-20T20:14:41.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 1 -- 172.21.15.28:0/2751255458 wait complete. 2024-05-20T20:14:41.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.819+0000 7fc506df7b80 20 client.27326 trim_cache size 0 max 16384 2024-05-20T20:14:41.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.852+0000 7fc506df7b80 1 Processor -- start 2024-05-20T20:14:41.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.854+0000 7fc506df7b80 1 -- start start 2024-05-20T20:14:41.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.854+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611bc390 0x5621611bbd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.854+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c17c0 0x5621611c40f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.854+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611c2bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.854+0000 7fc506df7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc4d400d5b0 con 0x5621611c17c0 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.854+0000 7fc506df7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc4d400f0f0 con 0x5621611c20e0 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.854+0000 7fc506df7b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc4d400dc80 con 0x5621611bc390 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.855+0000 7fc4fa400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611c2bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.855+0000 7fc4fa400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611c2bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34164/0 (socket says 172.21.15.28:34164) 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.855+0000 7fc4fa400700 1 -- 172.21.15.28:0/2682855136 learned_addr learned my addr 172.21.15.28:0/2682855136 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:41.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.855+0000 7fc4f9000700 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c17c0 0x5621611c40f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.856+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611bc390 0x5621611bbd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.856+0000 7fc4fa400700 1 -- 172.21.15.28:0/2682855136 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611bc390 msgr2=0x5621611bbd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.856+0000 7fc4fa400700 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611bc390 0x5621611bbd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.856+0000 7fc4fa400700 1 -- 172.21.15.28:0/2682855136 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c17c0 msgr2=0x5621611c40f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.856+0000 7fc4fa400700 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c17c0 0x5621611c40f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.856+0000 7fc4fa400700 1 -- 172.21.15.28:0/2682855136 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5621611c7a10 con 0x5621611c20e0 2024-05-20T20:14:41.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.856+0000 7fc4f9000700 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c17c0 0x5621611c40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:41.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.857+0000 7fc4fa400700 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611c2bd0 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7fc4f402eb80 tx=0x7fc4f4084000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=45c2e4ec657c3e5f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.857+0000 7fc4e6000700 1 -- 172.21.15.28:0/2682855136 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4f40753c0 con 0x5621611c20e0 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.857+0000 7fc4e6000700 1 -- 172.21.15.28:0/2682855136 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4f4075560 con 0x5621611c20e0 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.857+0000 7fc4e6000700 1 -- 172.21.15.28:0/2682855136 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4f408d660 con 0x5621611c20e0 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 -- 172.21.15.28:0/2682855136 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 msgr2=0x5621611c2bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611c2bd0 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7fc4f402eb80 tx=0x7fc4f4084000 comp rx=0 tx=0).stop 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 -- 172.21.15.28:0/2682855136 shutdown_connections 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611c2bd0 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611bc390 0x5621611bbd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 --2- 172.21.15.28:0/2682855136 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611c17c0 0x5621611c40f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 -- 172.21.15.28:0/2682855136 >> 172.21.15.28:0/2682855136 conn(0x5621610e9be0 msgr2=0x5621611c24c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:41.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.859+0000 7fc506df7b80 1 -- 172.21.15.28:0/2682855136 shutdown_connections 2024-05-20T20:14:41.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.860+0000 7fc506df7b80 1 -- 172.21.15.28:0/2682855136 wait complete. 2024-05-20T20:14:41.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.860+0000 7fc506df7b80 1 -- start start 2024-05-20T20:14:41.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 1 Processor -- start 2024-05-20T20:14:41.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611bc390 0x56216123be20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611cdf20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611ce530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc4d400ea20 con 0x5621611bc390 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc4d400e350 con 0x5621611c20e0 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc4d400b530 con 0x5621611c17c0 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.861+0000 7fc506df7b80 10 client.0 _ll_register_callbacks cb 0x5621610b3c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.862+0000 7fc4fa400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611ce530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.862+0000 7fc4f9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611cdf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.862+0000 7fc4f9000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611cdf20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50628/0 (socket says 172.21.15.28:50628) 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.862+0000 7fc4f9000700 1 -- 172.21.15.28:0/323502161 learned_addr learned my addr 172.21.15.28:0/323502161 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.862+0000 7fc4fa400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611ce530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34166/0 (socket says 172.21.15.28:34166) 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.863+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611bc390 0x56216123be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.863+0000 7fc4f9a00700 1 -- 172.21.15.28:0/323502161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 msgr2=0x5621611cdf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.864+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611cdf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.864+0000 7fc4f9a00700 1 -- 172.21.15.28:0/323502161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 msgr2=0x5621611ce530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.864+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611ce530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.864+0000 7fc4f9a00700 1 -- 172.21.15.28:0/323502161 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5621611dd120 con 0x5621611bc390 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.864+0000 7fc4f9a00700 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611bc390 0x56216123be20 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7fc4e80b1aa0 tx=0x7fc4e80cd1d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ebd17f7adece8e95 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.864+0000 7fc4db400700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.865+0000 7fc4db400700 1 -- 172.21.15.28:0/323502161 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4e80b10e0 con 0x5621611bc390 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.865+0000 7fc4db400700 1 -- 172.21.15.28:0/323502161 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7fc4e80e6a50 con 0x5621611bc390 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.866+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc4f4075560 con 0x5621611bc390 2024-05-20T20:14:41.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.866+0000 7fc4db400700 1 -- 172.21.15.28:0/323502161 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc4e80a9f40 con 0x5621611bc390 2024-05-20T20:14:41.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.867+0000 7fc506df7b80 10 client.16917 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:41.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.867+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5621612517b0 con 0x5621611bc390 2024-05-20T20:14:41.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.869+0000 7fc4db400700 1 -- 172.21.15.28:0/323502161 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4f4075560 con 0x5621611bc390 2024-05-20T20:14:41.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.869+0000 7fc4db400700 1 -- 172.21.15.28:0/323502161 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc4e80b3110 con 0x5621611bc390 2024-05-20T20:14:41.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.870+0000 7fc506df7b80 10 client.16917 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:41.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.870+0000 7fc506df7b80 10 client.16917 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:41.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.870+0000 7fc506df7b80 20 client.16917 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:41.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.870+0000 7fc506df7b80 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562161251da0 0x5621612541c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:41.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.870+0000 7fc506df7b80 4 client.16917 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:14:41.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.870+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x562161115570 con 0x562161251da0 2024-05-20T20:14:41.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.872+0000 7fc4f9000700 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562161251da0 0x5621612541c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:41.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.874+0000 7fc4f9000700 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562161251da0 0x5621612541c0 crc :-1 s=READY pgs=200 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3f363d9607329525 server_cookie=c93e29387b1332bf in_seq=0 out_seq=0 2024-05-20T20:14:41.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.875+0000 7fc4db400700 10 client.16917 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:41.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.876+0000 7fc4db400700 1 -- 172.21.15.28:0/323502161 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x562161115570 con 0x562161251da0 2024-05-20T20:14:41.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.876+0000 7fc4db400700 10 client.16917 handle_command_reply: tid=0 2024-05-20T20:14:41.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.876+0000 7fc506df7b80 1 client.16917 shutdown 2024-05-20T20:14:41.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.877+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611bc390 msgr2=0x56216123be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.877+0000 7fc506df7b80 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611bc390 0x56216123be20 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7fc4e80b1aa0 tx=0x7fc4e80cd1d0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.878+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 shutdown_connections 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.878+0000 7fc506df7b80 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562161251da0 0x5621612541c0 crc :-1 s=READY pgs=200 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.878+0000 7fc506df7b80 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5621611c20e0 0x5621611ce530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.878+0000 7fc506df7b80 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5621611bc390 0x56216123be20 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.878+0000 7fc506df7b80 1 --2- 172.21.15.28:0/323502161 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5621611c17c0 0x5621611cdf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.878+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 >> 172.21.15.28:0/323502161 conn(0x5621610e9be0 msgr2=0x5621611c6770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.879+0000 7fc4db400700 0 client.16917 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.883+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 shutdown_connections 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.883+0000 7fc506df7b80 1 -- 172.21.15.28:0/323502161 wait complete. 2024-05-20T20:14:41.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.883+0000 7fc506df7b80 20 client.16917 trim_cache size 0 max 16384 2024-05-20T20:14:41.899 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:14:41.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4dc0b2a70 msgr2=0x7fc4dc0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4dc0b2a70 0x7fc4dc0b4f30 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fc4f4052820 tx=0x7fc4f4051a80 comp rx=0 tx=0).stop 2024-05-20T20:14:41.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec009a00 msgr2=0x7fc4ec1386c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:41.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec009a00 0x7fc4ec1386c0 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7fc4e8005910 tx=0x7fc4e800b670 comp rx=0 tx=0).stop 2024-05-20T20:14:41.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 shutdown_connections 2024-05-20T20:14:41.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4dc0b2a70 0x7fc4dc0b4f30 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4ec0a6be0 0x7fc4ec132880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4ec009a00 0x7fc4ec1386c0 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 --2- 172.21.15.28:0/923572788 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4ec00a320 0x7fc4ec138c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:41.902 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.899+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 >> 172.21.15.28:0/923572788 conn(0x7fc4ec093a90 msgr2=0x7fc4ec095e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:41.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.901+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 shutdown_connections 2024-05-20T20:14:41.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:41.902+0000 7fc4fae00700 1 -- 172.21.15.28:0/923572788 wait complete. 2024-05-20T20:14:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.060+0000 7f5046e00640 1 -- 172.21.15.28:0/360101561 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5040115ba0 msgr2=0x7f5040117f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.060+0000 7f5046e00640 1 --2- 172.21.15.28:0/360101561 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5040115ba0 0x7f5040117f90 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f5038009c10 tx=0x7f503802a780 comp rx=0 tx=0).stop 2024-05-20T20:14:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.061+0000 7f5046e00640 1 -- 172.21.15.28:0/360101561 shutdown_connections 2024-05-20T20:14:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.061+0000 7f5046e00640 1 --2- 172.21.15.28:0/360101561 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5040115ba0 0x7f5040117f90 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.061+0000 7f5046e00640 1 --2- 172.21.15.28:0/360101561 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f504010a370 0x7f5040115660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.061+0000 7f5046e00640 1 --2- 172.21.15.28:0/360101561 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5040109a50 0x7f5040109e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.061+0000 7f5046e00640 1 -- 172.21.15.28:0/360101561 >> 172.21.15.28:0/360101561 conn(0x7f504006db20 msgr2=0x7f504006df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.062+0000 7f5046e00640 1 -- 172.21.15.28:0/360101561 shutdown_connections 2024-05-20T20:14:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.062+0000 7f5046e00640 1 -- 172.21.15.28:0/360101561 wait complete. 2024-05-20T20:14:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.063+0000 7f5046e00640 1 Processor -- start 2024-05-20T20:14:42.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.064+0000 7f5046e00640 1 -- start start 2024-05-20T20:14:42.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.064+0000 7f5046e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5040109a50 0x7f504010d0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.064+0000 7f5046e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f504010a370 0x7f504010b4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.064+0000 7f5046e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5040115ba0 0x7f504010bb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.064+0000 7f5046e00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f504011b490 con 0x7f5040109a50 2024-05-20T20:14:42.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.064+0000 7f5046e00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f504011b310 con 0x7f504010a370 2024-05-20T20:14:42.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.064+0000 7f5046e00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f504011b610 con 0x7f5040115ba0 2024-05-20T20:14:42.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.066+0000 7f5044e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5040115ba0 0x7f504010bb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.066+0000 7f5044e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5040115ba0 0x7f504010bb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50642/0 (socket says 172.21.15.28:50642) 2024-05-20T20:14:42.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.066+0000 7f5044e00640 1 -- 172.21.15.28:0/3245124332 learned_addr learned my addr 172.21.15.28:0/3245124332 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:42.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.066+0000 7f503f400640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f504010a370 0x7f504010b4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.066+0000 7f503fe00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5040109a50 0x7f504010d0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.067+0000 7f503f400640 1 -- 172.21.15.28:0/3245124332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5040115ba0 msgr2=0x7f504010bb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.067+0000 7f503f400640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5040115ba0 0x7f504010bb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.067+0000 7f503f400640 1 -- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5040109a50 msgr2=0x7f504010d0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.067+0000 7f503f400640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5040109a50 0x7f504010d0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.069 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.067+0000 7f503f400640 1 -- 172.21.15.28:0/3245124332 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f50401a5b10 con 0x7f504010a370 2024-05-20T20:14:42.069 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.067+0000 7f5044e00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5040115ba0 0x7f504010bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:42.069 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.069+0000 7f503f400640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f504010a370 0x7f504010b4c0 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f502c00c970 tx=0x7f502c00ce30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:42.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.069+0000 7f503cc00640 1 -- 172.21.15.28:0/3245124332 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f502c007bb0 con 0x7f504010a370 2024-05-20T20:14:42.070 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.070+0000 7f503cc00640 1 -- 172.21.15.28:0/3245124332 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f502c007d50 con 0x7f504010a370 2024-05-20T20:14:42.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.070+0000 7f503cc00640 1 -- 172.21.15.28:0/3245124332 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f502c018780 con 0x7f504010a370 2024-05-20T20:14:42.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.070+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f50401a5e00 con 0x7f504010a370 2024-05-20T20:14:42.071 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.071+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f50401a6300 con 0x7f504010a370 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.078+0000 7f503cc00640 1 -- 172.21.15.28:0/3245124332 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f502c00f450 con 0x7f504010a370 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 Processor -- start 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 -- start start 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bc0070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bca930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bcae70 0x556f86bcd260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556f86b30b90 con 0x556f86bbfc90 2024-05-20T20:14:42.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556f86b20730 con 0x556f86bcae70 2024-05-20T20:14:42.081 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.079+0000 7f5049bf8740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556f86b24770 con 0x556f86bc05b0 2024-05-20T20:14:42.081 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.080+0000 7f503f400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bcae70 0x556f86bcd260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.081 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.080+0000 7f5044e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bc0070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.081 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.080+0000 7f503f400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bcae70 0x556f86bcd260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34194/0 (socket says 172.21.15.28:34194) 2024-05-20T20:14:42.081 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.080+0000 7f503f400640 1 -- 172.21.15.28:0/2008178023 learned_addr learned my addr 172.21.15.28:0/2008178023 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:42.081 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.080+0000 7f503fe00640 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bca930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.081+0000 7f5044e00640 1 -- 172.21.15.28:0/2008178023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 msgr2=0x556f86bca930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:42.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.081+0000 7f5044e00640 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bca930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.081+0000 7f5044e00640 1 -- 172.21.15.28:0/2008178023 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bcae70 msgr2=0x556f86bcd260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:42.082 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.081+0000 7f5044e00640 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bcae70 0x556f86bcd260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.081+0000 7f5044e00640 1 -- 172.21.15.28:0/2008178023 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556f86ab98b0 con 0x556f86bbfc90 2024-05-20T20:14:42.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.081+0000 7f503fe00640 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bca930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:42.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.082+0000 7f5044e00640 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bc0070 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f5038006010 tx=0x7f5038032e90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29f3b189d44b5c93 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:42.083 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.083+0000 7f503cc00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50240b2c20 0x7f50240b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.084 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.083+0000 7f503cc00640 1 -- 172.21.15.28:0/3245124332 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f502c0e2520 con 0x7f504010a370 2024-05-20T20:14:42.085 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.084+0000 7f5035600640 1 -- 172.21.15.28:0/2008178023 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5038041030 con 0x556f86bbfc90 2024-05-20T20:14:42.085 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.084+0000 7f503fe00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50240b2c20 0x7f50240b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.085 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.085+0000 7f5035600640 1 -- 172.21.15.28:0/2008178023 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5038003e20 con 0x556f86bbfc90 2024-05-20T20:14:42.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.086+0000 7f5049bf8740 1 -- 172.21.15.28:0/2008178023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 msgr2=0x556f86bc0070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.086+0000 7f5049bf8740 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bc0070 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f5038006010 tx=0x7f5038032e90 comp rx=0 tx=0).stop 2024-05-20T20:14:42.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f503fe00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50240b2c20 0x7f50240b50e0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x556f86bc1230 tx=0x7f503000d500 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:42.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f5049bf8740 1 -- 172.21.15.28:0/2008178023 shutdown_connections 2024-05-20T20:14:42.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f5049bf8740 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bcae70 0x556f86bcd260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f5049bf8740 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bca930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f5049bf8740 1 --2- 172.21.15.28:0/2008178023 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bc0070 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f5049bf8740 1 -- 172.21.15.28:0/2008178023 >> 172.21.15.28:0/2008178023 conn(0x556f86bbc1a0 msgr2=0x556f86bbc580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:42.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f5049bf8740 1 -- 172.21.15.28:0/2008178023 shutdown_connections 2024-05-20T20:14:42.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.087+0000 7f5049bf8740 1 -- 172.21.15.28:0/2008178023 wait complete. 2024-05-20T20:14:42.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.088+0000 7f5049bf8740 1 -- start start 2024-05-20T20:14:42.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.088+0000 7f5049bf8740 1 Processor -- start 2024-05-20T20:14:42.091 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.090+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bbfc90 0x556f86bca740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.090+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86bc7a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.090+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bc7f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86bc7a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86bc7a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:34200/0 (socket says 172.21.15.28:34200) 2024-05-20T20:14:42.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 -- 172.21.15.28:0/3302500488 learned_addr learned my addr 172.21.15.28:0/3302500488 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:42.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556f86806790 con 0x556f86bcae70 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556f86ac8bc0 con 0x556f86bc05b0 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556f86bcda40 con 0x556f86bbfc90 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f5049bf8740 10 client.0 _ll_register_callbacks cb 0x556f86aa6400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 -- 172.21.15.28:0/3302500488 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bbfc90 msgr2=0x556f86bca740 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bbfc90 0x556f86bca740 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 -- 172.21.15.28:0/3302500488 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 msgr2=0x556f86bc7f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bc7f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.091+0000 7f503f400640 1 -- 172.21.15.28:0/3302500488 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556f86835210 con 0x556f86bc05b0 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.092+0000 7f503f400640 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86bc7a30 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f502c0a9510 tx=0x7f502c007f00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=40c968afcacb5a48 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:42.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.092+0000 7f502a000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:42.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.092+0000 7f502a000640 1 -- 172.21.15.28:0/3302500488 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f502c0a56d0 con 0x556f86bc05b0 2024-05-20T20:14:42.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.092+0000 7f502a000640 1 -- 172.21.15.28:0/3302500488 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f502c0e2030 con 0x556f86bc05b0 2024-05-20T20:14:42.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.096+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5038003e20 con 0x556f86bc05b0 2024-05-20T20:14:42.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.100+0000 7f502a000640 1 -- 172.21.15.28:0/3302500488 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f502c0a8f80 con 0x556f86bc05b0 2024-05-20T20:14:42.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.102+0000 7f5049bf8740 10 client.27344 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:42.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.102+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556f86901620 con 0x556f86bc05b0 2024-05-20T20:14:42.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.102+0000 7f502a000640 1 -- 172.21.15.28:0/3302500488 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5038003e20 con 0x556f86bc05b0 2024-05-20T20:14:42.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.103+0000 7f502a000640 1 -- 172.21.15.28:0/3302500488 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f502c0a8a70 con 0x556f86bc05b0 2024-05-20T20:14:42.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.104+0000 7f5049bf8740 10 client.27344 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:42.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.104+0000 7f5049bf8740 10 client.27344 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:42.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.105+0000 7f5049bf8740 20 client.27344 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:42.106 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.105+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86bdcd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.106 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.105+0000 7f5049bf8740 4 client.27344 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:42.106 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.105+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556f86b32ae0 con 0x556f86bda890 2024-05-20T20:14:42.106 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.105+0000 7f503fe00640 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86bdcd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.107+0000 7f503fe00640 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86bdcd00 crc :-1 s=READY pgs=201 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2375ee55190d0b3e server_cookie=870897eeee3d8e24 in_seq=0 out_seq=0 2024-05-20T20:14:42.107 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.107+0000 7f502a000640 10 client.27344 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:42.108 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.108+0000 7f502a000640 1 -- 172.21.15.28:0/3302500488 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x556f86b32ae0 con 0x556f86bda890 2024-05-20T20:14:42.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.108+0000 7f502a000640 10 client.27344 handle_command_reply: tid=0 2024-05-20T20:14:42.109 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.108+0000 7f5049bf8740 1 client.27344 shutdown 2024-05-20T20:14:42.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.109+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 msgr2=0x556f86bc7a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.109+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86bc7a30 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f502c0a9510 tx=0x7f502c007f00 comp rx=0 tx=0).stop 2024-05-20T20:14:42.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.111+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 shutdown_connections 2024-05-20T20:14:42.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.111+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86bdcd00 crc :-1 s=READY pgs=201 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.111+0000 7f502a000640 0 client.27344 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:42.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.111+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86bc7a30 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.112+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bc7f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.112+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3302500488 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bbfc90 0x556f86bca740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.112+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 >> 172.21.15.28:0/3302500488 conn(0x556f86bbc1a0 msgr2=0x556f86bc1f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:42.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.113+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 shutdown_connections 2024-05-20T20:14:42.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.113+0000 7f5049bf8740 1 -- 172.21.15.28:0/3302500488 wait complete. 2024-05-20T20:14:42.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.113+0000 7f5049bf8740 20 client.27344 trim_cache size 0 max 16384 2024-05-20T20:14:42.167 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.166+0000 7f5049bf8740 1 Processor -- start 2024-05-20T20:14:42.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.167+0000 7f5049bf8740 1 -- start start 2024-05-20T20:14:42.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.167+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bd6b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.167+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86be2610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.168 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.167+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bcae70 0x556f86be2b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.169 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.167+0000 7f5049bf8740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f50401c5be0 con 0x556f86bbfc90 2024-05-20T20:14:42.169 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.167+0000 7f5049bf8740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f50401c7720 con 0x556f86bc05b0 2024-05-20T20:14:42.169 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.167+0000 7f5049bf8740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f50401c62b0 con 0x556f86bcae70 2024-05-20T20:14:42.169 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.168+0000 7f5044e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bcae70 0x556f86be2b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.169+0000 7f5044e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bcae70 0x556f86be2b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50668/0 (socket says 172.21.15.28:50668) 2024-05-20T20:14:42.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.169+0000 7f5044e00640 1 -- 172.21.15.28:0/3711799313 learned_addr learned my addr 172.21.15.28:0/3711799313 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:42.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.169+0000 7f5044e00640 1 -- 172.21.15.28:0/3711799313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 msgr2=0x556f86be2610 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:42.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.169+0000 7f5044e00640 1 --2- 172.21.15.28:0/3711799313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86be2610 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.169+0000 7f5044e00640 1 -- 172.21.15.28:0/3711799313 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 msgr2=0x556f86bd6b20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:14:42.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.170+0000 7f5044e00640 1 --2- 172.21.15.28:0/3711799313 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bd6b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.170+0000 7f5044e00640 1 -- 172.21.15.28:0/3711799313 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556f86bcd5a0 con 0x556f86bcae70 2024-05-20T20:14:42.171 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.171+0000 7f5044e00640 1 --2- 172.21.15.28:0/3711799313 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bcae70 0x556f86be2b50 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f5038004af0 tx=0x7f5038009aa0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a40133260587f067 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:42.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.172+0000 7f5034c00640 1 -- 172.21.15.28:0/3711799313 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5038041040 con 0x556f86bcae70 2024-05-20T20:14:42.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.172+0000 7f5034c00640 1 -- 172.21.15.28:0/3711799313 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5038030e20 con 0x556f86bcae70 2024-05-20T20:14:42.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.172+0000 7f5034c00640 1 -- 172.21.15.28:0/3711799313 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5038004010 con 0x556f86bcae70 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 -- 172.21.15.28:0/3711799313 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bcae70 msgr2=0x556f86be2b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3711799313 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bcae70 0x556f86be2b50 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f5038004af0 tx=0x7f5038009aa0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 -- 172.21.15.28:0/3711799313 shutdown_connections 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3711799313 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bc05b0 0x556f86be2610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3711799313 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bcae70 0x556f86be2b50 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3711799313 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bbfc90 0x556f86bd6b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 -- 172.21.15.28:0/3711799313 >> 172.21.15.28:0/3711799313 conn(0x556f86bbc1a0 msgr2=0x556f86bbb310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 -- 172.21.15.28:0/3711799313 shutdown_connections 2024-05-20T20:14:42.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.173+0000 7f5049bf8740 1 -- 172.21.15.28:0/3711799313 wait complete. 2024-05-20T20:14:42.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.174+0000 7f5049bf8740 1 -- start start 2024-05-20T20:14:42.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.174+0000 7f5049bf8740 1 Processor -- start 2024-05-20T20:14:42.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bbfc90 0x556f86bd30c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bd0450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5049bf8740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bd0990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5049bf8740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f50401c7050 con 0x556f86bcae70 2024-05-20T20:14:42.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5049bf8740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f50401c6980 con 0x556f86bbfc90 2024-05-20T20:14:42.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5049bf8740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f50401acba0 con 0x556f86bc05b0 2024-05-20T20:14:42.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f503f400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bd0450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f503f400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bd0450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50676/0 (socket says 172.21.15.28:50676) 2024-05-20T20:14:42.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f503f400640 1 -- 172.21.15.28:0/3244868072 learned_addr learned my addr 172.21.15.28:0/3244868072 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:42.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5049bf8740 10 client.0 _ll_register_callbacks cb 0x7f502c0a5050 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:42.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.175+0000 7f5044e00640 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bd0990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.176+0000 7f503fe00640 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bbfc90 0x556f86bd30c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.176+0000 7f5044e00640 1 -- 172.21.15.28:0/3244868072 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 msgr2=0x556f86bd0450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:42.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.176+0000 7f5044e00640 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bd0450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.176+0000 7f5044e00640 1 -- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bbfc90 msgr2=0x556f86bd30c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:42.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.176+0000 7f5044e00640 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bbfc90 0x556f86bd30c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.177+0000 7f5044e00640 1 -- 172.21.15.28:0/3244868072 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556f86bd0ed0 con 0x556f86bcae70 2024-05-20T20:14:42.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.177+0000 7f5044e00640 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bd0990 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f5038034040 tx=0x7f5038009840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d566c85b48402856 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:42.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.177+0000 7f502a000640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:14:42.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.178+0000 7f502a000640 1 -- 172.21.15.28:0/3244868072 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5038041040 con 0x556f86bcae70 2024-05-20T20:14:42.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.178+0000 7f502a000640 1 -- 172.21.15.28:0/3244868072 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f5038004790 con 0x556f86bcae70 2024-05-20T20:14:42.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.178+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5038030e20 con 0x556f86bcae70 2024-05-20T20:14:42.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.179+0000 7f502a000640 1 -- 172.21.15.28:0/3244868072 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f503805e050 con 0x556f86bcae70 2024-05-20T20:14:42.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.179+0000 7f5049bf8740 10 client.16929 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:42.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.179+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556f86c451d0 con 0x556f86bcae70 2024-05-20T20:14:42.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f502a000640 1 -- 172.21.15.28:0/3244868072 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5038030e20 con 0x556f86bcae70 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f502a000640 1 -- 172.21.15.28:0/3244868072 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f503804f4a0 con 0x556f86bcae70 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f5049bf8740 10 client.16929 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f5049bf8740 10 client.16929 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f5049bf8740 20 client.16929 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86c473b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f5049bf8740 4 client.16929 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.180+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x556f86b2c2c0 con 0x556f86bda890 2024-05-20T20:14:42.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.181+0000 7f503fe00640 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86c473b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:42.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.181+0000 7f503fe00640 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86c473b0 crc :-1 s=READY pgs=202 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3642f36bf1ece62 server_cookie=70c1712394d19aaa in_seq=0 out_seq=0 2024-05-20T20:14:42.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.181+0000 7f502a000640 10 client.16929 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:42.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.229+0000 7f502a000640 1 -- 172.21.15.28:0/3244868072 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x556f86b2c2c0 con 0x556f86bda890 2024-05-20T20:14:42.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.229+0000 7f502a000640 10 client.16929 handle_command_reply: tid=0 2024-05-20T20:14:42.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.229+0000 7f5049bf8740 1 client.16929 shutdown 2024-05-20T20:14:42.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.231+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 msgr2=0x556f86bd0990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.231+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bd0990 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f5038034040 tx=0x7f5038009840 comp rx=0 tx=0).stop 2024-05-20T20:14:42.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.233+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 shutdown_connections 2024-05-20T20:14:42.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.233+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x556f86bda890 0x556f86c473b0 crc :-1 s=READY pgs=202 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.233+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556f86bcae70 0x556f86bd0990 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.233+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556f86bc05b0 0x556f86bd0450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.233+0000 7f5049bf8740 1 --2- 172.21.15.28:0/3244868072 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556f86bbfc90 0x556f86bd30c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.233+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 >> 172.21.15.28:0/3244868072 conn(0x556f86bbc1a0 msgr2=0x556f86bbb310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:42.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.233+0000 7f502a000640 0 client.16929 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:42.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.235+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 shutdown_connections 2024-05-20T20:14:42.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.235+0000 7f5049bf8740 1 -- 172.21.15.28:0/3244868072 wait complete. 2024-05-20T20:14:42.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.236+0000 7f5049bf8740 20 client.16929 trim_cache size 0 max 16384 2024-05-20T20:14:42.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:14:42.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:14:42.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:14:42.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.241+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50240b2c20 msgr2=0x7f50240b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.241+0000 7f5046e00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50240b2c20 0x7f50240b50e0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x556f86bc1230 tx=0x7f503000d500 comp rx=0 tx=0).stop 2024-05-20T20:14:42.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.241+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f504010a370 msgr2=0x7f504010b4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:42.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.241+0000 7f5046e00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f504010a370 0x7f504010b4c0 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f502c00c970 tx=0x7f502c00ce30 comp rx=0 tx=0).stop 2024-05-20T20:14:42.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.242+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 shutdown_connections 2024-05-20T20:14:42.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.242+0000 7f5046e00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f504010a370 0x7f504010b4c0 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.242+0000 7f5046e00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f50240b2c20 0x7f50240b50e0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.242+0000 7f5046e00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5040115ba0 0x7f504010bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.242+0000 7f5046e00640 1 --2- 172.21.15.28:0/3245124332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5040109a50 0x7f504010d0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:42.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.242+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 >> 172.21.15.28:0/3245124332 conn(0x7f504006db20 msgr2=0x7f5040071710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:42.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.243+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 shutdown_connections 2024-05-20T20:14:42.244 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:42.243+0000 7f5046e00640 1 -- 172.21.15.28:0/3245124332 wait complete. 2024-05-20T20:14:43.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:43 smithi154 ceph-mon[36314]: pgmap v281: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 236 MiB/s wr, 2.03k op/s 2024-05-20T20:14:43.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:43 smithi033 ceph-mon[36179]: pgmap v281: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 236 MiB/s wr, 2.03k op/s 2024-05-20T20:14:43.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:43 smithi028 ceph-mon[31178]: pgmap v281: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 236 MiB/s wr, 2.03k op/s 2024-05-20T20:14:45.174 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:14:45.174 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:14:45.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:45 smithi154 ceph-mon[36314]: pgmap v282: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 322 MiB/s wr, 2.56k op/s 2024-05-20T20:14:45.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:45 smithi033 ceph-mon[36179]: pgmap v282: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 322 MiB/s wr, 2.56k op/s 2024-05-20T20:14:45.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:45 smithi028 ceph-mon[31178]: pgmap v282: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 322 MiB/s wr, 2.56k op/s 2024-05-20T20:14:45.951 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:47.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:47 smithi154 ceph-mon[36314]: pgmap v283: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 248 MiB/s wr, 1.78k op/s 2024-05-20T20:14:47.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:47 smithi033 ceph-mon[36179]: pgmap v283: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 248 MiB/s wr, 1.78k op/s 2024-05-20T20:14:47.489 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:47 smithi028 ceph-mon[31178]: pgmap v283: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 248 MiB/s wr, 1.78k op/s 2024-05-20T20:14:49.512 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:49 smithi028 ceph-mon[31178]: pgmap v284: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 258 MiB/s wr, 1.76k op/s 2024-05-20T20:14:49.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:49 smithi154 ceph-mon[36314]: pgmap v284: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 258 MiB/s wr, 1.76k op/s 2024-05-20T20:14:49.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:49 smithi033 ceph-mon[36179]: pgmap v284: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 258 MiB/s wr, 1.76k op/s 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.238+0000 7ffba9e00700 1 --2- 172.21.15.28:0/1505125901 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1016c0 0x7ffbac101aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello state changed while learned_addr, mark_down or replacing must be happened just now 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.239+0000 7ffba8a00700 1 -- 172.21.15.28:0/1505125901 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffb9c003c00 con 0x7ffbac101fe0 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.239+0000 7ffbb1200700 1 -- 172.21.15.28:0/1505125901 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac101fe0 msgr2=0x7ffbac113ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.239+0000 7ffbb1200700 1 --2- 172.21.15.28:0/1505125901 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac101fe0 0x7ffbac113ea0 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7ffb9c009990 tx=0x7ffb9c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.240+0000 7ffbb1200700 1 -- 172.21.15.28:0/1505125901 shutdown_connections 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.240+0000 7ffbb1200700 1 --2- 172.21.15.28:0/1505125901 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbac1143e0 0x7ffbac1167d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.240+0000 7ffbb1200700 1 --2- 172.21.15.28:0/1505125901 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1016c0 0x7ffbac101aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.242 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.240+0000 7ffbb1200700 1 --2- 172.21.15.28:0/1505125901 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac101fe0 0x7ffbac113ea0 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.243 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.240+0000 7ffbb1200700 1 -- 172.21.15.28:0/1505125901 >> 172.21.15.28:0/1505125901 conn(0x7ffbac078640 msgr2=0x7ffbac0ff9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:50.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.244+0000 7ffbb1200700 1 -- 172.21.15.28:0/1505125901 shutdown_connections 2024-05-20T20:14:50.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.244+0000 7ffbb1200700 1 -- 172.21.15.28:0/1505125901 wait complete. 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.246+0000 7ffbb1200700 1 Processor -- start 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.248+0000 7ffbb1200700 1 -- start start 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.248+0000 7ffbb1200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac1016c0 0x7ffbac06e710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.248+0000 7ffbb1200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbac101fe0 0x7ffbac06ec50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.248+0000 7ffbb1200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1143e0 0x7ffbac0731b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.248+0000 7ffbb1200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffbac06d2f0 con 0x7ffbac101fe0 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.248+0000 7ffbb1200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffbac06d170 con 0x7ffbac1143e0 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.248+0000 7ffbb1200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffbac06d470 con 0x7ffbac1016c0 2024-05-20T20:14:50.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.250+0000 7ffbaa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1143e0 0x7ffbac0731b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.250+0000 7ffbaa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1143e0 0x7ffbac0731b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57962/0 (socket says 172.21.15.28:57962) 2024-05-20T20:14:50.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.250+0000 7ffbaa800700 1 -- 172.21.15.28:0/3845542295 learned_addr learned my addr 172.21.15.28:0/3845542295 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:50.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.251+0000 7ffba9e00700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac1016c0 0x7ffbac06e710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.253+0000 7ffbaa800700 1 -- 172.21.15.28:0/3845542295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac1016c0 msgr2=0x7ffbac06e710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.253+0000 7ffbaa800700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac1016c0 0x7ffbac06e710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.253+0000 7ffbaa800700 1 -- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbac101fe0 msgr2=0x7ffbac06ec50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.253+0000 7ffba9400700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbac101fe0 0x7ffbac06ec50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.253+0000 7ffbaa800700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbac101fe0 0x7ffbac06ec50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.253+0000 7ffbaa800700 1 -- 172.21.15.28:0/3845542295 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffbac0736f0 con 0x7ffbac1143e0 2024-05-20T20:14:50.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.253+0000 7ffba9400700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbac101fe0 0x7ffbac06ec50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:50.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.255+0000 7ffbaa800700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1143e0 0x7ffbac0731b0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7ffba4007d30 tx=0x7ffba400a340 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:50.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.256+0000 7ffba2a00700 1 -- 172.21.15.28:0/3845542295 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffba4016030 con 0x7ffbac1143e0 2024-05-20T20:14:50.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.256+0000 7ffba2a00700 1 -- 172.21.15.28:0/3845542295 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffba4004bb0 con 0x7ffbac1143e0 2024-05-20T20:14:50.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.256+0000 7ffba2a00700 1 -- 172.21.15.28:0/3845542295 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffba4005740 con 0x7ffbac1143e0 2024-05-20T20:14:50.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.256+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffbac0739e0 con 0x7ffbac1143e0 2024-05-20T20:14:50.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.256+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffbac073ed0 con 0x7ffbac1143e0 2024-05-20T20:14:50.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.264+0000 7ffbbbbf9b80 1 Processor -- start 2024-05-20T20:14:50.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.265+0000 7ffba2a00700 1 -- 172.21.15.28:0/3845542295 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ffba4019070 con 0x7ffbac1143e0 2024-05-20T20:14:50.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.268+0000 7ffba2a00700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffb900b2bc0 0x7ffb900b5080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.268+0000 7ffbbbbf9b80 1 -- start start 2024-05-20T20:14:50.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.269+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4b310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.269+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 0x563a83d452e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.269+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d45d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.269+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563a83ca3ec0 con 0x563a83d4b850 2024-05-20T20:14:50.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.269+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563a83c93a60 con 0x563a83d45900 2024-05-20T20:14:50.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.269+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563a83c97aa0 con 0x563a83d4af30 2024-05-20T20:14:50.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.270+0000 7ffbaa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d45d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.270+0000 7ffbaa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d45d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57966/0 (socket says 172.21.15.28:57966) 2024-05-20T20:14:50.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.270+0000 7ffbaa800700 1 -- 172.21.15.28:0/2643678980 learned_addr learned my addr 172.21.15.28:0/2643678980 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:50.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.270+0000 7ffba9e00700 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4b310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.270+0000 7ffba9400700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffb900b2bc0 0x7ffb900b5080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.271+0000 7ffba2a00700 1 -- 172.21.15.28:0/3845542295 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7ffba40e1e20 con 0x7ffbac1143e0 2024-05-20T20:14:50.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.273+0000 7ffbaa800700 1 -- 172.21.15.28:0/2643678980 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 msgr2=0x563a83d4b310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:50.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.273+0000 7ffbaa800700 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4b310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.273+0000 7ffbaa800700 1 -- 172.21.15.28:0/2643678980 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 msgr2=0x563a83d452e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:50.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.273+0000 7ffbaa800700 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 0x563a83d452e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.273+0000 7ffbaa800700 1 -- 172.21.15.28:0/2643678980 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563a83c5c4a0 con 0x563a83d45900 2024-05-20T20:14:50.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.275+0000 7ffbaa800700 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d45d60 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7ffba40df550 tx=0x7ffba40a8ff0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fc97b408a4fe7ad6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:50.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.276+0000 7ffb97400700 1 -- 172.21.15.28:0/2643678980 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffba40a8cb0 con 0x563a83d45900 2024-05-20T20:14:50.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.276+0000 7ffb97400700 1 -- 172.21.15.28:0/2643678980 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffba40a98a0 con 0x563a83d45900 2024-05-20T20:14:50.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.277+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2643678980 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 msgr2=0x563a83d45d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.277+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d45d60 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7ffba40df550 tx=0x7ffba40a8ff0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.279+0000 7ffba9400700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffb900b2bc0 0x7ffb900b5080 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7ffb9c009750 tx=0x7ffb9c00b580 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:50.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.281+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2643678980 shutdown_connections 2024-05-20T20:14:50.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.281+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d45d60 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.281+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4b310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.281+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2643678980 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 0x563a83d452e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.281+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2643678980 >> 172.21.15.28:0/2643678980 conn(0x563a83c73c60 msgr2=0x563a83d40090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:50.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.282+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2643678980 shutdown_connections 2024-05-20T20:14:50.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.282+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2643678980 wait complete. 2024-05-20T20:14:50.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.284+0000 7ffbbbbf9b80 1 -- start start 2024-05-20T20:14:50.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.285+0000 7ffbbbbf9b80 1 Processor -- start 2024-05-20T20:14:50.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.286+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d45900 0x563a83d51410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.286+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4e7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.286+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4b850 0x563a83d4ecf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.286+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563a83b9a020 con 0x563a83d45900 2024-05-20T20:14:50.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.286+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563a83c347e0 con 0x563a83d4b850 2024-05-20T20:14:50.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.286+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563a83d52140 con 0x563a83d4af30 2024-05-20T20:14:50.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.286+0000 7ffbbbbf9b80 10 client.0 _ll_register_callbacks cb 0x563a83c3dcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:50.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.287+0000 7ffba9400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4e7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.287+0000 7ffbaa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4b850 0x563a83d4ecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.287+0000 7ffbaa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4b850 0x563a83d4ecf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57982/0 (socket says 172.21.15.28:57982) 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.287+0000 7ffbaa800700 1 -- 172.21.15.28:0/1691600370 learned_addr learned my addr 172.21.15.28:0/1691600370 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.288+0000 7ffbaa800700 1 -- 172.21.15.28:0/1691600370 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 msgr2=0x563a83d4e7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.288+0000 7ffbaa800700 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4e7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.288+0000 7ffbaa800700 1 -- 172.21.15.28:0/1691600370 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d45900 msgr2=0x563a83d51410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.288+0000 7ffbaa800700 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d45900 0x563a83d51410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.288+0000 7ffbaa800700 1 -- 172.21.15.28:0/1691600370 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563a83be3950 con 0x563a83d4b850 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.289+0000 7ffbaa800700 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4b850 0x563a83d4ecf0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7ffba40a5960 tx=0x7ffba4005180 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cf4b90f79709f32a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.289+0000 7ffb8be00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.290+0000 7ffb8be00700 1 -- 172.21.15.28:0/1691600370 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffba40a8cb0 con 0x563a83d4b850 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.290+0000 7ffb8be00700 1 -- 172.21.15.28:0/1691600370 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7ffba40a9600 con 0x563a83d4b850 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.290+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffba40a98a0 con 0x563a83d4b850 2024-05-20T20:14:50.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.290+0000 7ffb8be00700 1 -- 172.21.15.28:0/1691600370 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffba40df370 con 0x563a83d4b850 2024-05-20T20:14:50.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.290+0000 7ffbbbbf9b80 10 client.27356 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:50.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.290+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563a83c640e0 con 0x563a83d4b850 2024-05-20T20:14:50.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.291+0000 7ffb8be00700 1 -- 172.21.15.28:0/1691600370 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffba40a98a0 con 0x563a83d4b850 2024-05-20T20:14:50.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.292+0000 7ffb8be00700 1 -- 172.21.15.28:0/1691600370 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffba4023060 con 0x563a83d4b850 2024-05-20T20:14:50.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.292+0000 7ffbbbbf9b80 10 client.27356 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:50.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.292+0000 7ffbbbbf9b80 10 client.27356 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:50.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.292+0000 7ffbbbbf9b80 20 client.27356 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:50.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.292+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83d5eac0 0x563a83d60eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.292+0000 7ffbbbbf9b80 4 client.27356 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:50.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.292+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563a83ca5e10 con 0x563a83d5eac0 2024-05-20T20:14:50.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.297+0000 7ffba9e00700 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83d5eac0 0x563a83d60eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.298+0000 7ffba9e00700 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83d5eac0 0x563a83d60eb0 crc :-1 s=READY pgs=203 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c6819c263461f117 server_cookie=86762f18d8457c0c in_seq=0 out_seq=0 2024-05-20T20:14:50.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.298+0000 7ffb8be00700 10 client.27356 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:50.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.298+0000 7ffb8be00700 1 -- 172.21.15.28:0/1691600370 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x563a83ca5e10 con 0x563a83d5eac0 2024-05-20T20:14:50.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.298+0000 7ffb8be00700 10 client.27356 handle_command_reply: tid=0 2024-05-20T20:14:50.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 client.27356 shutdown 2024-05-20T20:14:50.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4b850 msgr2=0x563a83d4ecf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4b850 0x563a83d4ecf0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7ffba40a5960 tx=0x7ffba4005180 comp rx=0 tx=0).stop 2024-05-20T20:14:50.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 shutdown_connections 2024-05-20T20:14:50.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83d5eac0 0x563a83d60eb0 crc :-1 s=READY pgs=203 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.302 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4b850 0x563a83d4ecf0 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d45900 0x563a83d51410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/1691600370 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4af30 0x563a83d4e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffb8be00700 0 client.27356 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:50.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.300+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 >> 172.21.15.28:0/1691600370 conn(0x563a83c73c60 msgr2=0x563a83d442d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:50.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.301+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 shutdown_connections 2024-05-20T20:14:50.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.301+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/1691600370 wait complete. 2024-05-20T20:14:50.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.302+0000 7ffbbbbf9b80 20 client.27356 trim_cache size 0 max 16384 2024-05-20T20:14:50.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 Processor -- start 2024-05-20T20:14:50.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 -- start start 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d45900 0x563a83d4eca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4af30 0x563a83d452e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 0x563a83d4bc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffb8c00d720 con 0x563a83d4b850 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffb8c00f260 con 0x563a83d4af30 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.351+0000 7ffbbbbf9b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffb8c00ddf0 con 0x563a83d45900 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d45900 0x563a83d4eca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d45900 0x563a83d4eca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52262/0 (socket says 172.21.15.28:52262) 2024-05-20T20:14:50.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 -- 172.21.15.28:0/3723092134 learned_addr learned my addr 172.21.15.28:0/3723092134 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:50.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 -- 172.21.15.28:0/3723092134 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4af30 msgr2=0x563a83d452e0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:50.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 --2- 172.21.15.28:0/3723092134 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4af30 0x563a83d452e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 -- 172.21.15.28:0/3723092134 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 msgr2=0x563a83d4bc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:50.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 --2- 172.21.15.28:0/3723092134 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 0x563a83d4bc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.352+0000 7ffba9e00700 1 -- 172.21.15.28:0/3723092134 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563a83d41dc0 con 0x563a83d45900 2024-05-20T20:14:50.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.353+0000 7ffba9e00700 1 --2- 172.21.15.28:0/3723092134 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d45900 0x563a83d4eca0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7ffb9800add0 tx=0x7ffb98012730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fcb79b18817433f1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:50.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.353+0000 7ffb96a00700 1 -- 172.21.15.28:0/3723092134 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffb98021020 con 0x563a83d45900 2024-05-20T20:14:50.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.353+0000 7ffb96a00700 1 -- 172.21.15.28:0/3723092134 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffb98012c60 con 0x563a83d45900 2024-05-20T20:14:50.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.353+0000 7ffb96a00700 1 -- 172.21.15.28:0/3723092134 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffb98010690 con 0x563a83d45900 2024-05-20T20:14:50.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.354+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/3723092134 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d45900 msgr2=0x563a83d4eca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.355 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.354+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/3723092134 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d45900 0x563a83d4eca0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7ffb9800add0 tx=0x7ffb98012730 comp rx=0 tx=0).stop 2024-05-20T20:14:50.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.355+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/3723092134 shutdown_connections 2024-05-20T20:14:50.356 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.355+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/3723092134 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4b850 0x563a83d4bc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.355+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/3723092134 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d45900 0x563a83d4eca0 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.355+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/3723092134 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d4af30 0x563a83d452e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.355+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/3723092134 >> 172.21.15.28:0/3723092134 conn(0x563a83c73c60 msgr2=0x563a83d4e590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:50.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.357+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/3723092134 shutdown_connections 2024-05-20T20:14:50.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.357+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/3723092134 wait complete. 2024-05-20T20:14:50.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.358+0000 7ffbbbbf9b80 1 -- start start 2024-05-20T20:14:50.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.358+0000 7ffbbbbf9b80 1 Processor -- start 2024-05-20T20:14:50.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.360+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d544a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.360+0000 7ffbbbbf9b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4af30 0x563a83d549e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.360+0000 7ffba9e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d544a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.360+0000 7ffba9e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d544a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58006/0 (socket says 172.21.15.28:58006) 2024-05-20T20:14:50.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.361+0000 7ffba9e00700 1 -- 172.21.15.28:0/2493669360 learned_addr learned my addr 172.21.15.28:0/2493669360 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:50.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.361+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4b850 0x563a83d54f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.361+0000 7ffba9400700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4af30 0x563a83d549e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.362+0000 7ffbaa800700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4b850 0x563a83d54f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.362+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffb8c00eb90 con 0x563a83d4af30 2024-05-20T20:14:50.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.363+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffb8c00e4c0 con 0x563a83d45900 2024-05-20T20:14:50.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.363+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffb8c00b680 con 0x563a83d4b850 2024-05-20T20:14:50.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffbbbbf9b80 10 client.0 _ll_register_callbacks cb 0x563a83c3dcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:50.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffbaa800700 1 -- 172.21.15.28:0/2493669360 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 msgr2=0x563a83d544a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffbaa800700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d544a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffbaa800700 1 -- 172.21.15.28:0/2493669360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4af30 msgr2=0x563a83d549e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffbaa800700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4af30 0x563a83d549e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffbaa800700 1 -- 172.21.15.28:0/2493669360 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563a83dc9330 con 0x563a83d4b850 2024-05-20T20:14:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffb98012c60 con 0x563a83d4b850 2024-05-20T20:14:50.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.364+0000 7ffba9400700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4af30 0x563a83d549e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:14:50.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.365+0000 7ffbaa800700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4b850 0x563a83d54f20 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7ffba40a5220 tx=0x7ffba40e2070 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3dceebd5407f2eea server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:50.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.365+0000 7ffb8be00700 10 client.27271 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:14:50.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffb8be00700 1 -- 172.21.15.28:0/2493669360 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffba40a8cb0 con 0x563a83d4b850 2024-05-20T20:14:50.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffb8be00700 1 -- 172.21.15.28:0/2493669360 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7ffba40a9400 con 0x563a83d4b850 2024-05-20T20:14:50.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffb8be00700 1 -- 172.21.15.28:0/2493669360 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffba4005e30 con 0x563a83d4b850 2024-05-20T20:14:50.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 10 client.27271 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x563a83dc9330 con 0x563a83d4b850 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffb8be00700 1 -- 172.21.15.28:0/2493669360 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffba40be1d0 con 0x563a83d4b850 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 10 client.27271 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 10 client.27271 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 20 client.27271 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83dca680 0x563a83dcca70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 4 client.27271 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.366+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x563a83c9f5f0 con 0x563a83dca680 2024-05-20T20:14:50.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.368+0000 7ffba9e00700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83dca680 0x563a83dcca70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:50.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.373+0000 7ffba9e00700 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83dca680 0x563a83dcca70 crc :-1 s=READY pgs=204 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2b926dec86947bdb server_cookie=74725498aa583d45 in_seq=0 out_seq=0 2024-05-20T20:14:50.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.373+0000 7ffb8be00700 10 client.27271 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:50.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.375+0000 7ffb8be00700 1 -- 172.21.15.28:0/2493669360 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x563a83c9f5f0 con 0x563a83dca680 2024-05-20T20:14:50.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.375+0000 7ffb8be00700 10 client.27271 handle_command_reply: tid=0 2024-05-20T20:14:50.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.375+0000 7ffbbbbf9b80 1 client.27271 shutdown 2024-05-20T20:14:50.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.375+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4b850 msgr2=0x563a83d54f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.375+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4b850 0x563a83d54f20 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7ffba40a5220 tx=0x7ffba40e2070 comp rx=0 tx=0).stop 2024-05-20T20:14:50.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 shutdown_connections 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563a83dca680 0x563a83dcca70 crc :-1 s=READY pgs=204 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563a83d4b850 0x563a83d54f20 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563a83d45900 0x563a83d544a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 --2- 172.21.15.28:0/2493669360 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563a83d4af30 0x563a83d549e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffb8be00700 0 client.27271 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 >> 172.21.15.28:0/2493669360 conn(0x563a83c73c60 msgr2=0x563a83d3fa60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:14:50.377 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:14:50.378 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "4fd8b4d4-dbd5-4365-a59e-3609381527bc", 2024-05-20T20:14:50.378 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:14:50.378 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:14:50.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 shutdown_connections 2024-05-20T20:14:50.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 1 -- 172.21.15.28:0/2493669360 wait complete. 2024-05-20T20:14:50.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.376+0000 7ffbbbbf9b80 20 client.27271 trim_cache size 0 max 16384 2024-05-20T20:14:50.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.378+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffb900b2bc0 msgr2=0x7ffb900b5080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.378+0000 7ffbb1200700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffb900b2bc0 0x7ffb900b5080 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7ffb9c009750 tx=0x7ffb9c00b580 comp rx=0 tx=0).stop 2024-05-20T20:14:50.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.378+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1143e0 msgr2=0x7ffbac0731b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:50.379 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.378+0000 7ffbb1200700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1143e0 0x7ffbac0731b0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7ffba4007d30 tx=0x7ffba400a340 comp rx=0 tx=0).stop 2024-05-20T20:14:50.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.380+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 shutdown_connections 2024-05-20T20:14:50.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.380+0000 7ffbb1200700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffb900b2bc0 0x7ffb900b5080 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.380+0000 7ffbb1200700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffbac1143e0 0x7ffbac0731b0 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.380+0000 7ffbb1200700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffbac1016c0 0x7ffbac06e710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.380+0000 7ffbb1200700 1 --2- 172.21.15.28:0/3845542295 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffbac101fe0 0x7ffbac06ec50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:50.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.380+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 >> 172.21.15.28:0/3845542295 conn(0x7ffbac078640 msgr2=0x7ffbac113b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:50.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.381+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 shutdown_connections 2024-05-20T20:14:50.385 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:50.381+0000 7ffbb1200700 1 -- 172.21.15.28:0/3845542295 wait complete. 2024-05-20T20:14:51.146 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:51 smithi028 ceph-mon[31178]: pgmap v285: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 322 MiB/s wr, 1.98k op/s 2024-05-20T20:14:51.146 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:51 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:14:51.146 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:51 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:14:51.146 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:51 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:14:51.146 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:51 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:14:51.295 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:14:51.427 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:51 smithi154 ceph-mon[36314]: pgmap v285: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 322 MiB/s wr, 1.98k op/s 2024-05-20T20:14:51.427 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:51 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:14:51.427 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:51 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:14:51.427 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:51 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:14:51.427 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:51 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:14:51.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:51 smithi033 ceph-mon[36179]: pgmap v285: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 322 MiB/s wr, 1.98k op/s 2024-05-20T20:14:51.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:51 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:14:51.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:51 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:14:51.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:51 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:14:51.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:51 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:14:52.044 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:53.167 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:53 smithi028 ceph-mon[31178]: pgmap v286: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 265 MiB/s wr, 1.49k op/s 2024-05-20T20:14:53.167 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:53 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:14:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:53 smithi154 ceph-mon[36314]: pgmap v286: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 265 MiB/s wr, 1.49k op/s 2024-05-20T20:14:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:53 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:14:53.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:53 smithi033 ceph-mon[36179]: pgmap v286: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 265 MiB/s wr, 1.49k op/s 2024-05-20T20:14:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:53 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:14:55.358 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:55 smithi028 ceph-mon[31178]: pgmap v287: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 359 MiB/s wr, 1.81k op/s 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.354+0000 7f1fdda00700 1 -- 172.21.15.28:0/2503592812 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8076090 msgr2=0x7f1fd8114200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.354+0000 7f1fdda00700 1 --2- 172.21.15.28:0/2503592812 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8076090 0x7f1fd8114200 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f1fc80098f0 tx=0x7f1fc8009c00 comp rx=0 tx=0).stop 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.354+0000 7f1fdda00700 1 -- 172.21.15.28:0/2503592812 shutdown_connections 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.354+0000 7f1fdda00700 1 --2- 172.21.15.28:0/2503592812 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1fd8114740 0x7f1fd8116b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.354+0000 7f1fdda00700 1 --2- 172.21.15.28:0/2503592812 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1fd8075770 0x7f1fd8075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.354+0000 7f1fdda00700 1 --2- 172.21.15.28:0/2503592812 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8076090 0x7f1fd8114200 secure :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f1fc80098f0 tx=0x7f1fc8009c00 comp rx=0 tx=0).stop 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.354+0000 7f1fdda00700 1 -- 172.21.15.28:0/2503592812 >> 172.21.15.28:0/2503592812 conn(0x7f1fd807a6f0 msgr2=0x7f1fd807ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.357+0000 7f1fdda00700 1 -- 172.21.15.28:0/2503592812 shutdown_connections 2024-05-20T20:14:55.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.358+0000 7f1fdda00700 1 -- 172.21.15.28:0/2503592812 wait complete. 2024-05-20T20:14:55.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.362+0000 7f1fdda00700 1 Processor -- start 2024-05-20T20:14:55.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.362+0000 7f1fdda00700 1 -- start start 2024-05-20T20:14:55.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.364+0000 7f1fdda00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8075770 0x7f1fd8074aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.364+0000 7f1fdda00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1fd8114740 0x7f1fd8074fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.364+0000 7f1fdda00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1fd806db30 0x7f1fd806dfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.364+0000 7f1fdda00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd8119ec0 con 0x7f1fd8114740 2024-05-20T20:14:55.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.364+0000 7f1fdda00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd8119d40 con 0x7f1fd8075770 2024-05-20T20:14:55.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.364+0000 7f1fdda00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd811a040 con 0x7f1fd806db30 2024-05-20T20:14:55.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.364+0000 7f1fd6a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8075770 0x7f1fd8074aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.365+0000 7f1fd6a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8075770 0x7f1fd8074aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58026/0 (socket says 172.21.15.28:58026) 2024-05-20T20:14:55.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.365+0000 7f1fd6a00700 1 -- 172.21.15.28:0/192443070 learned_addr learned my addr 172.21.15.28:0/192443070 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.365+0000 7f1fd7400700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1fd806db30 0x7f1fd806dfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.366+0000 7f1fd6a00700 1 -- 172.21.15.28:0/192443070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1fd806db30 msgr2=0x7f1fd806dfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.366+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1fd806db30 0x7f1fd806dfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.366+0000 7f1fd6a00700 1 -- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1fd8114740 msgr2=0x7f1fd8074fe0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:14:55.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.366+0000 7f1fd6000700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1fd8114740 0x7f1fd8074fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.366+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1fd8114740 0x7f1fd8074fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.366+0000 7f1fd6a00700 1 -- 172.21.15.28:0/192443070 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1fd806e7c0 con 0x7f1fd8075770 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.366+0000 7f1fd6000700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1fd8114740 0x7f1fd8074fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.367+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8075770 0x7f1fd8074aa0 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f1fd0007fa0 tx=0x7f1fd000d3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.367+0000 7f1fcf400700 1 -- 172.21.15.28:0/192443070 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fd001a070 con 0x7f1fd8075770 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.367+0000 7f1fcf400700 1 -- 172.21.15.28:0/192443070 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1fd000dda0 con 0x7f1fd8075770 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.367+0000 7f1fcf400700 1 -- 172.21.15.28:0/192443070 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fd00146d0 con 0x7f1fd8075770 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.369+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1fd80725d0 con 0x7f1fd8075770 2024-05-20T20:14:55.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.369+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1fd8072a30 con 0x7f1fd8075770 2024-05-20T20:14:55.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.377+0000 7f1fe836ab80 1 Processor -- start 2024-05-20T20:14:55.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.380+0000 7f1fe836ab80 1 -- start start 2024-05-20T20:14:55.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.381+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfd2610 0x56170bfd29f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.381 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.381+0000 7f1fcf400700 1 -- 172.21.15.28:0/192443070 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1fd00159d0 con 0x7f1fd8075770 2024-05-20T20:14:55.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.385+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfccb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.386+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfcd140 0x56170bfcd5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.386 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.386+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56170bf2ae40 con 0x56170bfd2610 2024-05-20T20:14:55.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.386+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56170bf1a9e0 con 0x56170bfd2f30 2024-05-20T20:14:55.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.386+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56170bf1ea20 con 0x56170bfcd140 2024-05-20T20:14:55.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.387+0000 7f1fd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfcd140 0x56170bfcd5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.387+0000 7f1fd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfcd140 0x56170bfcd5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52296/0 (socket says 172.21.15.28:52296) 2024-05-20T20:14:55.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.387+0000 7f1fd6000700 1 -- 172.21.15.28:0/634267146 learned_addr learned my addr 172.21.15.28:0/634267146 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.388+0000 7f1fd7400700 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfd2610 0x56170bfd29f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.388+0000 7f1fd6000700 1 -- 172.21.15.28:0/634267146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 msgr2=0x56170bfccb20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:14:55.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.389+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfccb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.389+0000 7f1fd6000700 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfccb20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.389+0000 7f1fd6000700 1 -- 172.21.15.28:0/634267146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfd2610 msgr2=0x56170bfd29f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.389+0000 7f1fd6000700 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfd2610 0x56170bfd29f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.389+0000 7f1fd6000700 1 -- 172.21.15.28:0/634267146 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56170bee3420 con 0x56170bfcd140 2024-05-20T20:14:55.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.390+0000 7f1fd6000700 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfcd140 0x56170bfcd5c0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f1fc80098f0 tx=0x7f1fc8009590 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cefdf6314a67e39f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.392+0000 7f1fcf400700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1fbc0b2b30 0x7f1fbc0b4ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.392+0000 7f1fcf400700 1 -- 172.21.15.28:0/192443070 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1fd00e1ff0 con 0x7f1fd8075770 2024-05-20T20:14:55.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.393+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfccb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:55.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.393+0000 7f1fc3e00700 1 -- 172.21.15.28:0/634267146 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fc8005040 con 0x56170bfcd140 2024-05-20T20:14:55.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.393+0000 7f1fc3e00700 1 -- 172.21.15.28:0/634267146 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1fc8005690 con 0x56170bfcd140 2024-05-20T20:14:55.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.393+0000 7f1fc3e00700 1 -- 172.21.15.28:0/634267146 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fc800b870 con 0x56170bfcd140 2024-05-20T20:14:55.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.393+0000 7f1fd7400700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1fbc0b2b30 0x7f1fbc0b4ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.395+0000 7f1fe836ab80 1 -- 172.21.15.28:0/634267146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfcd140 msgr2=0x56170bfcd5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.395+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfcd140 0x56170bfcd5c0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f1fc80098f0 tx=0x7f1fc8009590 comp rx=0 tx=0).stop 2024-05-20T20:14:55.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fe836ab80 1 -- 172.21.15.28:0/634267146 shutdown_connections 2024-05-20T20:14:55.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfcd140 0x56170bfcd5c0 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfd2610 0x56170bfd29f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/634267146 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfccb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fe836ab80 1 -- 172.21.15.28:0/634267146 >> 172.21.15.28:0/634267146 conn(0x56170befabe0 msgr2=0x56170bfc7690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fe836ab80 1 -- 172.21.15.28:0/634267146 shutdown_connections 2024-05-20T20:14:55.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fd7400700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1fbc0b2b30 0x7f1fbc0b4ff0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f1fd806afe0 tx=0x7f1fc4009110 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.397+0000 7f1fe836ab80 1 -- 172.21.15.28:0/634267146 wait complete. 2024-05-20T20:14:55.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.400+0000 7f1fe836ab80 1 -- start start 2024-05-20T20:14:55.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.401+0000 7f1fe836ab80 1 Processor -- start 2024-05-20T20:14:55.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.402+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfd8cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.402+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd6060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.402+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfd65a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.402+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56170be20fa0 con 0x56170bfcd140 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56170bebb760 con 0x56170bfd2f30 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56170bfd9980 con 0x56170bfd2610 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd6060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd6060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52302/0 (socket says 172.21.15.28:52302) 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd6000700 1 -- 172.21.15.28:0/833620933 learned_addr learned my addr 172.21.15.28:0/833620933 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd7400700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfd65a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd7400700 1 -- 172.21.15.28:0/833620933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 msgr2=0x56170bfd6060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd7400700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd6060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd7400700 1 -- 172.21.15.28:0/833620933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 msgr2=0x56170bfd8cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfd8cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fe836ab80 10 client.0 _ll_register_callbacks cb 0x56170bec4c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd7400700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfd8cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd7400700 1 -- 172.21.15.28:0/833620933 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56170be6a8d0 con 0x56170bfd2f30 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfd8cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.403+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1fc8005690 con 0x56170bfd2f30 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.404+0000 7f1fd7400700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfd65a0 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f1fc40146c0 tx=0x7f1fc4004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b1902c1d1a425ab5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.404+0000 7f1fc0c00700 10 client.27368 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.404+0000 7f1fc0c00700 1 -- 172.21.15.28:0/833620933 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fc402f050 con 0x56170bfd2f30 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.404+0000 7f1fc0c00700 1 -- 172.21.15.28:0/833620933 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1fc4021600 con 0x56170bfd2f30 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fc0c00700 1 -- 172.21.15.28:0/833620933 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1fc4025070 con 0x56170bfd2f30 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 10 client.27368 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x56170beeb060 con 0x56170bfd2f30 2024-05-20T20:14:55.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fc0c00700 1 -- 172.21.15.28:0/833620933 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1fc4020070 con 0x56170bfd2f30 2024-05-20T20:14:55.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 10 client.27368 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:55.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 10 client.27368 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:55.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 20 client.27368 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170bfe6120 0x56170bfe8590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 4 client.27368 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.405+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56170bf2cd90 con 0x56170bfe6120 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.406+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170bfe6120 0x56170bfe8590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.406+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170bfe6120 0x56170bfe8590 crc :-1 s=READY pgs=205 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e970d1f9b6132035 server_cookie=b83ebb595c03be8d in_seq=0 out_seq=0 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.406+0000 7f1fc0c00700 10 client.27368 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.407+0000 7f1fc0c00700 1 -- 172.21.15.28:0/833620933 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56170bf2cd90 con 0x56170bfe6120 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.407+0000 7f1fc0c00700 10 client.27368 handle_command_reply: tid=0 2024-05-20T20:14:55.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.407+0000 7f1fe836ab80 1 client.27368 shutdown 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 msgr2=0x56170bfd65a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfd65a0 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f1fc40146c0 tx=0x7f1fc4004750 comp rx=0 tx=0).stop 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 shutdown_connections 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170bfe6120 0x56170bfe8590 crc :-1 s=READY pgs=205 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfd65a0 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfd8cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/833620933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd6060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 >> 172.21.15.28:0/833620933 conn(0x56170befabe0 msgr2=0x56170bfc6d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.409+0000 7f1fc0c00700 0 client.27368 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.410+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 shutdown_connections 2024-05-20T20:14:55.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.410+0000 7f1fe836ab80 1 -- 172.21.15.28:0/833620933 wait complete. 2024-05-20T20:14:55.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.410+0000 7f1fe836ab80 20 client.27368 trim_cache size 0 max 16384 2024-05-20T20:14:55.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.480+0000 7f1fe836ab80 1 Processor -- start 2024-05-20T20:14:55.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.480+0000 7f1fe836ab80 1 -- start start 2024-05-20T20:14:55.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.480+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfccb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.481+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd5060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.481+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfc74f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.481+0000 7f1fd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.481+0000 7f1fd6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd5060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52318/0 (socket says 172.21.15.28:52318) 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.481+0000 7f1fd6000700 1 -- 172.21.15.28:0/2912697929 learned_addr learned my addr 172.21.15.28:0/2912697929 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.481+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd81bce40 con 0x56170bfcd140 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.482+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd81be980 con 0x56170bfd2f30 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.482+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd81bd510 con 0x56170bfd2610 2024-05-20T20:14:55.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.482+0000 7f1fd7400700 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfc74f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.484+0000 7f1fd6000700 1 -- 172.21.15.28:0/2912697929 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 msgr2=0x56170bfc74f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.484+0000 7f1fd6000700 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfc74f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.484+0000 7f1fd6000700 1 -- 172.21.15.28:0/2912697929 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 msgr2=0x56170bfccb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.485+0000 7f1fd6000700 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfccb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.485+0000 7f1fd6000700 1 -- 172.21.15.28:0/2912697929 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56170bfd8900 con 0x56170bfd2610 2024-05-20T20:14:55.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.485+0000 7f1fd7400700 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfc74f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:14:55.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.486+0000 7f1fd6000700 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd5060 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f1fc8009680 tx=0x7f1fc8004d80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=178b4f0697b5852e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.486+0000 7f1fc3400700 1 -- 172.21.15.28:0/2912697929 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fc8004df0 con 0x56170bfd2610 2024-05-20T20:14:55.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.486+0000 7f1fc3400700 1 -- 172.21.15.28:0/2912697929 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1fc8005440 con 0x56170bfd2610 2024-05-20T20:14:55.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.488+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 msgr2=0x56170bfd5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.488+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd5060 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f1fc8009680 tx=0x7f1fc8004d80 comp rx=0 tx=0).stop 2024-05-20T20:14:55.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.488+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 shutdown_connections 2024-05-20T20:14:55.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.488+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2f30 0x56170bfc74f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.488+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfccb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.489+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/2912697929 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2610 0x56170bfd5060 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.489+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 >> 172.21.15.28:0/2912697929 conn(0x56170befabe0 msgr2=0x56170bfd4950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.489+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 shutdown_connections 2024-05-20T20:14:55.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.489+0000 7f1fe836ab80 1 -- 172.21.15.28:0/2912697929 wait complete. 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.490+0000 7f1fe836ab80 1 -- start start 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.491+0000 7f1fe836ab80 1 Processor -- start 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.492+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfdec10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.492+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2610 0x56170bfdf150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.492+0000 7f1fe836ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2f30 0x56170bfdbbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.492+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd81be2b0 con 0x56170bfcd140 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.492+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd81bdbe0 con 0x56170bfd2610 2024-05-20T20:14:55.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.492+0000 7f1fe836ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1fd81bae00 con 0x56170bfd2f30 2024-05-20T20:14:55.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.492+0000 7f1fe836ab80 10 client.0 _ll_register_callbacks cb 0x56170bec4c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:55.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.494+0000 7f1fd7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2f30 0x56170bfdbbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.494+0000 7f1fd7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2f30 0x56170bfdbbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52350/0 (socket says 172.21.15.28:52350) 2024-05-20T20:14:55.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.494+0000 7f1fd7400700 1 -- 172.21.15.28:0/3837966607 learned_addr learned my addr 172.21.15.28:0/3837966607 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.495+0000 7f1fd7400700 1 -- 172.21.15.28:0/3837966607 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2610 msgr2=0x56170bfdf150 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:55.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.495+0000 7f1fd7400700 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2610 0x56170bfdf150 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.495+0000 7f1fd7400700 1 -- 172.21.15.28:0/3837966607 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 msgr2=0x56170bfdec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.495+0000 7f1fd7400700 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfdec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.495+0000 7f1fd7400700 1 -- 172.21.15.28:0/3837966607 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56170bfdc120 con 0x56170bfd2f30 2024-05-20T20:14:55.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.496+0000 7f1fd7400700 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2f30 0x56170bfdbbe0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f1fc4014f20 tx=0x7f1fc4016000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=49dd9fae42c784a9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f064ae00640 1 -- 172.21.15.28:0/3500538786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06441063e0 msgr2=0x7f064410dfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f064ae00640 1 --2- 172.21.15.28:0/3500538786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06441063e0 0x7f064410dfb0 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f063c00adb0 tx=0x7f063c02b7e0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f064ae00640 1 -- 172.21.15.28:0/3500538786 shutdown_connections 2024-05-20T20:14:55.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f064ae00640 1 --2- 172.21.15.28:0/3500538786 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06441063e0 0x7f064410dfb0 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f064ae00640 1 --2- 172.21.15.28:0/3500538786 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0644101cc0 0x7f0644105d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f064ae00640 1 --2- 172.21.15.28:0/3500538786 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0644101310 0x7f06441016f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f064ae00640 1 -- 172.21.15.28:0/3500538786 >> 172.21.15.28:0/3500538786 conn(0x7f0644077ee0 msgr2=0x7f06440ff260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.498+0000 7f064ae00640 1 -- 172.21.15.28:0/3500538786 shutdown_connections 2024-05-20T20:14:55.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.498+0000 7f064ae00640 1 -- 172.21.15.28:0/3500538786 wait complete. 2024-05-20T20:14:55.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.499+0000 7f064ae00640 1 Processor -- start 2024-05-20T20:14:55.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.499+0000 7f064ae00640 1 -- start start 2024-05-20T20:14:55.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.500+0000 7f064ae00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0644101310 0x7f0644196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.500+0000 7f064ae00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06441063e0 0x7f0644197300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.500+0000 7f0643e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0644101310 0x7f0644196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.500+0000 7f0643e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0644101310 0x7f0644196dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52364/0 (socket says 172.21.15.28:52364) 2024-05-20T20:14:55.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.500+0000 7f0643e00640 1 -- 172.21.15.28:0/2066959920 learned_addr learned my addr 172.21.15.28:0/2066959920 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.500+0000 7f064ae00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f064419b680 0x7f064419bb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.500+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0644110b20 con 0x7f06441063e0 2024-05-20T20:14:55.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f06441109a0 con 0x7f064419b680 2024-05-20T20:14:55.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0644110ca0 con 0x7f0644101310 2024-05-20T20:14:55.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f1fc0c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:14:55.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f1fc0c00700 1 -- 172.21.15.28:0/3837966607 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1fc4041060 con 0x56170bfd2f30 2024-05-20T20:14:55.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.497+0000 7f1fc0c00700 1 -- 172.21.15.28:0/3837966607 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f1fc402a850 con 0x56170bfd2f30 2024-05-20T20:14:55.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1fc8005440 con 0x56170bfd2f30 2024-05-20T20:14:55.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f0648e00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f064419b680 0x7f064419bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f0643e00640 1 -- 172.21.15.28:0/2066959920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f064419b680 msgr2=0x7f064419bb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f0643e00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f064419b680 0x7f064419bb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f0643e00640 1 -- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06441063e0 msgr2=0x7f0644197300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f0643e00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06441063e0 0x7f0644197300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.502+0000 7f0643e00640 1 -- 172.21.15.28:0/2066959920 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f064419c170 con 0x7f0644101310 2024-05-20T20:14:55.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.502+0000 7f0648e00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f064419b680 0x7f064419bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:14:55.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.501+0000 7f1fc0c00700 1 -- 172.21.15.28:0/3837966607 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1fc402f040 con 0x56170bfd2f30 2024-05-20T20:14:55.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.502+0000 7f1fe836ab80 10 client.27292 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:55.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.502+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56170c062470 con 0x56170bfd2f30 2024-05-20T20:14:55.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.502+0000 7f0643e00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0644101310 0x7f0644196dc0 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f063400e9f0 tx=0x7f063400eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f0640c00640 1 -- 172.21.15.28:0/2066959920 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f063400cd80 con 0x7f0644101310 2024-05-20T20:14:55.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f0640c00640 1 -- 172.21.15.28:0/2066959920 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f06340108e0 con 0x7f0644101310 2024-05-20T20:14:55.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f0640c00640 1 -- 172.21.15.28:0/2066959920 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0634018980 con 0x7f0644101310 2024-05-20T20:14:55.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f064419c460 con 0x7f0644101310 2024-05-20T20:14:55.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f06441a5e10 con 0x7f0644101310 2024-05-20T20:14:55.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.502+0000 7f1fc0c00700 1 -- 172.21.15.28:0/3837966607 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1fc8005440 con 0x56170bfd2f30 2024-05-20T20:14:55.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f1fc0c00700 1 -- 172.21.15.28:0/3837966607 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1fc403c8f0 con 0x56170bfd2f30 2024-05-20T20:14:55.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f1fe836ab80 10 client.27292 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:55.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f1fe836ab80 10 client.27292 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:55.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f1fe836ab80 20 client.27292 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:55.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170c062b80 0x56170c065000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f1fe836ab80 4 client.27292 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:14:55.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.503+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x56170bf26570 con 0x56170c062b80 2024-05-20T20:14:55.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.504+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170c062b80 0x56170c065000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.504+0000 7f1fc0c00700 10 client.27292 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.504+0000 7f1fd6a00700 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170c062b80 0x56170c065000 crc :-1 s=READY pgs=206 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=db65ed2813d5b8bf server_cookie=fcd95fa652cbaf20 in_seq=0 out_seq=0 2024-05-20T20:14:55.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.504+0000 7f1fc0c00700 1 -- 172.21.15.28:0/3837966607 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x56170bf26570 con 0x56170c062b80 2024-05-20T20:14:55.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.504+0000 7f1fc0c00700 10 client.27292 handle_command_reply: tid=0 2024-05-20T20:14:55.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.505+0000 7f1fe836ab80 1 client.27292 shutdown 2024-05-20T20:14:55.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.510+0000 7f064dbf6740 1 Processor -- start 2024-05-20T20:14:55.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.510+0000 7f064dbf6740 1 -- start start 2024-05-20T20:14:55.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.510+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674ec70 0x5611a674f050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.511+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a6759910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.514+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 0x5611a675c240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.514+0000 7f064dbf6740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5611a66bfb90 con 0x5611a6759e50 2024-05-20T20:14:55.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.514+0000 7f064dbf6740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5611a66af730 con 0x5611a674f590 2024-05-20T20:14:55.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.514+0000 7f064dbf6740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5611a66b3770 con 0x5611a674ec70 2024-05-20T20:14:55.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.515+0000 7f0648e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674ec70 0x5611a674f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.515+0000 7f0648e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674ec70 0x5611a674f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52380/0 (socket says 172.21.15.28:52380) 2024-05-20T20:14:55.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.515+0000 7f0648e00640 1 -- 172.21.15.28:0/2027577774 learned_addr learned my addr 172.21.15.28:0/2027577774 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.515+0000 7f0640c00640 1 -- 172.21.15.28:0/2066959920 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0634010430 con 0x7f0644101310 2024-05-20T20:14:55.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f0648e00640 1 -- 172.21.15.28:0/2027577774 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 msgr2=0x5611a6759910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.509+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2f30 msgr2=0x56170bfdbbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.510+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2f30 0x56170bfdbbe0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f1fc4014f20 tx=0x7f1fc4016000 comp rx=0 tx=0).stop 2024-05-20T20:14:55.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 shutdown_connections 2024-05-20T20:14:55.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56170c062b80 0x56170c065000 crc :-1 s=READY pgs=206 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56170bfd2f30 0x56170bfdbbe0 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56170bfcd140 0x56170bfdec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f1fe836ab80 1 --2- 172.21.15.28:0/3837966607 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56170bfd2610 0x56170bfdf150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 >> 172.21.15.28:0/3837966607 conn(0x56170befabe0 msgr2=0x56170bfd51e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f1fc0c00700 0 client.27292 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.520 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f0648e00640 1 --2- 172.21.15.28:0/2027577774 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a6759910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f0648e00640 1 -- 172.21.15.28:0/2027577774 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 msgr2=0x5611a675c240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f0648e00640 1 --2- 172.21.15.28:0/2027577774 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 0x5611a675c240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.516+0000 7f0648e00640 1 -- 172.21.15.28:0/2027577774 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5611a66488b0 con 0x5611a674ec70 2024-05-20T20:14:55.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.517+0000 7f0648e00640 1 --2- 172.21.15.28:0/2027577774 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674ec70 0x5611a674f050 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f063c00ab10 tx=0x7f063c031000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7489af2924705ca2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.517+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 shutdown_connections 2024-05-20T20:14:55.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.517+0000 7f1fe836ab80 1 -- 172.21.15.28:0/3837966607 wait complete. 2024-05-20T20:14:55.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.517+0000 7f0639600640 1 -- 172.21.15.28:0/2027577774 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f063c003970 con 0x5611a674ec70 2024-05-20T20:14:55.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.519+0000 7f0639600640 1 -- 172.21.15.28:0/2027577774 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f063c032400 con 0x5611a674ec70 2024-05-20T20:14:55.524 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.517+0000 7f1fe836ab80 20 client.27292 trim_cache size 0 max 16384 2024-05-20T20:14:55.524 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:14:55.524 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:14:55.524 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:14:55.525 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:14:55.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.518+0000 7f0640c00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06280b2ba0 0x7f06280b5060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.519+0000 7f0640c00640 1 -- 172.21.15.28:0/2066959920 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f06340e4da0 con 0x7f0644101310 2024-05-20T20:14:55.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f064dbf6740 1 -- 172.21.15.28:0/2027577774 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674ec70 msgr2=0x5611a674f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2027577774 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674ec70 0x5611a674f050 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f063c00ab10 tx=0x7f063c031000 comp rx=0 tx=0).stop 2024-05-20T20:14:55.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.520+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1fbc0b2b30 msgr2=0x7f1fbc0b4ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.520+0000 7f1fdda00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1fbc0b2b30 0x7f1fbc0b4ff0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f1fd806afe0 tx=0x7f1fc4009110 comp rx=0 tx=0).stop 2024-05-20T20:14:55.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.520+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8075770 msgr2=0x7f1fd8074aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.520+0000 7f1fdda00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8075770 0x7f1fd8074aa0 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f1fd0007fa0 tx=0x7f1fd000d3b0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f0643e00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06280b2ba0 0x7f06280b5060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f064dbf6740 1 -- 172.21.15.28:0/2027577774 shutdown_connections 2024-05-20T20:14:55.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2027577774 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a6759910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2027577774 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 0x5611a675c240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2027577774 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674ec70 0x5611a674f050 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f064dbf6740 1 -- 172.21.15.28:0/2027577774 >> 172.21.15.28:0/2027577774 conn(0x5611a674b180 msgr2=0x5611a674b560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 shutdown_connections 2024-05-20T20:14:55.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f1fdda00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1fd806db30 0x7f1fd806dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f1fdda00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1fbc0b2b30 0x7f1fbc0b4ff0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f1fdda00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1fd8075770 0x7f1fd8074aa0 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f1fdda00700 1 --2- 172.21.15.28:0/192443070 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1fd8114740 0x7f1fd8074fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.521+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 >> 172.21.15.28:0/192443070 conn(0x7f1fd807a6f0 msgr2=0x7f1fd8112c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.522+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 shutdown_connections 2024-05-20T20:14:55.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.522+0000 7f064dbf6740 1 -- 172.21.15.28:0/2027577774 shutdown_connections 2024-05-20T20:14:55.528 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.522+0000 7f064dbf6740 1 -- 172.21.15.28:0/2027577774 wait complete. 2024-05-20T20:14:55.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:14:55.522+0000 7f1fdda00700 1 -- 172.21.15.28:0/192443070 wait complete. 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.523+0000 7f064dbf6740 1 -- start start 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.523+0000 7f064dbf6740 1 Processor -- start 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.524+0000 7f0643e00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06280b2ba0 0x7f06280b5060 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x5611a6750210 tx=0x7f0634018420 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.524+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674ec70 0x5611a6759700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a67569f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 0x5611a6756f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f0643400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a67569f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f064dbf6740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5611a6395790 con 0x5611a674ec70 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f064dbf6740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5611a6657bc0 con 0x5611a674f590 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f064dbf6740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5611a675ca70 con 0x5611a6759e50 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f0643400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a67569f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58098/0 (socket says 172.21.15.28:58098) 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f0643400640 1 -- 172.21.15.28:0/2793418881 learned_addr learned my addr 172.21.15.28:0/2793418881 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.525+0000 7f064dbf6740 10 client.0 _ll_register_callbacks cb 0x5611a6635400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.526+0000 7f0648e00640 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674ec70 0x5611a6759700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.526+0000 7f0643400640 1 -- 172.21.15.28:0/2793418881 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 msgr2=0x5611a6756f30 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.526+0000 7f0643400640 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 0x5611a6756f30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.526+0000 7f0643400640 1 -- 172.21.15.28:0/2793418881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674ec70 msgr2=0x5611a6759700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.526+0000 7f0643400640 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674ec70 0x5611a6759700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.526+0000 7f0643400640 1 -- 172.21.15.28:0/2793418881 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5611a63c4210 con 0x5611a674f590 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.526+0000 7f062e000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:55.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.527+0000 7f0643400640 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a67569f0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f0630009dc0 tx=0x7f063000b4b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=de41fa4a0229abdc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.527+0000 7f062e000640 1 -- 172.21.15.28:0/2793418881 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0630015020 con 0x5611a674f590 2024-05-20T20:14:55.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.527+0000 7f062e000640 1 -- 172.21.15.28:0/2793418881 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f0630011420 con 0x5611a674f590 2024-05-20T20:14:55.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.527+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f063c032400 con 0x5611a674f590 2024-05-20T20:14:55.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.536+0000 7f062e000640 1 -- 172.21.15.28:0/2793418881 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0630020050 con 0x5611a674f590 2024-05-20T20:14:55.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.537+0000 7f064dbf6740 10 client.27377 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:55.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.537+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5611a6490620 con 0x5611a674f590 2024-05-20T20:14:55.538 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.538+0000 7f062e000640 1 -- 172.21.15.28:0/2793418881 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f063c032400 con 0x5611a674f590 2024-05-20T20:14:55.540 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.541+0000 7f062e000640 1 -- 172.21.15.28:0/2793418881 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0630011660 con 0x5611a674f590 2024-05-20T20:14:55.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.541+0000 7f064dbf6740 10 client.27377 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:55.541 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.541+0000 7f064dbf6740 10 client.27377 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:55.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.541+0000 7f064dbf6740 20 client.27377 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:55.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.541+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a6772490 0x5611a6774910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.541+0000 7f064dbf6740 4 client.27377 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:14:55.542 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.541+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5611a66c1ae0 con 0x5611a6772490 2024-05-20T20:14:55.545 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.546+0000 7f0648e00640 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a6772490 0x5611a6774910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.547 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.547+0000 7f062e000640 10 client.27377 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.550 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.551+0000 7f0648e00640 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a6772490 0x5611a6774910 crc :-1 s=READY pgs=207 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=aa8db372cfcc32bb server_cookie=838b84c1eb45b6e8 in_seq=0 out_seq=0 2024-05-20T20:14:55.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.552+0000 7f062e000640 1 -- 172.21.15.28:0/2793418881 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5611a66c1ae0 con 0x5611a6772490 2024-05-20T20:14:55.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.552+0000 7f062e000640 10 client.27377 handle_command_reply: tid=0 2024-05-20T20:14:55.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.552+0000 7f064dbf6740 1 client.27377 shutdown 2024-05-20T20:14:55.552 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.552+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 msgr2=0x5611a67569f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.552+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a67569f0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f0630009dc0 tx=0x7f063000b4b0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.553+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 shutdown_connections 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.553+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a6772490 0x5611a6774910 crc :-1 s=READY pgs=207 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.553+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674f590 0x5611a67569f0 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.553+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 0x5611a6756f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.553+0000 7f064dbf6740 1 --2- 172.21.15.28:0/2793418881 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674ec70 0x5611a6759700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.553+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 >> 172.21.15.28:0/2793418881 conn(0x5611a674b180 msgr2=0x5611a6750eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.553 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.553+0000 7f062e000640 0 client.27377 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.559+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 shutdown_connections 2024-05-20T20:14:55.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.560+0000 7f064dbf6740 1 -- 172.21.15.28:0/2793418881 wait complete. 2024-05-20T20:14:55.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.561+0000 7f064dbf6740 20 client.27377 trim_cache size 0 max 16384 2024-05-20T20:14:55.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:55 smithi154 ceph-mon[36314]: pgmap v287: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 359 MiB/s wr, 1.81k op/s 2024-05-20T20:14:55.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:55 smithi033 ceph-mon[36179]: pgmap v287: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 359 MiB/s wr, 1.81k op/s 2024-05-20T20:14:55.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.641+0000 7f064dbf6740 1 Processor -- start 2024-05-20T20:14:55.641 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.642+0000 7f064dbf6740 1 -- start start 2024-05-20T20:14:55.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.643+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676e3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.643+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674f590 0x5611a6752140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.643+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 0x5611a674a030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.643+0000 7f064dbf6740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f06441ac280 con 0x5611a6759e50 2024-05-20T20:14:55.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.643+0000 7f064dbf6740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f06441addc0 con 0x5611a674ec70 2024-05-20T20:14:55.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.643+0000 7f064dbf6740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f06441ac950 con 0x5611a674f590 2024-05-20T20:14:55.644 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.644+0000 7f0648e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676e3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.665+0000 7f0648e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676e3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58108/0 (socket says 172.21.15.28:58108) 2024-05-20T20:14:55.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.665+0000 7f0648e00640 1 -- 172.21.15.28:0/627141765 learned_addr learned my addr 172.21.15.28:0/627141765 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.665 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.665+0000 7f0643400640 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674f590 0x5611a6752140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.680 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.677+0000 7f0648e00640 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 0x5611a674a030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.677+0000 7f0648e00640 1 -- 172.21.15.28:0/627141765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674f590 msgr2=0x5611a6752140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.677+0000 7f0648e00640 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674f590 0x5611a6752140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.677+0000 7f0648e00640 1 -- 172.21.15.28:0/627141765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 msgr2=0x5611a674a030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.677+0000 7f0648e00640 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 0x5611a674a030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.677+0000 7f0648e00640 1 -- 172.21.15.28:0/627141765 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5611a674f380 con 0x5611a674ec70 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.679+0000 7f0648e00640 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676e3e0 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f063c004af0 tx=0x7f063c030f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c2c844b74910da2f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.680+0000 7f0638c00640 1 -- 172.21.15.28:0/627141765 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f063c003b70 con 0x5611a674ec70 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.680+0000 7f0638c00640 1 -- 172.21.15.28:0/627141765 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x5611a66c1ae0 con 0x5611a674ec70 2024-05-20T20:14:55.681 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.680+0000 7f0638c00640 1 -- 172.21.15.28:0/627141765 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f063c030510 con 0x5611a674ec70 2024-05-20T20:14:55.683 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.683+0000 7f064dbf6740 1 -- 172.21.15.28:0/627141765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 msgr2=0x5611a676e3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.683 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.683+0000 7f064dbf6740 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676e3e0 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f063c004af0 tx=0x7f063c030f10 comp rx=0 tx=0).stop 2024-05-20T20:14:55.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.699+0000 7f064dbf6740 1 -- 172.21.15.28:0/627141765 shutdown_connections 2024-05-20T20:14:55.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.699+0000 7f064dbf6740 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a6759e50 0x5611a674a030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.699+0000 7f064dbf6740 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a674f590 0x5611a6752140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.699+0000 7f064dbf6740 1 --2- 172.21.15.28:0/627141765 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676e3e0 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.699+0000 7f064dbf6740 1 -- 172.21.15.28:0/627141765 >> 172.21.15.28:0/627141765 conn(0x5611a674b180 msgr2=0x5611a6749920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.700 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.699+0000 7f064dbf6740 1 -- 172.21.15.28:0/627141765 shutdown_connections 2024-05-20T20:14:55.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.705+0000 7f064dbf6740 1 -- 172.21.15.28:0/627141765 wait complete. 2024-05-20T20:14:55.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.705+0000 7f064dbf6740 1 -- start start 2024-05-20T20:14:55.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.705+0000 7f064dbf6740 1 Processor -- start 2024-05-20T20:14:55.712 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.710+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676a5d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f0648e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676a5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f0648e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676a5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58118/0 (socket says 172.21.15.28:58118) 2024-05-20T20:14:55.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f064dbf6740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674f590 0x5611a676ab10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f0648e00640 1 -- 172.21.15.28:0/1822641521 learned_addr learned my addr 172.21.15.28:0/1822641521 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:14:55.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f064dbf6740 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 0x5611a6752e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.713 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f06441ad6f0 con 0x5611a674f590 2024-05-20T20:14:55.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f06441ad020 con 0x5611a674ec70 2024-05-20T20:14:55.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.713+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f06441aa260 con 0x5611a6759e50 2024-05-20T20:14:55.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.714+0000 7f064dbf6740 10 client.0 _ll_register_callbacks cb 0x7f0630006e90 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:14:55.714 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.715+0000 7f0648e00640 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 0x5611a6752e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.715+0000 7f0648e00640 1 -- 172.21.15.28:0/1822641521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 msgr2=0x5611a6752e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.715+0000 7f0648e00640 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 0x5611a6752e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.715+0000 7f0648e00640 1 -- 172.21.15.28:0/1822641521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674f590 msgr2=0x5611a676ab10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:14:55.715 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.716+0000 7f0648e00640 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674f590 0x5611a676ab10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.716 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.716+0000 7f0648e00640 1 -- 172.21.15.28:0/1822641521 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5611a67533a0 con 0x5611a674ec70 2024-05-20T20:14:55.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.725+0000 7f0648e00640 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676a5d0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f063c00a730 tx=0x7f063c053890 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4e0c0837b7b01d32 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:14:55.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.726+0000 7f062e000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:14:55.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.726+0000 7f062e000640 1 -- 172.21.15.28:0/1822641521 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f063c053c40 con 0x5611a674ec70 2024-05-20T20:14:55.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.726+0000 7f062e000640 1 -- 172.21.15.28:0/1822641521 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f063c053400 con 0x5611a674ec70 2024-05-20T20:14:55.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.726+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x5611a66c1ae0 con 0x5611a674ec70 2024-05-20T20:14:55.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.726+0000 7f062e000640 1 -- 172.21.15.28:0/1822641521 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f063c053600 con 0x5611a674ec70 2024-05-20T20:14:55.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.726+0000 7f064dbf6740 10 client.27383 fetch_fsmap learned FSMap version 29 2024-05-20T20:14:55.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.726+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5611a67dc8a0 con 0x5611a674ec70 2024-05-20T20:14:55.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f062e000640 1 -- 172.21.15.28:0/1822641521 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x5611a66c1ae0 con 0x5611a674ec70 2024-05-20T20:14:55.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f062e000640 1 -- 172.21.15.28:0/1822641521 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f063c030910 con 0x5611a674ec70 2024-05-20T20:14:55.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f064dbf6740 10 client.27383 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:14:55.732 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f064dbf6740 10 client.27383 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:14:55.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f064dbf6740 20 client.27383 populate_metadata read hostname 'smithi028' 2024-05-20T20:14:55.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f064dbf6740 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a67dca70 0x5611a67dee60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:14:55.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f064dbf6740 4 client.27383 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:14:55.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.730+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5611a66bb2c0 con 0x5611a67dca70 2024-05-20T20:14:55.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.732+0000 7f0643400640 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a67dca70 0x5611a67dee60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:14:55.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.733+0000 7f062e000640 10 client.27383 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.733+0000 7f0643400640 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a67dca70 0x5611a67dee60 crc :-1 s=READY pgs=208 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=62f7f5fb7e8e7af6 server_cookie=6ab04da88eb561e3 in_seq=0 out_seq=0 2024-05-20T20:14:55.766 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.766+0000 7f062e000640 1 -- 172.21.15.28:0/1822641521 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5611a66bb2c0 con 0x5611a67dca70 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.766+0000 7f062e000640 10 client.27383 handle_command_reply: tid=0 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.766+0000 7f064dbf6740 1 client.27383 shutdown 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.766+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 msgr2=0x5611a676a5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.766+0000 7f064dbf6740 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676a5d0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f063c00a730 tx=0x7f063c053890 comp rx=0 tx=0).stop 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.767+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 shutdown_connections 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.767+0000 7f064dbf6740 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5611a67dca70 0x5611a67dee60 crc :-1 s=READY pgs=208 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.767+0000 7f062e000640 0 client.27383 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.767+0000 7f064dbf6740 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5611a6759e50 0x5611a6752e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.767+0000 7f064dbf6740 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5611a674f590 0x5611a676ab10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.767+0000 7f064dbf6740 1 --2- 172.21.15.28:0/1822641521 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5611a674ec70 0x5611a676a5d0 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.767 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.767+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 >> 172.21.15.28:0/1822641521 conn(0x5611a674b180 msgr2=0x5611a674a1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.786+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 shutdown_connections 2024-05-20T20:14:55.791 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:14:55.791 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:14:55.791 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:14:55.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.786+0000 7f064dbf6740 1 -- 172.21.15.28:0/1822641521 wait complete. 2024-05-20T20:14:55.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.786+0000 7f064dbf6740 20 client.27383 trim_cache size 0 max 16384 2024-05-20T20:14:55.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.790+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06280b2ba0 msgr2=0x7f06280b5060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.790+0000 7f064ae00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06280b2ba0 0x7f06280b5060 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x5611a6750210 tx=0x7f0634018420 comp rx=0 tx=0).stop 2024-05-20T20:14:55.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.790+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0644101310 msgr2=0x7f0644196dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:14:55.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.790+0000 7f064ae00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0644101310 0x7f0644196dc0 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f063400e9f0 tx=0x7f063400eeb0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.792 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.791+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 shutdown_connections 2024-05-20T20:14:55.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.791+0000 7f064ae00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f064419b680 0x7f064419bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.791+0000 7f064ae00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f06280b2ba0 0x7f06280b5060 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.791+0000 7f064ae00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f06441063e0 0x7f0644197300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.791+0000 7f064ae00640 1 --2- 172.21.15.28:0/2066959920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0644101310 0x7f0644196dc0 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:14:55.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.791+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 >> 172.21.15.28:0/2066959920 conn(0x7f0644077ee0 msgr2=0x7f0644104120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:14:55.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.792+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 shutdown_connections 2024-05-20T20:14:55.793 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:14:55.792+0000 7f064ae00640 1 -- 172.21.15.28:0/2066959920 wait complete. 2024-05-20T20:14:57.084 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:14:57.085 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:14:57.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:57 smithi028 ceph-mon[31178]: pgmap v288: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 269 MiB/s wr, 1.20k op/s 2024-05-20T20:14:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:57 smithi154 ceph-mon[36314]: pgmap v288: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 269 MiB/s wr, 1.20k op/s 2024-05-20T20:14:57.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:57 smithi033 ceph-mon[36179]: pgmap v288: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 269 MiB/s wr, 1.20k op/s 2024-05-20T20:14:57.865 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:14:59.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:14:59 smithi028 ceph-mon[31178]: pgmap v289: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 274 MiB/s wr, 1.18k op/s 2024-05-20T20:14:59.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:14:59 smithi154 ceph-mon[36314]: pgmap v289: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 274 MiB/s wr, 1.18k op/s 2024-05-20T20:14:59.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:14:59 smithi033 ceph-mon[36179]: pgmap v289: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 274 MiB/s wr, 1.18k op/s 2024-05-20T20:15:00.504 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:00 smithi028 ceph-mon[31178]: pgmap v290: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 332 MiB/s wr, 1.37k op/s 2024-05-20T20:15:00.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:00 smithi154 ceph-mon[36314]: pgmap v290: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 332 MiB/s wr, 1.37k op/s 2024-05-20T20:15:00.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:00 smithi033 ceph-mon[36179]: pgmap v290: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 332 MiB/s wr, 1.37k op/s 2024-05-20T20:15:01.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.934+0000 7f7c63200700 1 -- 172.21.15.28:0/2779939375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c6410ae60 msgr2=0x7f7c64116150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:01.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.934+0000 7f7c63200700 1 --2- 172.21.15.28:0/2779939375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c6410ae60 0x7f7c64116150 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f7c500099c0 tx=0x7f7c50009cd0 comp rx=0 tx=0).stop 2024-05-20T20:15:01.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.935+0000 7f7c63200700 1 -- 172.21.15.28:0/2779939375 shutdown_connections 2024-05-20T20:15:01.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.935+0000 7f7c63200700 1 --2- 172.21.15.28:0/2779939375 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64116690 0x7f7c64118a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:01.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.935+0000 7f7c63200700 1 --2- 172.21.15.28:0/2779939375 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6410a540 0x7f7c6410a920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:01.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.935+0000 7f7c63200700 1 --2- 172.21.15.28:0/2779939375 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c6410ae60 0x7f7c64116150 secure :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f7c500099c0 tx=0x7f7c50009cd0 comp rx=0 tx=0).stop 2024-05-20T20:15:01.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.935+0000 7f7c63200700 1 -- 172.21.15.28:0/2779939375 >> 172.21.15.28:0/2779939375 conn(0x7f7c6406e520 msgr2=0x7f7c6406e930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:01.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.938+0000 7f7c63200700 1 -- 172.21.15.28:0/2779939375 shutdown_connections 2024-05-20T20:15:01.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.939+0000 7f7c63200700 1 -- 172.21.15.28:0/2779939375 wait complete. 2024-05-20T20:15:01.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.940+0000 7f7c63200700 1 Processor -- start 2024-05-20T20:15:01.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.941+0000 7f7c63200700 1 -- start start 2024-05-20T20:15:01.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.941+0000 7f7c63200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6410a540 0x7f7c64113290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:01.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.941+0000 7f7c63200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64116690 0x7f7c641137d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:01.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.941+0000 7f7c63200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64113d10 0x7f7c641aa620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:01.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.943+0000 7f7c63200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7c6411e160 con 0x7f7c64116690 2024-05-20T20:15:01.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.943+0000 7f7c61e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6410a540 0x7f7c64113290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:01.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.943+0000 7f7c63200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7c6411dfe0 con 0x7f7c6410a540 2024-05-20T20:15:01.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.943+0000 7f7c63200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7c6411e2e0 con 0x7f7c64113d10 2024-05-20T20:15:01.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.942+0000 7f7c62800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64113d10 0x7f7c641aa620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:01.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.944+0000 7f7c62800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64113d10 0x7f7c641aa620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:60984/0 (socket says 172.21.15.28:60984) 2024-05-20T20:15:01.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.944+0000 7f7c62800700 1 -- 172.21.15.28:0/1708225469 learned_addr learned my addr 172.21.15.28:0/1708225469 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:01.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.943+0000 7f7c61400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64116690 0x7f7c641137d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:01.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.945+0000 7f7c61e00700 1 -- 172.21.15.28:0/1708225469 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64113d10 msgr2=0x7f7c641aa620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:01.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.945+0000 7f7c61e00700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64113d10 0x7f7c641aa620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:01.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.945+0000 7f7c61e00700 1 -- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64116690 msgr2=0x7f7c641137d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:01.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.945+0000 7f7c61e00700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64116690 0x7f7c641137d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:01.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.945+0000 7f7c61e00700 1 -- 172.21.15.28:0/1708225469 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7c641aab60 con 0x7f7c6410a540 2024-05-20T20:15:01.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.946+0000 7f7c61e00700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6410a540 0x7f7c64113290 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f7c5400d950 tx=0x7f7c5400dd10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.948+0000 7f7c5aa00700 1 -- 172.21.15.28:0/1708225469 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c54017070 con 0x7f7c6410a540 2024-05-20T20:15:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.948+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7c641aae50 con 0x7f7c6410a540 2024-05-20T20:15:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.948+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7c641ab3b0 con 0x7f7c6410a540 2024-05-20T20:15:01.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.949+0000 7f7c5aa00700 1 -- 172.21.15.28:0/1708225469 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7c5400be10 con 0x7f7c6410a540 2024-05-20T20:15:01.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.949+0000 7f7c5aa00700 1 -- 172.21.15.28:0/1708225469 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c54017070 con 0x7f7c6410a540 2024-05-20T20:15:01.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.951+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7c44005330 con 0x7f7c6410a540 2024-05-20T20:15:01.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.954+0000 7f7c5aa00700 1 -- 172.21.15.28:0/1708225469 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7c5401a070 con 0x7f7c6410a540 2024-05-20T20:15:01.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.957+0000 7f7c5aa00700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c480b2ca0 0x7f7c480b5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:01.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.957+0000 7f7c5aa00700 1 -- 172.21.15.28:0/1708225469 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(87..87 src has 1..87) ==== 8542+0+0 (secure 0 0 0) 0x7f7c540e1680 con 0x7f7c6410a540 2024-05-20T20:15:01.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.961+0000 7f7c61400700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c480b2ca0 0x7f7c480b5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:01.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.961+0000 7f7c5aa00700 1 -- 172.21.15.28:0/1708225469 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7c540a8ff0 con 0x7f7c6410a540 2024-05-20T20:15:01.964 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:01.965+0000 7f7c61400700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c480b2ca0 0x7f7c480b5160 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f7c50000c00 tx=0x7f7c5001a040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:02.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.607+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f7c440058f0 con 0x7f7c6410a540 2024-05-20T20:15:02.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.609+0000 7f7c5aa00700 1 -- 172.21.15.28:0/1708225469 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f7c540b1070 con 0x7f7c6410a540 2024-05-20T20:15:02.611 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:15:02.612 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:15:02.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c480b2ca0 msgr2=0x7f7c480b5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c480b2ca0 0x7f7c480b5160 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f7c50000c00 tx=0x7f7c5001a040 comp rx=0 tx=0).stop 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6410a540 msgr2=0x7f7c64113290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6410a540 0x7f7c64113290 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f7c5400d950 tx=0x7f7c5400dd10 comp rx=0 tx=0).stop 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 shutdown_connections 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64113d10 0x7f7c641aa620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c480b2ca0 0x7f7c480b5160 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6410a540 0x7f7c64113290 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 --2- 172.21.15.28:0/1708225469 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64116690 0x7f7c641137d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:02.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.617+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 >> 172.21.15.28:0/1708225469 conn(0x7f7c6406e520 msgr2=0x7f7c64071ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:02.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.618+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 shutdown_connections 2024-05-20T20:15:02.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:02.618+0000 7f7c63200700 1 -- 172.21.15.28:0/1708225469 wait complete. 2024-05-20T20:15:02.620 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:15:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:03 smithi154 ceph-mon[36314]: pgmap v291: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 264 MiB/s wr, 1.03k op/s 2024-05-20T20:15:03.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:03 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1708225469' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:15:03.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:03 smithi033 ceph-mon[36179]: pgmap v291: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 264 MiB/s wr, 1.03k op/s 2024-05-20T20:15:03.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:03 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1708225469' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:15:03.518 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:03 smithi028 ceph-mon[31178]: pgmap v291: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 264 MiB/s wr, 1.03k op/s 2024-05-20T20:15:03.518 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:03 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1708225469' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:15:04.390 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:15:05.134 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:05.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:05 smithi154 ceph-mon[36314]: pgmap v292: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 347 MiB/s wr, 1.50k op/s 2024-05-20T20:15:05.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:05 smithi033 ceph-mon[36179]: pgmap v292: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 347 MiB/s wr, 1.50k op/s 2024-05-20T20:15:05.519 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:05 smithi028 ceph-mon[31178]: pgmap v292: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 347 MiB/s wr, 1.50k op/s 2024-05-20T20:15:07.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:07 smithi154 ceph-mon[36314]: pgmap v293: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 258 MiB/s wr, 1.20k op/s 2024-05-20T20:15:07.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:07 smithi033 ceph-mon[36179]: pgmap v293: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 258 MiB/s wr, 1.20k op/s 2024-05-20T20:15:07.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:07 smithi028 ceph-mon[31178]: pgmap v293: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 258 MiB/s wr, 1.20k op/s 2024-05-20T20:15:09.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.041+0000 7f36f5200640 1 -- 172.21.15.28:0/2565237431 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 msgr2=0x7f36f0109a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.042+0000 7f36f5200640 1 --2- 172.21.15.28:0/2565237431 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 0x7f36f0109a50 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f36e8009be0 tx=0x7f36e802a770 comp rx=0 tx=0).stop 2024-05-20T20:15:09.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.042+0000 7f36f5200640 1 -- 172.21.15.28:0/2565237431 shutdown_connections 2024-05-20T20:15:09.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.043+0000 7f36f5200640 1 --2- 172.21.15.28:0/2565237431 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f36f0111620 0x7f36f0109f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.043+0000 7f36f5200640 1 --2- 172.21.15.28:0/2565237431 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f36f011c230 0x7f36f011e620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.043+0000 7f36f5200640 1 --2- 172.21.15.28:0/2565237431 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 0x7f36f0109a50 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.044+0000 7f36f5200640 1 -- 172.21.15.28:0/2565237431 >> 172.21.15.28:0/2565237431 conn(0x7f36f006db20 msgr2=0x7f36f006df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.044+0000 7f36f5200640 1 -- 172.21.15.28:0/2565237431 shutdown_connections 2024-05-20T20:15:09.044 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.044+0000 7f36f5200640 1 -- 172.21.15.28:0/2565237431 wait complete. 2024-05-20T20:15:09.053 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.053+0000 7f36f5200640 1 Processor -- start 2024-05-20T20:15:09.059 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.055+0000 7f36f5200640 1 -- start start 2024-05-20T20:15:09.066 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.067+0000 7f36f5200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 0x7f36f01a8090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.067+0000 7f36f5200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f36f011c230 0x7f36f01a85d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.067 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.067+0000 7f36f5200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f36f01ac950 0x7f36f01ace00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.067+0000 7f36f5200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f36f0121b50 con 0x7f36f011c230 2024-05-20T20:15:09.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.068+0000 7f36ede00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 0x7f36f01a8090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.068+0000 7f36ede00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 0x7f36f01a8090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58882/0 (socket says 172.21.15.28:58882) 2024-05-20T20:15:09.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.068+0000 7f36f5200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f36f01219d0 con 0x7f36f01ac950 2024-05-20T20:15:09.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.068+0000 7f36f5200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f36f0121cd0 con 0x7f36f0111240 2024-05-20T20:15:09.068 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.068+0000 7f36ede00640 1 -- 172.21.15.28:0/3912411022 learned_addr learned my addr 172.21.15.28:0/3912411022 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.069 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.069+0000 7f36ee800640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f36f01ac950 0x7f36f01ace00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.069 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.069+0000 7f36ed400640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f36f011c230 0x7f36f01a85d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.070+0000 7f36ee800640 1 -- 172.21.15.28:0/3912411022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 msgr2=0x7f36f01a8090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.070+0000 7f36ee800640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 0x7f36f01a8090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.070+0000 7f36ee800640 1 -- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f36f011c230 msgr2=0x7f36f01a85d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.070+0000 7f36ee800640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f36f011c230 0x7f36f01a85d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.072 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.070+0000 7f36ee800640 1 -- 172.21.15.28:0/3912411022 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f36f01ad440 con 0x7f36f01ac950 2024-05-20T20:15:09.075 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.075+0000 7f36ee800640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f36f01ac950 0x7f36f01ace00 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7f36dc00e940 tx=0x7f36dc00ee00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.077+0000 7f36e6a00640 1 -- 172.21.15.28:0/3912411022 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36dc00ccc0 con 0x7f36f01ac950 2024-05-20T20:15:09.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.077+0000 7f36e6a00640 1 -- 172.21.15.28:0/3912411022 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f36dc0035f0 con 0x7f36f01ac950 2024-05-20T20:15:09.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.077+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f36f01ad730 con 0x7f36f01ac950 2024-05-20T20:15:09.079 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.077+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f36f01b4f20 con 0x7f36f01ac950 2024-05-20T20:15:09.080 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.081+0000 7f36e6a00640 1 -- 172.21.15.28:0/3912411022 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36dc00ce60 con 0x7f36f01ac950 2024-05-20T20:15:09.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.096+0000 7f36f807d740 1 Processor -- start 2024-05-20T20:15:09.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.103+0000 7f36e6a00640 1 -- 172.21.15.28:0/3912411022 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f36dc004020 con 0x7f36f01ac950 2024-05-20T20:15:09.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.104+0000 7f36f807d740 1 -- start start 2024-05-20T20:15:09.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.104+0000 7f36f807d740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd7b070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.104+0000 7f36f807d740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd85930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.106 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.106+0000 7f36ed400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd7b070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.106 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.106+0000 7f36ed400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd7b070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45698/0 (socket says 172.21.15.28:45698) 2024-05-20T20:15:09.110 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.109+0000 7f36e6a00640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f36d40b2c20 0x7f36d40b50e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.110+0000 7f36e6a00640 1 -- 172.21.15.28:0/3912411022 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f36dc0e26e0 con 0x7f36f01ac950 2024-05-20T20:15:09.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.111+0000 7f36f807d740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd88260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.111+0000 7f36ede00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd85930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.111+0000 7f36ede00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd85930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58896/0 (socket says 172.21.15.28:58896) 2024-05-20T20:15:09.111 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.111+0000 7f36ed400640 1 -- 172.21.15.28:0/2661237157 learned_addr learned my addr 172.21.15.28:0/2661237157 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.112 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.111+0000 7f36ee800640 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd88260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.114+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5616fccebb90 con 0x5616fcd7ac90 2024-05-20T20:15:09.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.114+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5616fccdb730 con 0x5616fcd85e70 2024-05-20T20:15:09.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.114+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5616fccdf770 con 0x5616fcd7b5b0 2024-05-20T20:15:09.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.115+0000 7f36ee800640 1 -- 172.21.15.28:0/2661237157 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 msgr2=0x5616fcd85930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.115+0000 7f36ee800640 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd85930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.115+0000 7f36ee800640 1 -- 172.21.15.28:0/2661237157 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 msgr2=0x5616fcd7b070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.115+0000 7f36ee800640 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd7b070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.115+0000 7f36ee800640 1 -- 172.21.15.28:0/2661237157 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5616fcc748b0 con 0x5616fcd85e70 2024-05-20T20:15:09.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.116+0000 7f36ee800640 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd88260 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f36dc0dfd90 tx=0x7f36dc0a9150 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4152f7e6d6d7e5bb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.117+0000 7f36ed400640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f36d40b2c20 0x7f36d40b50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.118+0000 7f36db400640 1 -- 172.21.15.28:0/2661237157 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36dc0a9430 con 0x5616fcd85e70 2024-05-20T20:15:09.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.118+0000 7f36db400640 1 -- 172.21.15.28:0/2661237157 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f36dc0aa020 con 0x5616fcd85e70 2024-05-20T20:15:09.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.118+0000 7f36db400640 1 -- 172.21.15.28:0/2661237157 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36dc0a5d40 con 0x5616fcd85e70 2024-05-20T20:15:09.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.121+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 msgr2=0x5616fcd88260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.121+0000 7f36f807d740 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd88260 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7f36dc0dfd90 tx=0x7f36dc0a9150 comp rx=0 tx=0).stop 2024-05-20T20:15:09.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.122+0000 7f36ed400640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f36d40b2c20 0x7f36d40b50e0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f36e0005ec0 tx=0x7f36e00046c0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.122+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 shutdown_connections 2024-05-20T20:15:09.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.122+0000 7f36f807d740 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd88260 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.122+0000 7f36f807d740 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd85930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.122+0000 7f36f807d740 1 --2- 172.21.15.28:0/2661237157 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd7b070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.122+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 >> 172.21.15.28:0/2661237157 conn(0x5616fcd771a0 msgr2=0x5616fcd77580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.123+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 shutdown_connections 2024-05-20T20:15:09.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.123+0000 7f36f807d740 1 -- 172.21.15.28:0/2661237157 wait complete. 2024-05-20T20:15:09.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.124+0000 7f36f807d740 1 -- start start 2024-05-20T20:15:09.125 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.126+0000 7f36f807d740 1 Processor -- start 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.128+0000 7f36f807d740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd857f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.128+0000 7f36f807d740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd82ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.128+0000 7f36f807d740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd83060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.128+0000 7f36f807d740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5616fc9c1790 con 0x5616fcd85e70 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.128+0000 7f36f807d740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5616fcc83bc0 con 0x5616fcd7ac90 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.128+0000 7f36f807d740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5616fcd88aa0 con 0x5616fcd7b5b0 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.128+0000 7f36f807d740 10 client.0 _ll_register_callbacks cb 0x5616fcc61400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.129+0000 7f36ed400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd82ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.129+0000 7f36ed400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd82ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58898/0 (socket says 172.21.15.28:58898) 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.129+0000 7f36ed400640 1 -- 172.21.15.28:0/1972994791 learned_addr learned my addr 172.21.15.28:0/1972994791 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.130+0000 7f36ede00640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd857f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.131+0000 7f36ee800640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd83060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.133+0000 7f36ed400640 1 -- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 msgr2=0x5616fcd857f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.133+0000 7f36ed400640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd857f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.133+0000 7f36ed400640 1 -- 172.21.15.28:0/1972994791 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 msgr2=0x5616fcd83060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.133+0000 7f36ed400640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd83060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.133+0000 7f36ed400640 1 -- 172.21.15.28:0/1972994791 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5616fc9f0210 con 0x5616fcd7b5b0 2024-05-20T20:15:09.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.133+0000 7f36ede00640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd857f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:15:09.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.134+0000 7f36ed400640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd82ae0 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f36e0005e90 tx=0x7f36e0004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=190e184df4834be6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.134+0000 7f36cfe00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:09.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.134+0000 7f36cfe00640 1 -- 172.21.15.28:0/1972994791 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36e0012940 con 0x5616fcd7b5b0 2024-05-20T20:15:09.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.135+0000 7f36cfe00640 1 -- 172.21.15.28:0/1972994791 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f36e0012ae0 con 0x5616fcd7b5b0 2024-05-20T20:15:09.135 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.135+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f36dc0aa020 con 0x5616fcd7b5b0 2024-05-20T20:15:09.136 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.136+0000 7f36cfe00640 1 -- 172.21.15.28:0/1972994791 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f36e002c450 con 0x5616fcd7b5b0 2024-05-20T20:15:09.136 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.137+0000 7f36f807d740 10 client.27319 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:09.136 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.137+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5616fcabc620 con 0x5616fcd7b5b0 2024-05-20T20:15:09.137 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.137+0000 7f36cfe00640 1 -- 172.21.15.28:0/1972994791 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f36dc0aa020 con 0x5616fcd7b5b0 2024-05-20T20:15:09.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.139+0000 7f36cfe00640 1 -- 172.21.15.28:0/1972994791 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f36e0012ce0 con 0x5616fcd7b5b0 2024-05-20T20:15:09.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.139+0000 7f36f807d740 10 client.27319 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:09.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.139+0000 7f36f807d740 10 client.27319 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:09.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.139+0000 7f36f807d740 20 client.27319 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:09.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.139+0000 7f36f807d740 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fcd95ce0 0x5616fcd98160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.139+0000 7f36f807d740 4 client.27319 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:09.139 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.139+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5616fccedae0 con 0x5616fcd95ce0 2024-05-20T20:15:09.141 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.141+0000 7f36ede00640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fcd95ce0 0x5616fcd98160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.144+0000 7f36cfe00640 10 client.27319 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:09.144 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.144+0000 7f36ede00640 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fcd95ce0 0x5616fcd98160 crc :-1 s=READY pgs=209 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=11cfa5ce67deed1d server_cookie=197b0c53d4e0e812 in_seq=0 out_seq=0 2024-05-20T20:15:09.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.145+0000 7f36cfe00640 1 -- 172.21.15.28:0/1972994791 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5616fccedae0 con 0x5616fcd95ce0 2024-05-20T20:15:09.145 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.146+0000 7f36cfe00640 10 client.27319 handle_command_reply: tid=0 2024-05-20T20:15:09.146 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.146+0000 7f36f807d740 1 client.27319 shutdown 2024-05-20T20:15:09.147 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.148+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 msgr2=0x5616fcd82ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.148+0000 7f36f807d740 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd82ae0 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f36e0005e90 tx=0x7f36e0004730 comp rx=0 tx=0).stop 2024-05-20T20:15:09.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.151+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 shutdown_connections 2024-05-20T20:15:09.152 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.151+0000 7f36f807d740 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fcd95ce0 0x5616fcd98160 crc :-1 s=READY pgs=209 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.152+0000 7f36cfe00640 0 client.27319 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.152+0000 7f36f807d740 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd83060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.152+0000 7f36f807d740 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd82ae0 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.152+0000 7f36f807d740 1 --2- 172.21.15.28:0/1972994791 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd857f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.152+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 >> 172.21.15.28:0/1972994791 conn(0x5616fcd771a0 msgr2=0x5616fcd7cf20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.153+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 shutdown_connections 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.153+0000 7f36f807d740 1 -- 172.21.15.28:0/1972994791 wait complete. 2024-05-20T20:15:09.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.153+0000 7f36f807d740 20 client.27319 trim_cache size 0 max 16384 2024-05-20T20:15:09.189 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.189+0000 7f36f807d740 1 Processor -- start 2024-05-20T20:15:09.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.190+0000 7f36f807d740 1 -- start start 2024-05-20T20:15:09.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.191+0000 7f36f807d740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd85320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.190 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.191+0000 7f36f807d740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd9da30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.191+0000 7f36f807d740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd9df70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.192+0000 7f36f807d740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f36d000e980 con 0x5616fcd7ac90 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.192+0000 7f36f807d740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f36d00104c0 con 0x5616fcd85e70 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.192+0000 7f36f807d740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f36d000f050 con 0x5616fcd7b5b0 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.192+0000 7f36ede00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd85320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.192+0000 7f36ede00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd85320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45718/0 (socket says 172.21.15.28:45718) 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.192+0000 7f36ede00640 1 -- 172.21.15.28:0/448921386 learned_addr learned my addr 172.21.15.28:0/448921386 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.193+0000 7f36ede00640 1 -- 172.21.15.28:0/448921386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 msgr2=0x5616fcd9da30 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:15:09.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.193+0000 7f36ede00640 1 --2- 172.21.15.28:0/448921386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd9da30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.193+0000 7f36ede00640 1 -- 172.21.15.28:0/448921386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 msgr2=0x5616fcd9df70 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:15:09.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.193+0000 7f36ede00640 1 --2- 172.21.15.28:0/448921386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd9df70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.193+0000 7f36ede00640 1 -- 172.21.15.28:0/448921386 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5616fcd756b0 con 0x5616fcd7ac90 2024-05-20T20:15:09.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.193+0000 7f36ede00640 1 --2- 172.21.15.28:0/448921386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd85320 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f36e8006010 tx=0x7f36e8009330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=611d2ca8804ace11 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.194+0000 7f36daa00640 1 -- 172.21.15.28:0/448921386 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36e8036040 con 0x5616fcd7ac90 2024-05-20T20:15:09.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.194+0000 7f36daa00640 1 -- 172.21.15.28:0/448921386 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f36e8030dd0 con 0x5616fcd7ac90 2024-05-20T20:15:09.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.194+0000 7f36daa00640 1 -- 172.21.15.28:0/448921386 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36e80318e0 con 0x5616fcd7ac90 2024-05-20T20:15:09.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.195+0000 7f36f807d740 1 -- 172.21.15.28:0/448921386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 msgr2=0x5616fcd85320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.195+0000 7f36f807d740 1 --2- 172.21.15.28:0/448921386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd85320 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f36e8006010 tx=0x7f36e8009330 comp rx=0 tx=0).stop 2024-05-20T20:15:09.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.196+0000 7f36f807d740 1 -- 172.21.15.28:0/448921386 shutdown_connections 2024-05-20T20:15:09.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.196+0000 7f36f807d740 1 --2- 172.21.15.28:0/448921386 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd85e70 0x5616fcd9df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.196+0000 7f36f807d740 1 --2- 172.21.15.28:0/448921386 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd9da30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.197+0000 7f36f807d740 1 --2- 172.21.15.28:0/448921386 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd7ac90 0x5616fcd85320 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.196 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.197+0000 7f36f807d740 1 -- 172.21.15.28:0/448921386 >> 172.21.15.28:0/448921386 conn(0x5616fcd771a0 msgr2=0x5616fcd71e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.197+0000 7f36f807d740 1 -- 172.21.15.28:0/448921386 shutdown_connections 2024-05-20T20:15:09.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.197+0000 7f36f807d740 1 -- 172.21.15.28:0/448921386 wait complete. 2024-05-20T20:15:09.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.198+0000 7f36f807d740 1 -- start start 2024-05-20T20:15:09.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.198+0000 7f36f807d740 1 Processor -- start 2024-05-20T20:15:09.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.200+0000 7f36f807d740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd8acc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.200+0000 7f36f807d740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd8b200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.200+0000 7f36f807d740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd8b740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36ede00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd8acc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36ede00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd8acc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58498/0 (socket says 172.21.15.28:58498) 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36ede00640 1 -- 172.21.15.28:0/2365871630 learned_addr learned my addr 172.21.15.28:0/2365871630 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f36d000fdf0 con 0x5616fcd85e70 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f36d000f720 con 0x5616fcd7ac90 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f36d000c980 con 0x5616fcd7b5b0 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36ed400640 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd8b200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36f807d740 10 client.0 _ll_register_callbacks cb 0x7f36e000da10 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:09.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36ee800640 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd8b740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.201+0000 7f36ede00640 1 -- 172.21.15.28:0/2365871630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 msgr2=0x5616fcd8b200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.202+0000 7f36ede00640 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd8b200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.202+0000 7f36ede00640 1 -- 172.21.15.28:0/2365871630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 msgr2=0x5616fcd8b740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.202+0000 7f36ede00640 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd8b740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.202+0000 7f36ede00640 1 -- 172.21.15.28:0/2365871630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5616fcdfe900 con 0x5616fcd7ac90 2024-05-20T20:15:09.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.202+0000 7f36ede00640 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd8acc0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f36e8034040 tx=0x7f36e8009570 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4c1a18b65a31dcea server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.202+0000 7f36cfe00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:09.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.203+0000 7f36cfe00640 1 -- 172.21.15.28:0/2365871630 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f36e805b030 con 0x5616fcd7ac90 2024-05-20T20:15:09.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.203+0000 7f36cfe00640 1 -- 172.21.15.28:0/2365871630 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f36e8030ae0 con 0x5616fcd7ac90 2024-05-20T20:15:09.203 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.204+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f36e8030dd0 con 0x5616fcd7ac90 2024-05-20T20:15:09.207 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.208+0000 7f36cfe00640 1 -- 172.21.15.28:0/2365871630 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f36e8036040 con 0x5616fcd7ac90 2024-05-20T20:15:09.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.209+0000 7f36f807d740 10 client.27404 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:09.209 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.210+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5616fcdffb80 con 0x5616fcd7ac90 2024-05-20T20:15:09.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.210+0000 7f36cfe00640 1 -- 172.21.15.28:0/2365871630 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f36e8030dd0 con 0x5616fcd7ac90 2024-05-20T20:15:09.210 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.210+0000 7f36cfe00640 1 -- 172.21.15.28:0/2365871630 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f36e804b440 con 0x5616fcd7ac90 2024-05-20T20:15:09.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.212+0000 7f36f807d740 10 client.27404 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:09.211 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.212+0000 7f36f807d740 10 client.27404 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:09.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.212+0000 7f36f807d740 20 client.27404 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:09.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.212+0000 7f36f807d740 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fce001e0 0x5616fce02600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.212+0000 7f36f807d740 4 client.27404 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:15:09.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.212+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5616fcce72c0 con 0x5616fce001e0 2024-05-20T20:15:09.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.212+0000 7f36ed400640 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fce001e0 0x5616fce02600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.212 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.213+0000 7f36ed400640 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fce001e0 0x5616fce02600 crc :-1 s=READY pgs=210 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8109431d0a078d49 server_cookie=820c6e94743003b8 in_seq=0 out_seq=0 2024-05-20T20:15:09.213 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.213+0000 7f36cfe00640 10 client.27404 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:09.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.218+0000 7f204e800700 1 -- 172.21.15.28:0/1263306233 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 msgr2=0x7f2040096510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.218+0000 7f204e800700 1 --2- 172.21.15.28:0/1263306233 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 0x7f2040096510 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7f203c01a9a0 tx=0x7f203c01acb0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.219+0000 7f204e800700 1 -- 172.21.15.28:0/1263306233 shutdown_connections 2024-05-20T20:15:09.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.220+0000 7f204e800700 1 --2- 172.21.15.28:0/1263306233 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2040096a50 0x7f20400a2bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.220+0000 7f204e800700 1 --2- 172.21.15.28:0/1263306233 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f204009ad50 0x7f204009b130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.220+0000 7f204e800700 1 --2- 172.21.15.28:0/1263306233 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 0x7f2040096510 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.220+0000 7f204e800700 1 -- 172.21.15.28:0/1263306233 >> 172.21.15.28:0/1263306233 conn(0x7f204000b890 msgr2=0x7f204000bca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.224+0000 7f204e800700 1 -- 172.21.15.28:0/1263306233 shutdown_connections 2024-05-20T20:15:09.225 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.224+0000 7f204e800700 1 -- 172.21.15.28:0/1263306233 wait complete. 2024-05-20T20:15:09.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.227+0000 7f204e800700 1 Processor -- start 2024-05-20T20:15:09.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.230+0000 7f204e800700 1 -- start start 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.230+0000 7f204e800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 0x7f204012c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.230+0000 7f204e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2040096a50 0x7f204012c970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f204009ad50 0x7f2040130cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204e800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f20400a5f00 con 0x7f2040096a50 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204e800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f20400a5d80 con 0x7f2040096090 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204e800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f20400a6080 con 0x7f204009ad50 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f204009ad50 0x7f2040130cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f204009ad50 0x7f2040130cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58942/0 (socket says 172.21.15.28:58942) 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204de00700 1 -- 172.21.15.28:0/3518618054 learned_addr learned my addr 172.21.15.28:0/3518618054 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.231+0000 7f204d400700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 0x7f204012c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.232+0000 7f204de00700 1 -- 172.21.15.28:0/3518618054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 msgr2=0x7f204012c430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.232+0000 7f204ca00700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2040096a50 0x7f204012c970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.232+0000 7f204de00700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 0x7f204012c430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.232+0000 7f204de00700 1 -- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2040096a50 msgr2=0x7f204012c970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.232+0000 7f204de00700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2040096a50 0x7f204012c970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.232+0000 7f204de00700 1 -- 172.21.15.28:0/3518618054 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f20401313f0 con 0x7f204009ad50 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.232+0000 7f204ca00700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2040096a50 0x7f204012c970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:09.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.233+0000 7f204de00700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f204009ad50 0x7f2040130cb0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f2048052c80 tx=0x7f20480733b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.235+0000 7f2046000700 1 -- 172.21.15.28:0/3518618054 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f204806c2d0 con 0x7f204009ad50 2024-05-20T20:15:09.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.235+0000 7f2046000700 1 -- 172.21.15.28:0/3518618054 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2048079030 con 0x7f204009ad50 2024-05-20T20:15:09.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.235+0000 7f2046000700 1 -- 172.21.15.28:0/3518618054 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f204806abf0 con 0x7f204009ad50 2024-05-20T20:15:09.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.235+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f20401316e0 con 0x7f204009ad50 2024-05-20T20:15:09.236 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.236+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2040007700 con 0x7f204009ad50 2024-05-20T20:15:09.245 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.245+0000 7f205a6acb80 1 Processor -- start 2024-05-20T20:15:09.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.252+0000 7f2046000700 1 -- 172.21.15.28:0/3518618054 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f2048073990 con 0x7f204009ad50 2024-05-20T20:15:09.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.254+0000 7f205a6acb80 1 -- start start 2024-05-20T20:15:09.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.255+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed32ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.255+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed2cd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.255+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed2d390 0x55d19ed2d7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.255+0000 7f205a6acb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d19ec8ae40 con 0x55d19ed327c0 2024-05-20T20:15:09.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f204d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed2cd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f205a6acb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d19ec7a9e0 con 0x55d19ed330e0 2024-05-20T20:15:09.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f205a6acb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d19ec7ea20 con 0x55d19ed2d390 2024-05-20T20:15:09.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f204ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed32ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f204ca00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed32ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:45766/0 (socket says 172.21.15.28:45766) 2024-05-20T20:15:09.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f204ca00700 1 -- 172.21.15.28:0/4007171274 learned_addr learned my addr 172.21.15.28:0/4007171274 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f204d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed2cd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58522/0 (socket says 172.21.15.28:58522) 2024-05-20T20:15:09.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.256+0000 7f2046000700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f20300b2b40 0x7f20300b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f204d400700 1 -- 172.21.15.28:0/4007171274 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed2d390 msgr2=0x55d19ed2d7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f204d400700 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed2d390 0x55d19ed2d7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f204d400700 1 -- 172.21.15.28:0/4007171274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 msgr2=0x55d19ed32ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f204d400700 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed32ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f204d400700 1 -- 172.21.15.28:0/4007171274 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d19ec43420 con 0x55d19ed330e0 2024-05-20T20:15:09.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f204ca00700 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed32ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:09.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f2046000700 1 -- 172.21.15.28:0/3518618054 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f2048146500 con 0x7f204009ad50 2024-05-20T20:15:09.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.257+0000 7f204d400700 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed2cd70 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f203400dbb0 tx=0x7f203400dec0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=24242554718db04c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.258+0000 7f204de00700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f20300b2b40 0x7f20300b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.258+0000 7f203aa00700 1 -- 172.21.15.28:0/4007171274 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2034004230 con 0x55d19ed330e0 2024-05-20T20:15:09.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.258+0000 7f203aa00700 1 -- 172.21.15.28:0/4007171274 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f203400fe20 con 0x55d19ed330e0 2024-05-20T20:15:09.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.258+0000 7f203aa00700 1 -- 172.21.15.28:0/4007171274 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2034005020 con 0x55d19ed330e0 2024-05-20T20:15:09.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.259+0000 7f205a6acb80 1 -- 172.21.15.28:0/4007171274 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 msgr2=0x55d19ed2cd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.259+0000 7f205a6acb80 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed2cd70 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f203400dbb0 tx=0x7f203400dec0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.260+0000 7f204de00700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f20300b2b40 0x7f20300b5000 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f2048107fd0 tx=0x7f20480696c0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.261+0000 7f205a6acb80 1 -- 172.21.15.28:0/4007171274 shutdown_connections 2024-05-20T20:15:09.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.261+0000 7f205a6acb80 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed2d390 0x55d19ed2d7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.261+0000 7f205a6acb80 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed32ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.261+0000 7f205a6acb80 1 --2- 172.21.15.28:0/4007171274 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed2cd70 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.261+0000 7f205a6acb80 1 -- 172.21.15.28:0/4007171274 >> 172.21.15.28:0/4007171274 conn(0x55d19ec5abe0 msgr2=0x55d19ed27810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.261+0000 7f205a6acb80 1 -- 172.21.15.28:0/4007171274 shutdown_connections 2024-05-20T20:15:09.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.261+0000 7f205a6acb80 1 -- 172.21.15.28:0/4007171274 wait complete. 2024-05-20T20:15:09.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.262+0000 7f205a6acb80 1 -- start start 2024-05-20T20:15:09.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.262+0000 7f205a6acb80 1 Processor -- start 2024-05-20T20:15:09.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.264+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed38f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.264+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed362d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.264+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed36810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.264+0000 7f205a6acb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d19eb80fa0 con 0x55d19ed327c0 2024-05-20T20:15:09.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.264+0000 7f205a6acb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d19ec1b760 con 0x55d19ed2d390 2024-05-20T20:15:09.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.264+0000 7f205a6acb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d19ed39c00 con 0x55d19ed330e0 2024-05-20T20:15:09.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.264+0000 7f205a6acb80 10 client.0 _ll_register_callbacks cb 0x55d19ec24c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:09.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.265+0000 7f204ca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed38f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.265+0000 7f204ca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed38f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58530/0 (socket says 172.21.15.28:58530) 2024-05-20T20:15:09.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.265+0000 7f204ca00700 1 -- 172.21.15.28:0/531227211 learned_addr learned my addr 172.21.15.28:0/531227211 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.266+0000 7f204ca00700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed36810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.267+0000 7f204ca00700 1 -- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 msgr2=0x55d19ed36810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.267+0000 7f204ca00700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed36810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.267+0000 7f204d400700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed362d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.267+0000 7f204ca00700 1 -- 172.21.15.28:0/531227211 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 msgr2=0x55d19ed362d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.268+0000 7f204ca00700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed362d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.268+0000 7f204ca00700 1 -- 172.21.15.28:0/531227211 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d19ebca8d0 con 0x55d19ed2d390 2024-05-20T20:15:09.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.270+0000 7f204d400700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed362d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:09.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.270+0000 7f202f400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:09.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.271+0000 7f204ca00700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed38f30 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f203c005b40 tx=0x7f203c005a90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2603e8dd7d4b52d2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.271+0000 7f202f400700 1 -- 172.21.15.28:0/531227211 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f203c003860 con 0x55d19ed2d390 2024-05-20T20:15:09.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.272+0000 7f202f400700 1 -- 172.21.15.28:0/531227211 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f203c02e020 con 0x55d19ed2d390 2024-05-20T20:15:09.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.273+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f203400fe20 con 0x55d19ed2d390 2024-05-20T20:15:09.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.276+0000 7f202f400700 1 -- 172.21.15.28:0/531227211 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f203c003860 con 0x55d19ed2d390 2024-05-20T20:15:09.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.276+0000 7f205a6acb80 10 client.27416 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:09.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.276+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d19ec4b060 con 0x55d19ed2d390 2024-05-20T20:15:09.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.277+0000 7f202f400700 1 -- 172.21.15.28:0/531227211 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f203c005070 con 0x55d19ed2d390 2024-05-20T20:15:09.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.279+0000 7f202f400700 1 -- 172.21.15.28:0/531227211 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f203c037040 con 0x55d19ed2d390 2024-05-20T20:15:09.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.280+0000 7f205a6acb80 10 client.27416 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:09.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.280+0000 7f205a6acb80 10 client.27416 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:15:09.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.280+0000 7f205a6acb80 20 client.27416 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:09.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.280+0000 7f205a6acb80 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19ed46840 0x55d19ed48c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.280+0000 7f205a6acb80 4 client.27416 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:09.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.280+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f203400fe20 con 0x55d19ed46840 2024-05-20T20:15:09.280 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.281+0000 7f204d400700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19ed46840 0x55d19ed48c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.282+0000 7f204d400700 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19ed46840 0x55d19ed48c30 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=7268ff4ccd00329 server_cookie=71ed024e36b251a6 in_seq=0 out_seq=0 2024-05-20T20:15:09.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.282+0000 7f202f400700 10 client.27416 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:15:09.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.283+0000 7f202f400700 1 -- 172.21.15.28:0/531227211 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x7f203400fe20 con 0x55d19ed46840 2024-05-20T20:15:09.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.283+0000 7f202f400700 10 client.27416 handle_command_reply: tid=0 2024-05-20T20:15:09.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.284+0000 7f205a6acb80 1 client.27416 shutdown 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 msgr2=0x55d19ed38f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed38f30 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f203c005b40 tx=0x7f203c005a90 comp rx=0 tx=0).stop 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 shutdown_connections 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19ed46840 0x55d19ed48c30 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed36810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed38f30 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 --2- 172.21.15.28:0/531227211 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed362d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.289+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 >> 172.21.15.28:0/531227211 conn(0x55d19ec5abe0 msgr2=0x55d19ed26fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.292+0000 7f202f400700 0 client.27416 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:15:09.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.292+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 shutdown_connections 2024-05-20T20:15:09.292 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.293+0000 7f205a6acb80 1 -- 172.21.15.28:0/531227211 wait complete. 2024-05-20T20:15:09.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.293+0000 7f205a6acb80 20 client.27416 trim_cache size 0 max 16384 2024-05-20T20:15:09.334 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.334+0000 7f205a6acb80 1 Processor -- start 2024-05-20T20:15:09.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.335+0000 7f205a6acb80 1 -- start start 2024-05-20T20:15:09.335 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.335+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed2cd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.335+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed42ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.335+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed367c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.335+0000 7f205a6acb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f20401426d0 con 0x55d19ed327c0 2024-05-20T20:15:09.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.335+0000 7f205a6acb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2040144210 con 0x55d19ed2d390 2024-05-20T20:15:09.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.335+0000 7f205a6acb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2040142da0 con 0x55d19ed330e0 2024-05-20T20:15:09.336 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204ca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed2cd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204ca00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed2cd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58542/0 (socket says 172.21.15.28:58542) 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204ca00700 1 -- 172.21.15.28:0/3446191417 learned_addr learned my addr 172.21.15.28:0/3446191417 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204ca00700 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed367c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204d400700 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed42ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204ca00700 1 -- 172.21.15.28:0/3446191417 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 msgr2=0x55d19ed367c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204ca00700 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed367c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.336+0000 7f204ca00700 1 -- 172.21.15.28:0/3446191417 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 msgr2=0x55d19ed42ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.337 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.337+0000 7f204ca00700 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed42ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.337+0000 7f204ca00700 1 -- 172.21.15.28:0/3446191417 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d19ed286a0 con 0x55d19ed2d390 2024-05-20T20:15:09.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.337+0000 7f204d400700 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed42ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:09.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.337+0000 7f204ca00700 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed2cd70 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f203c003c60 tx=0x7f203c01af90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ecd25a6419124d1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.339+0000 7f203a000700 1 -- 172.21.15.28:0/3446191417 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f203c0050e0 con 0x55d19ed2d390 2024-05-20T20:15:09.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.339+0000 7f203a000700 1 -- 172.21.15.28:0/3446191417 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f203c005730 con 0x55d19ed2d390 2024-05-20T20:15:09.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.339+0000 7f203a000700 1 -- 172.21.15.28:0/3446191417 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f203c0050e0 con 0x55d19ed2d390 2024-05-20T20:15:09.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.341+0000 7f205a6acb80 1 -- 172.21.15.28:0/3446191417 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 msgr2=0x55d19ed2cd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.341 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.341+0000 7f205a6acb80 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed2cd70 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f203c003c60 tx=0x7f203c01af90 comp rx=0 tx=0).stop 2024-05-20T20:15:09.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.342+0000 7f205a6acb80 1 -- 172.21.15.28:0/3446191417 shutdown_connections 2024-05-20T20:15:09.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.342+0000 7f205a6acb80 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed330e0 0x55d19ed367c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.342+0000 7f205a6acb80 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed2d390 0x55d19ed2cd70 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.342 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.342+0000 7f205a6acb80 1 --2- 172.21.15.28:0/3446191417 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed327c0 0x55d19ed42ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.343 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.342+0000 7f205a6acb80 1 -- 172.21.15.28:0/3446191417 >> 172.21.15.28:0/3446191417 conn(0x55d19ec5abe0 msgr2=0x55d19ed360b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.344+0000 7f205a6acb80 1 -- 172.21.15.28:0/3446191417 shutdown_connections 2024-05-20T20:15:09.344 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.344+0000 7f205a6acb80 1 -- 172.21.15.28:0/3446191417 wait complete. 2024-05-20T20:15:09.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.345+0000 7f205a6acb80 1 -- start start 2024-05-20T20:15:09.345 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.346+0000 7f205a6acb80 1 Processor -- start 2024-05-20T20:15:09.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.347+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed2d390 0x55d19ed3f070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed327c0 0x55d19ed3f5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f205a6acb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed4d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f205a6acb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2040143b40 con 0x55d19ed2d390 2024-05-20T20:15:09.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f205a6acb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2040143470 con 0x55d19ed330e0 2024-05-20T20:15:09.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f205a6acb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2040140720 con 0x55d19ed327c0 2024-05-20T20:15:09.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f205a6acb80 10 client.0 _ll_register_callbacks cb 0x55d19ec24c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:09.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f204d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed327c0 0x55d19ed3f5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f204d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed327c0 0x55d19ed3f5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:58974/0 (socket says 172.21.15.28:58974) 2024-05-20T20:15:09.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.348+0000 7f204d400700 1 -- 172.21.15.28:0/879541877 learned_addr learned my addr 172.21.15.28:0/879541877 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:09.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.349+0000 7f204ca00700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed4d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.351+0000 7f204ca00700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed2d390 0x55d19ed3f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.351+0000 7f204d400700 1 -- 172.21.15.28:0/879541877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 msgr2=0x55d19ed4d8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.351+0000 7f204d400700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed4d8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.351+0000 7f204d400700 1 -- 172.21.15.28:0/879541877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed2d390 msgr2=0x55d19ed3f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:09.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.356+0000 7f204d400700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed2d390 0x55d19ed3f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.356+0000 7f204d400700 1 -- 172.21.15.28:0/879541877 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d19ed32df0 con 0x55d19ed327c0 2024-05-20T20:15:09.357 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.356+0000 7f204ca00700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed2d390 0x55d19ed3f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:09.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.358+0000 7f204d400700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed327c0 0x55d19ed3f5b0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f2034005fd0 tx=0x7f2034005f60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5a05eaea8ced4a39 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:09.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.358+0000 7f202f400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:09.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.359+0000 7f202f400700 1 -- 172.21.15.28:0/879541877 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f203402c060 con 0x55d19ed327c0 2024-05-20T20:15:09.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.359+0000 7f202f400700 1 -- 172.21.15.28:0/879541877 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f2034027450 con 0x55d19ed327c0 2024-05-20T20:15:09.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.359+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f203c005730 con 0x55d19ed327c0 2024-05-20T20:15:09.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.359+0000 7f202f400700 1 -- 172.21.15.28:0/879541877 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f20340049d0 con 0x55d19ed327c0 2024-05-20T20:15:09.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.359+0000 7f205a6acb80 10 client.27337 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:09.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.359+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d19edb0cd0 con 0x55d19ed327c0 2024-05-20T20:15:09.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.360+0000 7f202f400700 1 -- 172.21.15.28:0/879541877 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f203c005730 con 0x55d19ed327c0 2024-05-20T20:15:09.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.363+0000 7f202f400700 1 -- 172.21.15.28:0/879541877 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f2034027670 con 0x55d19ed327c0 2024-05-20T20:15:09.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.364+0000 7f205a6acb80 10 client.27337 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:09.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.364+0000 7f205a6acb80 10 client.27337 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:15:09.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.364+0000 7f205a6acb80 20 client.27337 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:09.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.364+0000 7f205a6acb80 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19edb10b0 0x55d19edb34a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:09.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.364+0000 7f205a6acb80 4 client.27337 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:15:09.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.364+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55d19ec8cd90 con 0x55d19edb10b0 2024-05-20T20:15:09.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.366+0000 7f204ca00700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19edb10b0 0x55d19edb34a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:09.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.367+0000 7f204ca00700 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19edb10b0 0x55d19edb34a0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=9d79de35e1f24aaf server_cookie=971d61a6b77915d3 in_seq=0 out_seq=0 2024-05-20T20:15:09.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.367+0000 7f202f400700 10 client.27337 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:15:09.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.367+0000 7f202f400700 1 -- 172.21.15.28:0/879541877 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55d19ec8cd90 con 0x55d19edb10b0 2024-05-20T20:15:09.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.367+0000 7f202f400700 10 client.27337 handle_command_reply: tid=0 2024-05-20T20:15:09.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.367+0000 7f205a6acb80 1 client.27337 shutdown 2024-05-20T20:15:09.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed327c0 msgr2=0x55d19ed3f5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed327c0 0x55d19ed3f5b0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f2034005fd0 tx=0x7f2034005f60 comp rx=0 tx=0).stop 2024-05-20T20:15:09.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 shutdown_connections 2024-05-20T20:15:09.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d19ed330e0 0x55d19ed4d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d19ed2d390 0x55d19ed3f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x55d19edb10b0 0x55d19edb34a0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 --2- 172.21.15.28:0/879541877 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d19ed327c0 0x55d19ed3f5b0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f202f400700 0 client.27337 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:15:09.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 >> 172.21.15.28:0/879541877 conn(0x55d19ec5abe0 msgr2=0x55d19ed334c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.368+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 shutdown_connections 2024-05-20T20:15:09.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.369+0000 7f205a6acb80 1 -- 172.21.15.28:0/879541877 wait complete. 2024-05-20T20:15:09.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.369+0000 7f205a6acb80 20 client.27337 trim_cache size 0 max 16384 2024-05-20T20:15:09.370 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:15:09.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.371+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f20300b2b40 msgr2=0x7f20300b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.371+0000 7f204e800700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f20300b2b40 0x7f20300b5000 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f2048107fd0 tx=0x7f20480696c0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.371+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f204009ad50 msgr2=0x7f2040130cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.371+0000 7f204e800700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f204009ad50 0x7f2040130cb0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f2048052c80 tx=0x7f20480733b0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 shutdown_connections 2024-05-20T20:15:09.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f204009ad50 0x7f2040130cb0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f20300b2b40 0x7f20300b5000 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2040096090 0x7f204012c430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 --2- 172.21.15.28:0/3518618054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2040096a50 0x7f204012c970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 >> 172.21.15.28:0/3518618054 conn(0x7f204000b890 msgr2=0x7f2040094180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 shutdown_connections 2024-05-20T20:15:09.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:09.372+0000 7f204e800700 1 -- 172.21.15.28:0/3518618054 wait complete. 2024-05-20T20:15:09.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:09 smithi154 ceph-mon[36314]: pgmap v294: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 268 MiB/s wr, 1.33k op/s 2024-05-20T20:15:09.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:09 smithi154 ceph-mon[36314]: osdmap e88: 12 total, 12 up, 12 in 2024-05-20T20:15:09.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:09 smithi033 ceph-mon[36179]: pgmap v294: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 268 MiB/s wr, 1.33k op/s 2024-05-20T20:15:09.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:09 smithi033 ceph-mon[36179]: osdmap e88: 12 total, 12 up, 12 in 2024-05-20T20:15:09.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.415+0000 7f36cfe00640 1 -- 172.21.15.28:0/2365871630 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5616fcce72c0 con 0x5616fce001e0 2024-05-20T20:15:09.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.415+0000 7f36cfe00640 10 client.27404 handle_command_reply: tid=0 2024-05-20T20:15:09.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 client.27404 shutdown 2024-05-20T20:15:09.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 msgr2=0x5616fcd8acc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd8acc0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7f36e8034040 tx=0x7f36e8009570 comp rx=0 tx=0).stop 2024-05-20T20:15:09.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 shutdown_connections 2024-05-20T20:15:09.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5616fce001e0 0x5616fce02600 crc :-1 s=READY pgs=210 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5616fcd85e70 0x5616fcd8b740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5616fcd7b5b0 0x5616fcd8b200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 --2- 172.21.15.28:0/2365871630 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5616fcd7ac90 0x5616fcd8acc0 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.416+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 >> 172.21.15.28:0/2365871630 conn(0x5616fcd771a0 msgr2=0x5616fcd75a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.417+0000 7f36cfe00640 0 client.27404 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:09.418 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:09 smithi028 ceph-mon[31178]: pgmap v294: 97 pgs: 97 active+clean; 63 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 268 MiB/s wr, 1.33k op/s 2024-05-20T20:15:09.418 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:09 smithi028 ceph-mon[31178]: osdmap e88: 12 total, 12 up, 12 in 2024-05-20T20:15:09.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.421+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 shutdown_connections 2024-05-20T20:15:09.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.422+0000 7f36f807d740 1 -- 172.21.15.28:0/2365871630 wait complete. 2024-05-20T20:15:09.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.423+0000 7f36f807d740 20 client.27404 trim_cache size 0 max 16384 2024-05-20T20:15:09.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:15:09.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:15:09.423 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:15:09.425 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.426+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f36d40b2c20 msgr2=0x7f36d40b50e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.426 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.426+0000 7f36f5200640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f36d40b2c20 0x7f36d40b50e0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f36e0005ec0 tx=0x7f36e00046c0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.426 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.426+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f36f01ac950 msgr2=0x7f36f01ace00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:09.426 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.426+0000 7f36f5200640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f36f01ac950 0x7f36f01ace00 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7f36dc00e940 tx=0x7f36dc00ee00 comp rx=0 tx=0).stop 2024-05-20T20:15:09.427 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.427+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 shutdown_connections 2024-05-20T20:15:09.427 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.427+0000 7f36f5200640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f36f01ac950 0x7f36f01ace00 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.427 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.427+0000 7f36f5200640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f36d40b2c20 0x7f36d40b50e0 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.427 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.427+0000 7f36f5200640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f36f011c230 0x7f36f01a85d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.428 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.427+0000 7f36f5200640 1 --2- 172.21.15.28:0/3912411022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f36f0111240 0x7f36f01a8090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:09.428 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.427+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 >> 172.21.15.28:0/3912411022 conn(0x7f36f006db20 msgr2=0x7f36f010ef00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:09.428 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.428+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 shutdown_connections 2024-05-20T20:15:09.428 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:09.428+0000 7f36f5200640 1 -- 172.21.15.28:0/3912411022 wait complete. 2024-05-20T20:15:10.436 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:15:11.241 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:11.466 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:11 smithi028 ceph-mon[31178]: pgmap v296: 97 pgs: 16 active+clean+snaptrim, 57 active+clean, 24 active+clean+snaptrim_wait; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 290 MiB/s wr, 1.77k op/s 2024-05-20T20:15:11.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:11 smithi154 ceph-mon[36314]: pgmap v296: 97 pgs: 16 active+clean+snaptrim, 57 active+clean, 24 active+clean+snaptrim_wait; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 290 MiB/s wr, 1.77k op/s 2024-05-20T20:15:11.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:11 smithi033 ceph-mon[36179]: pgmap v296: 97 pgs: 16 active+clean+snaptrim, 57 active+clean, 24 active+clean+snaptrim_wait; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 290 MiB/s wr, 1.77k op/s 2024-05-20T20:15:13.553 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:13 smithi028 ceph-mon[31178]: pgmap v297: 97 pgs: 18 active+clean+snaptrim, 49 active+clean, 30 active+clean+snaptrim_wait; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 286 MiB/s wr, 1.81k op/s 2024-05-20T20:15:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:13 smithi154 ceph-mon[36314]: pgmap v297: 97 pgs: 18 active+clean+snaptrim, 49 active+clean, 30 active+clean+snaptrim_wait; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 286 MiB/s wr, 1.81k op/s 2024-05-20T20:15:13.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:13 smithi033 ceph-mon[36179]: pgmap v297: 97 pgs: 18 active+clean+snaptrim, 49 active+clean, 30 active+clean+snaptrim_wait; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 286 MiB/s wr, 1.81k op/s 2024-05-20T20:15:15.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:15 smithi028 ceph-mon[31178]: pgmap v298: 97 pgs: 8 active+clean+snaptrim, 72 active+clean, 17 active+clean+snaptrim_wait; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 273 MiB/s wr, 1.99k op/s 2024-05-20T20:15:15.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.611+0000 7f1336200700 1 -- 172.21.15.28:0/265961559 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1328094ee0 msgr2=0x7f1328095360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.611+0000 7f1336200700 1 --2- 172.21.15.28:0/265961559 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1328094ee0 0x7f1328095360 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f131c009990 tx=0x7f131c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.611+0000 7f1336200700 1 -- 172.21.15.28:0/265961559 shutdown_connections 2024-05-20T20:15:15.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.611+0000 7f1336200700 1 --2- 172.21.15.28:0/265961559 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f13280958a0 0x7f13280a1ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.611+0000 7f1336200700 1 --2- 172.21.15.28:0/265961559 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f132809aca0 0x7f132809b080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.611+0000 7f1336200700 1 --2- 172.21.15.28:0/265961559 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1328094ee0 0x7f1328095360 secure :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7f131c009990 tx=0x7f131c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.611+0000 7f1336200700 1 -- 172.21.15.28:0/265961559 >> 172.21.15.28:0/265961559 conn(0x7f132800a7a0 msgr2=0x7f132800abb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:15.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.614+0000 7f1336200700 1 -- 172.21.15.28:0/265961559 shutdown_connections 2024-05-20T20:15:15.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.620+0000 7f1336200700 1 -- 172.21.15.28:0/265961559 wait complete. 2024-05-20T20:15:15.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.622+0000 7f1336200700 1 Processor -- start 2024-05-20T20:15:15.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.623+0000 7f1336200700 1 -- start start 2024-05-20T20:15:15.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.624+0000 7f1336200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f13280958a0 0x7f1328134630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:15 smithi154 ceph-mon[36314]: pgmap v298: 97 pgs: 8 active+clean+snaptrim, 72 active+clean, 17 active+clean+snaptrim_wait; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 273 MiB/s wr, 1.99k op/s 2024-05-20T20:15:15.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.624+0000 7f1336200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132809aca0 0x7f1328134b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.624+0000 7f1336200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132812e700 0x7f132812ebb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:15 smithi033 ceph-mon[36179]: pgmap v298: 97 pgs: 8 active+clean+snaptrim, 72 active+clean, 17 active+clean+snaptrim_wait; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 273 MiB/s wr, 1.99k op/s 2024-05-20T20:15:15.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.624+0000 7f1336200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f13280a5fa0 con 0x7f13280958a0 2024-05-20T20:15:15.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.624+0000 7f1336200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f13280a5e20 con 0x7f132809aca0 2024-05-20T20:15:15.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.624+0000 7f1336200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f13280a6120 con 0x7f132812e700 2024-05-20T20:15:15.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.625+0000 7f1335800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132812e700 0x7f132812ebb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.626+0000 7f1335800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132812e700 0x7f132812ebb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59000/0 (socket says 172.21.15.28:59000) 2024-05-20T20:15:15.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.626+0000 7f1335800700 1 -- 172.21.15.28:0/2604623600 learned_addr learned my addr 172.21.15.28:0/2604623600 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:15.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.626+0000 7f1335800700 1 -- 172.21.15.28:0/2604623600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132809aca0 msgr2=0x7f1328134b70 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:15:15.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.626+0000 7f1335800700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132809aca0 0x7f1328134b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.626+0000 7f1335800700 1 -- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f13280958a0 msgr2=0x7f1328134630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.626+0000 7f1335800700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f13280958a0 0x7f1328134630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.626+0000 7f1335800700 1 -- 172.21.15.28:0/2604623600 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f132812f320 con 0x7f132812e700 2024-05-20T20:15:15.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.627+0000 7f1335800700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132812e700 0x7f132812ebb0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f1330000eb0 tx=0x7f1330075e20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:15.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.628+0000 7f132d600700 1 -- 172.21.15.28:0/2604623600 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f133006ca80 con 0x7f132812e700 2024-05-20T20:15:15.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.628+0000 7f132d600700 1 -- 172.21.15.28:0/2604623600 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1330085070 con 0x7f132812e700 2024-05-20T20:15:15.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.628+0000 7f132d600700 1 -- 172.21.15.28:0/2604623600 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f133007f600 con 0x7f132812e700 2024-05-20T20:15:15.632 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.631+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f132813b3e0 con 0x7f132812e700 2024-05-20T20:15:15.633 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.632+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f132813b780 con 0x7f132812e700 2024-05-20T20:15:15.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.650+0000 7f1342218b80 1 Processor -- start 2024-05-20T20:15:15.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.651+0000 7f1342218b80 1 -- start start 2024-05-20T20:15:15.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.652+0000 7f1342218b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f97c0 0x5596019f9ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.652+0000 7f1342218b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019f3d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.652+0000 7f132d600700 1 -- 172.21.15.28:0/2604623600 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f133006cc20 con 0x7f132812e700 2024-05-20T20:15:15.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.653+0000 7f1342218b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f4390 0x5596019f47f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.653+0000 7f1342218b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559601951e40 con 0x5596019f97c0 2024-05-20T20:15:15.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.653+0000 7f1342218b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5596019419e0 con 0x5596019fa0e0 2024-05-20T20:15:15.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.653+0000 7f1342218b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559601945a20 con 0x5596019f4390 2024-05-20T20:15:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.654+0000 7f1334e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019f3d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.654+0000 7f1334e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019f3d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58580/0 (socket says 172.21.15.28:58580) 2024-05-20T20:15:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.654+0000 7f1334e00700 1 -- 172.21.15.28:0/4110817322 learned_addr learned my addr 172.21.15.28:0/4110817322 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.654+0000 7f132d600700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13180b2c00 0x7f13180b50c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.656 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.655+0000 7f132d600700 1 -- 172.21.15.28:0/2604623600 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f133014f970 con 0x7f132812e700 2024-05-20T20:15:15.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.656+0000 7f132fe00700 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f4390 0x5596019f47f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.657+0000 7f1334e00700 1 -- 172.21.15.28:0/4110817322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f4390 msgr2=0x5596019f47f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.657+0000 7f1334e00700 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f4390 0x5596019f47f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.657+0000 7f1334e00700 1 -- 172.21.15.28:0/4110817322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f97c0 msgr2=0x5596019f9ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.657+0000 7f1334e00700 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f97c0 0x5596019f9ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.657+0000 7f1334e00700 1 -- 172.21.15.28:0/4110817322 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55960190a420 con 0x5596019fa0e0 2024-05-20T20:15:15.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.659+0000 7f1334e00700 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019f3d70 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f132400bb10 tx=0x7f132400be20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2700364ce1d982ea server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:15.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.662+0000 7f1322000700 1 -- 172.21.15.28:0/4110817322 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1324016070 con 0x5596019fa0e0 2024-05-20T20:15:15.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.662+0000 7f1322000700 1 -- 172.21.15.28:0/4110817322 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f132400d810 con 0x5596019fa0e0 2024-05-20T20:15:15.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.662+0000 7f1322000700 1 -- 172.21.15.28:0/4110817322 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1324005450 con 0x5596019fa0e0 2024-05-20T20:15:15.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.663+0000 7f1342218b80 1 -- 172.21.15.28:0/4110817322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 msgr2=0x5596019f3d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.663+0000 7f1342218b80 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019f3d70 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f132400bb10 tx=0x7f132400be20 comp rx=0 tx=0).stop 2024-05-20T20:15:15.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.665+0000 7f1342218b80 1 -- 172.21.15.28:0/4110817322 shutdown_connections 2024-05-20T20:15:15.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.665+0000 7f1342218b80 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f4390 0x5596019f47f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.665+0000 7f1342218b80 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f97c0 0x5596019f9ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.665+0000 7f1342218b80 1 --2- 172.21.15.28:0/4110817322 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019f3d70 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.665+0000 7f1342218b80 1 -- 172.21.15.28:0/4110817322 >> 172.21.15.28:0/4110817322 conn(0x559601921be0 msgr2=0x5596019ee810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:15.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.669+0000 7f1342218b80 1 -- 172.21.15.28:0/4110817322 shutdown_connections 2024-05-20T20:15:15.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.669+0000 7f1342218b80 1 -- 172.21.15.28:0/4110817322 wait complete. 2024-05-20T20:15:15.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.670+0000 7f1342218b80 1 -- start start 2024-05-20T20:15:15.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.670+0000 7f1342218b80 1 Processor -- start 2024-05-20T20:15:15.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.673+0000 7f1335800700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13180b2c00 0x7f13180b50c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.674+0000 7f1342218b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019ffe10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.675+0000 7f1342218b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x5596019fd1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.675+0000 7f1342218b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019fd6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.675+0000 7f1342218b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x559601847fa0 con 0x5596019f4390 2024-05-20T20:15:15.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.675+0000 7f1342218b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5596018e2760 con 0x5596019fa0e0 2024-05-20T20:15:15.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.675+0000 7f1342218b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x559601a00c00 con 0x5596019f97c0 2024-05-20T20:15:15.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.675+0000 7f1342218b80 10 client.0 _ll_register_callbacks cb 0x5596018ebc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:15.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.676+0000 7f132fe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019ffe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.676+0000 7f1334e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x5596019fd1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.677+0000 7f1334e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x5596019fd1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59012/0 (socket says 172.21.15.28:59012) 2024-05-20T20:15:15.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.677+0000 7f1334e00700 1 -- 172.21.15.28:0/1686840674 learned_addr learned my addr 172.21.15.28:0/1686840674 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:15.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.678+0000 7f132fe00700 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019fd6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.679+0000 7f1334e00700 1 -- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 msgr2=0x5596019fd6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.679+0000 7f1334e00700 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019fd6f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.679+0000 7f1334e00700 1 -- 172.21.15.28:0/1686840674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 msgr2=0x5596019ffe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.679+0000 7f1334e00700 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019ffe10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.679+0000 7f1334e00700 1 -- 172.21.15.28:0/1686840674 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5596018918d0 con 0x5596019f97c0 2024-05-20T20:15:15.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.680+0000 7f132fe00700 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019ffe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:15.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.680+0000 7f1334e00700 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x5596019fd1b0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f1324000c00 tx=0x7f132400dd30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2ec38e17963dcc8f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:15.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.680+0000 7f1316a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:15.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.681+0000 7f1335800700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13180b2c00 0x7f13180b50c0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f13300693f0 tx=0x7f1330069f60 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:15.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.681+0000 7f1316a00700 1 -- 172.21.15.28:0/1686840674 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1324016070 con 0x5596019f97c0 2024-05-20T20:15:15.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.681+0000 7f1316a00700 1 -- 172.21.15.28:0/1686840674 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f1324004a30 con 0x5596019f97c0 2024-05-20T20:15:15.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.682+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f132400d810 con 0x5596019f97c0 2024-05-20T20:15:15.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.682+0000 7f1316a00700 1 -- 172.21.15.28:0/1686840674 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1324004c30 con 0x5596019f97c0 2024-05-20T20:15:15.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.682+0000 7f1342218b80 10 client.27349 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:15.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.683+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559601912060 con 0x5596019f97c0 2024-05-20T20:15:15.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.683+0000 7f1316a00700 1 -- 172.21.15.28:0/1686840674 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f132400d810 con 0x5596019f97c0 2024-05-20T20:15:15.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.683+0000 7f1316a00700 1 -- 172.21.15.28:0/1686840674 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f132402e8b0 con 0x5596019f97c0 2024-05-20T20:15:15.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.683+0000 7f1342218b80 10 client.27349 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:15.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.683+0000 7f1342218b80 10 client.27349 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:15:15.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.683+0000 7f1342218b80 20 client.27349 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:15.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.684+0000 7f1342218b80 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a15f20 0x559601a18310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.684+0000 7f1342218b80 4 client.27349 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:15.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.684+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x559601953d90 con 0x559601a15f20 2024-05-20T20:15:15.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.684+0000 7f132fe00700 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a15f20 0x559601a18310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.687+0000 7f132fe00700 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a15f20 0x559601a18310 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=815fa6c657ffd77b server_cookie=80d3fcadb7962257 in_seq=0 out_seq=0 2024-05-20T20:15:15.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.687+0000 7f1316a00700 10 client.27349 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:15:15.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.688+0000 7f1316a00700 1 -- 172.21.15.28:0/1686840674 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x559601953d90 con 0x559601a15f20 2024-05-20T20:15:15.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.688+0000 7f1316a00700 10 client.27349 handle_command_reply: tid=0 2024-05-20T20:15:15.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.688+0000 7f1342218b80 1 client.27349 shutdown 2024-05-20T20:15:15.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.688+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 msgr2=0x5596019fd1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.688+0000 7f1342218b80 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x5596019fd1b0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f1324000c00 tx=0x7f132400dd30 comp rx=0 tx=0).stop 2024-05-20T20:15:15.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.689+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 shutdown_connections 2024-05-20T20:15:15.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.689+0000 7f1342218b80 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a15f20 0x559601a18310 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.689+0000 7f1342218b80 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x5596019fd6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.691+0000 7f1316a00700 0 client.27349 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:15:15.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.691+0000 7f1342218b80 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019ffe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.691+0000 7f1342218b80 1 --2- 172.21.15.28:0/1686840674 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x5596019fd1b0 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.691+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 >> 172.21.15.28:0/1686840674 conn(0x559601921be0 msgr2=0x5596019edfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:15.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.693+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 shutdown_connections 2024-05-20T20:15:15.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.695+0000 7f1342218b80 1 -- 172.21.15.28:0/1686840674 wait complete. 2024-05-20T20:15:15.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.695+0000 7f1342218b80 20 client.27349 trim_cache size 0 max 16384 2024-05-20T20:15:15.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.746+0000 7f1342218b80 1 Processor -- start 2024-05-20T20:15:15.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.748+0000 7f1342218b80 1 -- start start 2024-05-20T20:15:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.748+0000 7f1342218b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019f3d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.748+0000 7f1342218b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019f97c0 0x5596019f78b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.748+0000 7f1342218b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019fa0e0 0x5596019fac90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.748+0000 7f1342218b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f132813ebe0 con 0x5596019f4390 2024-05-20T20:15:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.748+0000 7f1342218b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1328140720 con 0x5596019f97c0 2024-05-20T20:15:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.748+0000 7f1342218b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f132813f2b0 con 0x5596019fa0e0 2024-05-20T20:15:15.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.750+0000 7f132fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019fa0e0 0x5596019fac90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.750+0000 7f1334e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019f97c0 0x5596019f78b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.750+0000 7f1334e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019f97c0 0x5596019f78b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58592/0 (socket says 172.21.15.28:58592) 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.750+0000 7f1334e00700 1 -- 172.21.15.28:0/935927227 learned_addr learned my addr 172.21.15.28:0/935927227 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.751+0000 7f1334e00700 1 -- 172.21.15.28:0/935927227 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019fa0e0 msgr2=0x5596019fac90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.751+0000 7f132fe00700 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019f3d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.751+0000 7f1334e00700 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019fa0e0 0x5596019fac90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.751+0000 7f1334e00700 1 -- 172.21.15.28:0/935927227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 msgr2=0x5596019f3d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.751+0000 7f1334e00700 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019f3d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.752+0000 7f1334e00700 1 -- 172.21.15.28:0/935927227 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5596019fb720 con 0x5596019f97c0 2024-05-20T20:15:15.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.752+0000 7f132fe00700 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019f3d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:15.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.752+0000 7f1334e00700 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019f97c0 0x5596019f78b0 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f132400b500 tx=0x7f132400b400 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5c98284f209dbb0f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:15.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.753+0000 7f1321600700 1 -- 172.21.15.28:0/935927227 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1324016040 con 0x5596019f97c0 2024-05-20T20:15:15.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.753+0000 7f1321600700 1 -- 172.21.15.28:0/935927227 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1324033070 con 0x5596019f97c0 2024-05-20T20:15:15.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.753+0000 7f1321600700 1 -- 172.21.15.28:0/935927227 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1324004a90 con 0x5596019f97c0 2024-05-20T20:15:15.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.754+0000 7f1342218b80 1 -- 172.21.15.28:0/935927227 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019f97c0 msgr2=0x5596019f78b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.754+0000 7f1342218b80 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019f97c0 0x5596019f78b0 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f132400b500 tx=0x7f132400b400 comp rx=0 tx=0).stop 2024-05-20T20:15:15.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.754+0000 7f1342218b80 1 -- 172.21.15.28:0/935927227 shutdown_connections 2024-05-20T20:15:15.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.754+0000 7f1342218b80 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019fa0e0 0x5596019fac90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.754+0000 7f1342218b80 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x5596019f3d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.755+0000 7f1342218b80 1 --2- 172.21.15.28:0/935927227 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019f97c0 0x5596019f78b0 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.755+0000 7f1342218b80 1 -- 172.21.15.28:0/935927227 >> 172.21.15.28:0/935927227 conn(0x559601921be0 msgr2=0x5596019fa580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:15.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.755+0000 7f1342218b80 1 -- 172.21.15.28:0/935927227 shutdown_connections 2024-05-20T20:15:15.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.755+0000 7f1342218b80 1 -- 172.21.15.28:0/935927227 wait complete. 2024-05-20T20:15:15.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.757+0000 7f1342218b80 1 -- start start 2024-05-20T20:15:15.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.757+0000 7f1342218b80 1 Processor -- start 2024-05-20T20:15:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.758+0000 7f1342218b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x559601a7bf40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.758+0000 7f1342218b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x559601a05f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1342218b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x559601a06460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1342218b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1328140050 con 0x5596019f4390 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1342218b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f132813f980 con 0x5596019fa0e0 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1342218b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f132813cbc0 con 0x5596019f97c0 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1342218b80 10 client.0 _ll_register_callbacks cb 0x5596018ebc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1334e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x559601a05f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1334e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x559601a05f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:59022/0 (socket says 172.21.15.28:59022) 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f1334e00700 1 -- 172.21.15.28:0/1038922546 learned_addr learned my addr 172.21.15.28:0/1038922546 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:15.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f132fe00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x559601a06460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.759+0000 7f132fe00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x559601a7bf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.760+0000 7f1334e00700 1 -- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 msgr2=0x559601a06460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.760+0000 7f1334e00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x559601a06460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.760+0000 7f1334e00700 1 -- 172.21.15.28:0/1038922546 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 msgr2=0x559601a7bf40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:15.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.760+0000 7f1334e00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x559601a7bf40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.760+0000 7f1334e00700 1 -- 172.21.15.28:0/1038922546 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x559601a069a0 con 0x5596019f97c0 2024-05-20T20:15:15.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.760+0000 7f132fe00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x559601a7bf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:15.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.761+0000 7f1334e00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x559601a05f20 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f13240104f0 tx=0x7f1324010610 comp rx=0 tx=0).ready entity=mon.1 client_cookie=47a7820d6df9d080 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:15.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.761+0000 7f1316a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:15.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.761+0000 7f1316a00700 1 -- 172.21.15.28:0/1038922546 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f132402e670 con 0x5596019f97c0 2024-05-20T20:15:15.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.762+0000 7f1316a00700 1 -- 172.21.15.28:0/1038922546 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(88..88 src has 1..88) ==== 8586+0+0 (secure 0 0 0) 0x7f132402e810 con 0x5596019f97c0 2024-05-20T20:15:15.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.762+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1324033070 con 0x5596019f97c0 2024-05-20T20:15:15.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.762+0000 7f1316a00700 1 -- 172.21.15.28:0/1038922546 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f132400de50 con 0x5596019f97c0 2024-05-20T20:15:15.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.762+0000 7f1342218b80 10 client.27355 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:15.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.762+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x559601a7b7b0 con 0x5596019f97c0 2024-05-20T20:15:15.768 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.765+0000 7f1316a00700 1 -- 172.21.15.28:0/1038922546 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1324033070 con 0x5596019f97c0 2024-05-20T20:15:15.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.766+0000 7f1316a00700 1 -- 172.21.15.28:0/1038922546 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1324005000 con 0x5596019f97c0 2024-05-20T20:15:15.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.766+0000 7f1342218b80 10 client.27355 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:15.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.766+0000 7f1342218b80 10 client.27355 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:15:15.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.766+0000 7f1342218b80 20 client.27355 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:15.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.766+0000 7f1342218b80 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a0f080 0x559601a114a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:15.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.766+0000 7f1342218b80 4 client.27355 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:15:15.769 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.766+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55960194d570 con 0x559601a0f080 2024-05-20T20:15:15.770 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.769+0000 7f132fe00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a0f080 0x559601a114a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:15.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.771+0000 7f1316a00700 10 client.27355 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:15:15.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.771+0000 7f132fe00700 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a0f080 0x559601a114a0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=15cebe84994137d2 server_cookie=c556f8928c5a2cc2 in_seq=0 out_seq=0 2024-05-20T20:15:15.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.771+0000 7f1316a00700 1 -- 172.21.15.28:0/1038922546 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55960194d570 con 0x559601a0f080 2024-05-20T20:15:15.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.771+0000 7f1316a00700 10 client.27355 handle_command_reply: tid=0 2024-05-20T20:15:15.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.771+0000 7f1342218b80 1 client.27355 shutdown 2024-05-20T20:15:15.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.772+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 msgr2=0x559601a05f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.773 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.772+0000 7f1342218b80 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x559601a05f20 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f13240104f0 tx=0x7f1324010610 comp rx=0 tx=0).stop 2024-05-20T20:15:15.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 shutdown_connections 2024-05-20T20:15:15.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1342218b80 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x559601a0f080 0x559601a114a0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1342218b80 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5596019fa0e0 0x559601a06460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.775 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1342218b80 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5596019f4390 0x559601a7bf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1342218b80 1 --2- 172.21.15.28:0/1038922546 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5596019f97c0 0x559601a05f20 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1316a00700 0 client.27355 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:15:15.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 >> 172.21.15.28:0/1038922546 conn(0x559601921be0 msgr2=0x5596019f8280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:15.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.774+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 shutdown_connections 2024-05-20T20:15:15.776 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.775+0000 7f1342218b80 1 -- 172.21.15.28:0/1038922546 wait complete. 2024-05-20T20:15:15.778 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.775+0000 7f1342218b80 20 client.27355 trim_cache size 0 max 16384 2024-05-20T20:15:15.780 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:15:15.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.781+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13180b2c00 msgr2=0x7f13180b50c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.781+0000 7f1336200700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13180b2c00 0x7f13180b50c0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f13300693f0 tx=0x7f1330069f60 comp rx=0 tx=0).stop 2024-05-20T20:15:15.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132812e700 msgr2=0x7f132812ebb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:15.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132812e700 0x7f132812ebb0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f1330000eb0 tx=0x7f1330075e20 comp rx=0 tx=0).stop 2024-05-20T20:15:15.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 shutdown_connections 2024-05-20T20:15:15.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132812e700 0x7f132812ebb0 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f13280958a0 0x7f1328134630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13180b2c00 0x7f13180b50c0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 --2- 172.21.15.28:0/2604623600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132809aca0 0x7f1328134b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:15.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.782+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 >> 172.21.15.28:0/2604623600 conn(0x7f132800a7a0 msgr2=0x7f1328093a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:15.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.783+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 shutdown_connections 2024-05-20T20:15:15.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:15.783+0000 7f1336200700 1 -- 172.21.15.28:0/2604623600 wait complete. 2024-05-20T20:15:16.956 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:15:17.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:17 smithi028 ceph-mon[31178]: pgmap v299: 97 pgs: 4 active+clean+snaptrim, 83 active+clean, 10 active+clean+snaptrim_wait; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 276 MiB/s wr, 2.06k op/s 2024-05-20T20:15:17.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:17 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:15:17.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:17 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:17.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:17 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:17 smithi154 ceph-mon[36314]: pgmap v299: 97 pgs: 4 active+clean+snaptrim, 83 active+clean, 10 active+clean+snaptrim_wait; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 276 MiB/s wr, 2.06k op/s 2024-05-20T20:15:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:17 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:15:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:17 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:17 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:17.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:17 smithi033 ceph-mon[36179]: pgmap v299: 97 pgs: 4 active+clean+snaptrim, 83 active+clean, 10 active+clean+snaptrim_wait; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 276 MiB/s wr, 2.06k op/s 2024-05-20T20:15:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:17 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:15:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:17 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:17 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:17.782 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:18.221 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:18 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:18.222 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:18 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:18.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:18 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:18.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:18 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:18.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:18 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:18.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:18 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:19.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:19 smithi028 ceph-mon[31178]: pgmap v300: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 281 MiB/s wr, 2.20k op/s 2024-05-20T20:15:19.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:19 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:15:19.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:19 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:15:19.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:19 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:19 smithi154 ceph-mon[36314]: pgmap v300: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 281 MiB/s wr, 2.20k op/s 2024-05-20T20:15:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:19 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:15:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:19 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:15:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:19 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:19.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:19 smithi033 ceph-mon[36179]: pgmap v300: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 281 MiB/s wr, 2.20k op/s 2024-05-20T20:15:19.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:19 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:15:19.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:19 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:15:19.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:19 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:15:20.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:20 smithi028 ceph-mon[31178]: pgmap v301: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 307 MiB/s wr, 2.59k op/s 2024-05-20T20:15:20.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:20 smithi154 ceph-mon[36314]: pgmap v301: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 307 MiB/s wr, 2.59k op/s 2024-05-20T20:15:20.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:20 smithi033 ceph-mon[36179]: pgmap v301: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 307 MiB/s wr, 2.59k op/s 2024-05-20T20:15:21.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:21 smithi028 ceph-mon[31178]: osdmap e89: 12 total, 12 up, 12 in 2024-05-20T20:15:21.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:21 smithi154 ceph-mon[36314]: osdmap e89: 12 total, 12 up, 12 in 2024-05-20T20:15:21.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:21 smithi033 ceph-mon[36179]: osdmap e89: 12 total, 12 up, 12 in 2024-05-20T20:15:22.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:22 smithi154 ceph-mon[36314]: pgmap v303: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 259 MiB/s wr, 2.40k op/s 2024-05-20T20:15:22.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:22 smithi033 ceph-mon[36179]: pgmap v303: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 259 MiB/s wr, 2.40k op/s 2024-05-20T20:15:23.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:22 smithi028 ceph-mon[31178]: pgmap v303: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 259 MiB/s wr, 2.40k op/s 2024-05-20T20:15:25.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.221+0000 7f3c61800640 1 -- 172.21.15.28:0/1982403944 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10f540 msgr2=0x7f3c5c10f920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.221+0000 7f3c61800640 1 --2- 172.21.15.28:0/1982403944 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10f540 0x7f3c5c10f920 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f3c4c007510 tx=0x7f3c4c02b090 comp rx=0 tx=0).stop 2024-05-20T20:15:25.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.224+0000 7f3c61800640 1 -- 172.21.15.28:0/1982403944 shutdown_connections 2024-05-20T20:15:25.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.224+0000 7f3c61800640 1 --2- 172.21.15.28:0/1982403944 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c5c10a410 0x7f3c5c116300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.224+0000 7f3c61800640 1 --2- 172.21.15.28:0/1982403944 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c5c109a50 0x7f3c5c109ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.224+0000 7f3c61800640 1 --2- 172.21.15.28:0/1982403944 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10f540 0x7f3c5c10f920 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.225+0000 7f3c61800640 1 -- 172.21.15.28:0/1982403944 >> 172.21.15.28:0/1982403944 conn(0x7f3c5c06db20 msgr2=0x7f3c5c06df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:25.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.225+0000 7f3c61800640 1 -- 172.21.15.28:0/1982403944 shutdown_connections 2024-05-20T20:15:25.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.225+0000 7f3c61800640 1 -- 172.21.15.28:0/1982403944 wait complete. 2024-05-20T20:15:25.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.227+0000 7f3c61800640 1 Processor -- start 2024-05-20T20:15:25.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.228+0000 7f3c61800640 1 -- start start 2024-05-20T20:15:25.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.230+0000 7f3c61800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c5c109a50 0x7f3c5c114ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.231+0000 7f3c61800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10a410 0x7f3c5c124500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.231+0000 7f3c61800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c5c10f540 0x7f3c5c115220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.231+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10a410 0x7f3c5c124500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.231+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10a410 0x7f3c5c124500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:39970/0 (socket says 172.21.15.28:39970) 2024-05-20T20:15:25.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.231+0000 7f3c5a000640 1 -- 172.21.15.28:0/2322647247 learned_addr learned my addr 172.21.15.28:0/2322647247 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:25.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.231+0000 7f3c5b400640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c5c10f540 0x7f3c5c115220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.231+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c118880 con 0x7f3c5c109a50 2024-05-20T20:15:25.238 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.232+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c118700 con 0x7f3c5c10a410 2024-05-20T20:15:25.238 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.232+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c118a00 con 0x7f3c5c10f540 2024-05-20T20:15:25.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.235+0000 7f3c5b400640 1 -- 172.21.15.28:0/2322647247 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10a410 msgr2=0x7f3c5c124500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.235+0000 7f3c5b400640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10a410 0x7f3c5c124500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.235+0000 7f3c5b400640 1 -- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c5c109a50 msgr2=0x7f3c5c114ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.235+0000 7f3c5b400640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c5c109a50 0x7f3c5c114ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.241 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.235+0000 7f3c5b400640 1 -- 172.21.15.28:0/2322647247 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3c5c1bca00 con 0x7f3c5c10f540 2024-05-20T20:15:25.242 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.241+0000 7f3c5b400640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c5c10f540 0x7f3c5c115220 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f3c4800e970 tx=0x7f3c4800ee30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:25.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.242+0000 7f3c53400640 1 -- 172.21.15.28:0/2322647247 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c4800ccc0 con 0x7f3c5c10f540 2024-05-20T20:15:25.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.242+0000 7f3c53400640 1 -- 172.21.15.28:0/2322647247 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3c48005020 con 0x7f3c5c10f540 2024-05-20T20:15:25.243 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.243+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3c5c1bcc30 con 0x7f3c5c10f540 2024-05-20T20:15:25.245 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.243+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3c5c1bd130 con 0x7f3c5c10f540 2024-05-20T20:15:25.245 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.245+0000 7f3c53400640 1 -- 172.21.15.28:0/2322647247 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c48004a00 con 0x7f3c5c10f540 2024-05-20T20:15:25.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.260+0000 7f3c53400640 1 -- 172.21.15.28:0/2322647247 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3c48004ba0 con 0x7f3c5c10f540 2024-05-20T20:15:25.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.263+0000 7f3c53400640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c400b2a50 0x7f3c400b4f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.265+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c400b2a50 0x7f3c400b4f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.266+0000 7f3c53400640 1 -- 172.21.15.28:0/2322647247 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f3c480e2690 con 0x7f3c5c10f540 2024-05-20T20:15:25.270 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.269+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c400b2a50 0x7f3c400b4f10 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f3c4c007320 tx=0x7f3c4c000bc0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:25.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.279+0000 7f3c645f1740 1 Processor -- start 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.282+0000 7f3c645f1740 1 -- start start 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.282+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb7c90 0x563596cb8070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.282+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb85b0 0x563596cc2930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.282+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc5260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.282+0000 7f3c645f1740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563596c28b90 con 0x563596cc2e70 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.282+0000 7f3c645f1740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563596c18730 con 0x563596cb7c90 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.282+0000 7f3c645f1740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563596c1c770 con 0x563596cb85b0 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.283+0000 7f3c5aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb7c90 0x563596cb8070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.283+0000 7f3c5aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb7c90 0x563596cb8070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:39986/0 (socket says 172.21.15.28:39986) 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.283+0000 7f3c5aa00640 1 -- 172.21.15.28:0/1406814650 learned_addr learned my addr 172.21.15.28:0/1406814650 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:25.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.283+0000 7f3c5b400640 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc5260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.284+0000 7f3c5aa00640 1 -- 172.21.15.28:0/1406814650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb85b0 msgr2=0x563596cc2930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:15:25.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.284+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb85b0 0x563596cc2930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.284+0000 7f3c5aa00640 1 -- 172.21.15.28:0/1406814650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 msgr2=0x563596cc5260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:25.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.284+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc5260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.284+0000 7f3c5aa00640 1 -- 172.21.15.28:0/1406814650 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563596bb18b0 con 0x563596cb7c90 2024-05-20T20:15:25.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.286+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb7c90 0x563596cb8070 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f3c4c0047c0 tx=0x7f3c4c039000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bf1c17139df0b901 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:25.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.286+0000 7f3c47e00640 1 -- 172.21.15.28:0/1406814650 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c4c04d070 con 0x563596cb7c90 2024-05-20T20:15:25.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.287+0000 7f3c47e00640 1 -- 172.21.15.28:0/1406814650 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3c4c02b990 con 0x563596cb7c90 2024-05-20T20:15:25.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.287+0000 7f3c47e00640 1 -- 172.21.15.28:0/1406814650 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c4c003970 con 0x563596cb7c90 2024-05-20T20:15:25.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.288+0000 7f3c645f1740 1 -- 172.21.15.28:0/1406814650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb7c90 msgr2=0x563596cb8070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.288+0000 7f3c645f1740 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb7c90 0x563596cb8070 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f3c4c0047c0 tx=0x7f3c4c039000 comp rx=0 tx=0).stop 2024-05-20T20:15:25.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.290+0000 7f3c645f1740 1 -- 172.21.15.28:0/1406814650 shutdown_connections 2024-05-20T20:15:25.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.290+0000 7f3c645f1740 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc5260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.291+0000 7f3c645f1740 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb85b0 0x563596cc2930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.291+0000 7f3c645f1740 1 --2- 172.21.15.28:0/1406814650 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb7c90 0x563596cb8070 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.291+0000 7f3c645f1740 1 -- 172.21.15.28:0/1406814650 >> 172.21.15.28:0/1406814650 conn(0x563596cb41a0 msgr2=0x563596cb4580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:25.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.292+0000 7f3c645f1740 1 -- 172.21.15.28:0/1406814650 shutdown_connections 2024-05-20T20:15:25.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.292+0000 7f3c645f1740 1 -- 172.21.15.28:0/1406814650 wait complete. 2024-05-20T20:15:25.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.293+0000 7f3c645f1740 1 -- start start 2024-05-20T20:15:25.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.293+0000 7f3c645f1740 1 Processor -- start 2024-05-20T20:15:25.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.297+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb7c90 0x563596cc28c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.297+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cbfbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.297+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc02f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.297+0000 7f3c645f1740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5635968fe790 con 0x563596cc2e70 2024-05-20T20:15:25.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.297+0000 7f3c645f1740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563596bc0bc0 con 0x563596cb85b0 2024-05-20T20:15:25.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.297+0000 7f3c645f1740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563596cc5a40 con 0x563596cb7c90 2024-05-20T20:15:25.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c645f1740 10 client.0 _ll_register_callbacks cb 0x563596b9e400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:25.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb7c90 0x563596cc28c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cbfbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cbfbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:40000/0 (socket says 172.21.15.28:40000) 2024-05-20T20:15:25.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5a000640 1 -- 172.21.15.28:0/211957738 learned_addr learned my addr 172.21.15.28:0/211957738 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:25.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5aa00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb7c90 0x563596cc28c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35734/0 (socket says 172.21.15.28:35734) 2024-05-20T20:15:25.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5a000640 1 -- 172.21.15.28:0/211957738 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb7c90 msgr2=0x563596cc28c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:25.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5a000640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb7c90 0x563596cc28c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.298+0000 7f3c5a000640 1 -- 172.21.15.28:0/211957738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 msgr2=0x563596cc02f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:15:25.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.300+0000 7f3c5b400640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc02f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.300+0000 7f3c5a000640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc02f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.300+0000 7f3c5a000640 1 -- 172.21.15.28:0/211957738 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56359692d210 con 0x563596cb85b0 2024-05-20T20:15:25.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.300+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb7c90 0x563596cc28c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:25.302 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.300+0000 7f3c5b400640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc02f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:25.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.301+0000 7f3c5a000640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cbfbb0 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f3c5400e9c0 tx=0x7f3c5400edc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5891f5a6cae6d8b9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:25.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.301+0000 7f3c44c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:25.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.301+0000 7f3c44c00640 1 -- 172.21.15.28:0/211957738 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c5400cd80 con 0x563596cb85b0 2024-05-20T20:15:25.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.302+0000 7f3c44c00640 1 -- 172.21.15.28:0/211957738 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f3c54004b20 con 0x563596cb85b0 2024-05-20T20:15:25.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.302+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3c4c02b990 con 0x563596cb85b0 2024-05-20T20:15:25.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.302+0000 7f3c44c00640 1 -- 172.21.15.28:0/211957738 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3c54004d20 con 0x563596cb85b0 2024-05-20T20:15:25.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.302+0000 7f3c645f1740 10 client.27449 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:25.304 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.303+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5635969f9620 con 0x563596cb85b0 2024-05-20T20:15:25.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.303+0000 7f3c44c00640 1 -- 172.21.15.28:0/211957738 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3c540054d0 con 0x563596cb85b0 2024-05-20T20:15:25.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.305+0000 7f3c44c00640 1 -- 172.21.15.28:0/211957738 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3c540105c0 con 0x563596cb85b0 2024-05-20T20:15:25.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.305+0000 7f3c645f1740 10 client.27449 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:25.306 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.305+0000 7f3c645f1740 10 client.27449 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:25.307 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.305+0000 7f3c645f1740 20 client.27449 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:25.307 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.305+0000 7f3c645f1740 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596cd28c0 0x563596cd4cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.307 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.305+0000 7f3c645f1740 4 client.27449 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:25.307 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.305+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7f3c4c02b990 con 0x563596cd28c0 2024-05-20T20:15:25.307 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.306+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596cd28c0 0x563596cd4cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.307+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596cd28c0 0x563596cd4cb0 crc :-1 s=READY pgs=211 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f0e300de4b13e50b server_cookie=dba7135d7d37e353 in_seq=0 out_seq=0 2024-05-20T20:15:25.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.307+0000 7f3c44c00640 10 client.27449 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:25.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.310+0000 7f3c44c00640 1 -- 172.21.15.28:0/211957738 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x7f3c4c02b990 con 0x563596cd28c0 2024-05-20T20:15:25.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.310+0000 7f3c44c00640 10 client.27449 handle_command_reply: tid=0 2024-05-20T20:15:25.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.310+0000 7f3c645f1740 1 client.27449 shutdown 2024-05-20T20:15:25.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.317+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 msgr2=0x563596cbfbb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.317+0000 7f3c645f1740 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cbfbb0 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f3c5400e9c0 tx=0x7f3c5400edc0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.318+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 shutdown_connections 2024-05-20T20:15:25.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.318+0000 7f3c645f1740 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596cd28c0 0x563596cd4cb0 crc :-1 s=READY pgs=211 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.318+0000 7f3c645f1740 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cbfbb0 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.318+0000 7f3c645f1740 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cc2e70 0x563596cc02f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.318+0000 7f3c645f1740 1 --2- 172.21.15.28:0/211957738 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cb7c90 0x563596cc28c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.318+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 >> 172.21.15.28:0/211957738 conn(0x563596cb41a0 msgr2=0x563596cb9ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:25.319 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.319+0000 7f3c44c00640 0 client.27449 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:25.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.322+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 shutdown_connections 2024-05-20T20:15:25.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.322+0000 7f3c645f1740 1 -- 172.21.15.28:0/211957738 wait complete. 2024-05-20T20:15:25.323 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.322+0000 7f3c645f1740 20 client.27449 trim_cache size 0 max 16384 2024-05-20T20:15:25.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.374+0000 7f3c645f1740 1 Processor -- start 2024-05-20T20:15:25.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:25 smithi154 ceph-mon[36314]: pgmap v304: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 245 MiB/s wr, 2.45k op/s 2024-05-20T20:15:25.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:25 smithi033 ceph-mon[36179]: pgmap v304: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 245 MiB/s wr, 2.45k op/s 2024-05-20T20:15:25.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.374+0000 7f3c645f1740 1 -- start start 2024-05-20T20:15:25.378 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.378+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 0x563596cbeaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.379+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cb3a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.379+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc1af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.379+0000 7f3c645f1740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c1c5de0 con 0x563596cb7c90 2024-05-20T20:15:25.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.379+0000 7f3c645f1740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c1c7920 con 0x563596cb85b0 2024-05-20T20:15:25.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.379+0000 7f3c645f1740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c1c64b0 con 0x563596cc2e70 2024-05-20T20:15:25.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.380+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cb3a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.380+0000 7f3c5aa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 0x563596cbeaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.380+0000 7f3c5b400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc1af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.380+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cb3a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:40012/0 (socket says 172.21.15.28:40012) 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.380+0000 7f3c5a000640 1 -- 172.21.15.28:0/815160120 learned_addr learned my addr 172.21.15.28:0/815160120 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.381+0000 7f3c5b400640 1 -- 172.21.15.28:0/815160120 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 msgr2=0x563596cb3a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.381+0000 7f3c5b400640 1 --2- 172.21.15.28:0/815160120 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cb3a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.381+0000 7f3c5b400640 1 -- 172.21.15.28:0/815160120 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 msgr2=0x563596cbeaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.381+0000 7f3c5b400640 1 --2- 172.21.15.28:0/815160120 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 0x563596cbeaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.381+0000 7f3c5b400640 1 -- 172.21.15.28:0/815160120 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563596cc0660 con 0x563596cc2e70 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.382+0000 7f3c5b400640 1 --2- 172.21.15.28:0/815160120 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc1af0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f3c480dfdd0 tx=0x7f3c480a90d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=628421b197713f1b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:25.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.382+0000 7f3c47400640 1 -- 172.21.15.28:0/815160120 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c4801c040 con 0x563596cc2e70 2024-05-20T20:15:25.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.382+0000 7f3c47400640 1 -- 172.21.15.28:0/815160120 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3c480a9470 con 0x563596cc2e70 2024-05-20T20:15:25.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.382+0000 7f3c47400640 1 -- 172.21.15.28:0/815160120 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c480e0050 con 0x563596cc2e70 2024-05-20T20:15:25.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.384+0000 7f3c645f1740 1 -- 172.21.15.28:0/815160120 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 msgr2=0x563596cc1af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.385+0000 7f3c645f1740 1 --2- 172.21.15.28:0/815160120 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc1af0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f3c480dfdd0 tx=0x7f3c480a90d0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 -- 172.21.15.28:0/815160120 shutdown_connections 2024-05-20T20:15:25.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 --2- 172.21.15.28:0/815160120 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596cb3a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 --2- 172.21.15.28:0/815160120 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc1af0 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 --2- 172.21.15.28:0/815160120 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 0x563596cbeaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 -- 172.21.15.28:0/815160120 >> 172.21.15.28:0/815160120 conn(0x563596cb41a0 msgr2=0x563596cc13e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:25.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 -- 172.21.15.28:0/815160120 shutdown_connections 2024-05-20T20:15:25.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 -- 172.21.15.28:0/815160120 wait complete. 2024-05-20T20:15:25.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.388+0000 7f3c645f1740 1 -- start start 2024-05-20T20:15:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.391+0000 7f3c645f1740 1 Processor -- start 2024-05-20T20:15:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.391+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 0x563596ccaf90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.391+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596ccb4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.392+0000 7f3c645f1740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc8520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.392+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596ccb4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.392+0000 7f3c5a000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596ccb4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:40016/0 (socket says 172.21.15.28:40016) 2024-05-20T20:15:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.392+0000 7f3c5a000640 1 -- 172.21.15.28:0/3295673237 learned_addr learned my addr 172.21.15.28:0/3295673237 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.392+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c1c7250 con 0x563596cb7c90 2024-05-20T20:15:25.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.392+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c1c6b80 con 0x563596cb85b0 2024-05-20T20:15:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.392+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3c5c1c3de0 con 0x563596cc2e70 2024-05-20T20:15:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.393+0000 7f3c645f1740 10 client.0 _ll_register_callbacks cb 0x7f3c54007640 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.393+0000 7f3c5b400640 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc8520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.394 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.393+0000 7f3c5a000640 1 -- 172.21.15.28:0/3295673237 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 msgr2=0x563596cc8520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:25.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.393+0000 7f3c5a000640 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc8520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.393+0000 7f3c5a000640 1 -- 172.21.15.28:0/3295673237 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 msgr2=0x563596ccaf90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:25.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.393+0000 7f3c5a000640 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 0x563596ccaf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.393+0000 7f3c5a000640 1 -- 172.21.15.28:0/3295673237 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563596cc8d20 con 0x563596cb85b0 2024-05-20T20:15:25.397 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.396+0000 7f3c5b400640 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc8520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:25.398 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.397+0000 7f3c44c00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:25.398 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.397+0000 7f3c5a000640 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596ccb4d0 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f3c540109e0 tx=0x7f3c54004c90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2be2a9142c0acc18 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:25.398 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.398+0000 7f3c44c00640 1 -- 172.21.15.28:0/3295673237 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c54030020 con 0x563596cb85b0 2024-05-20T20:15:25.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.398+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3c480a9470 con 0x563596cb85b0 2024-05-20T20:15:25.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.398+0000 7f3c44c00640 1 -- 172.21.15.28:0/3295673237 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f3c54010430 con 0x563596cb85b0 2024-05-20T20:15:25.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.399+0000 7f3c44c00640 1 -- 172.21.15.28:0/3295673237 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3c5403b050 con 0x563596cb85b0 2024-05-20T20:15:25.401 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.400+0000 7f3c645f1740 10 client.27455 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:25.401 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.400+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563596d3d080 con 0x563596cb85b0 2024-05-20T20:15:25.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.400+0000 7f3c44c00640 1 -- 172.21.15.28:0/3295673237 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3c480a9470 con 0x563596cb85b0 2024-05-20T20:15:25.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.403+0000 7f3c44c00640 1 -- 172.21.15.28:0/3295673237 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3c5402cb30 con 0x563596cb85b0 2024-05-20T20:15:25.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.405+0000 7f3c645f1740 10 client.27455 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:25.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.405+0000 7f3c645f1740 10 client.27455 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:25.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.405+0000 7f3c645f1740 20 client.27455 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:25.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.405+0000 7f3c645f1740 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596d3d850 0x563596d3fc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:25.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.405+0000 7f3c645f1740 4 client.27455 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:15:25.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.405+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x563596c2aae0 con 0x563596d3d850 2024-05-20T20:15:25.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.410+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596d3d850 0x563596d3fc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:25.412 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.411+0000 7f3c5aa00640 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596d3d850 0x563596d3fc70 crc :-1 s=READY pgs=212 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4e04dbc289dd6459 server_cookie=a3079c7f054aee21 in_seq=0 out_seq=0 2024-05-20T20:15:25.414 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.413+0000 7f3c44c00640 10 client.27455 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:25.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.499+0000 7f3c44c00640 1 -- 172.21.15.28:0/3295673237 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x563596c2aae0 con 0x563596d3d850 2024-05-20T20:15:25.499 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.499+0000 7f3c44c00640 10 client.27455 handle_command_reply: tid=0 2024-05-20T20:15:25.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.499+0000 7f3c645f1740 1 client.27455 shutdown 2024-05-20T20:15:25.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 msgr2=0x563596ccb4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.500 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596ccb4d0 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f3c540109e0 tx=0x7f3c54004c90 comp rx=0 tx=0).stop 2024-05-20T20:15:25.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 shutdown_connections 2024-05-20T20:15:25.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563596d3d850 0x563596d3fc70 crc :-1 s=READY pgs=212 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c44c00640 0 client.27455 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:25.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563596cb85b0 0x563596ccb4d0 unknown :-1 s=CLOSED pgs=521 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563596cc2e70 0x563596cc8520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 --2- 172.21.15.28:0/3295673237 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563596cb7c90 0x563596ccaf90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.501 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 >> 172.21.15.28:0/3295673237 conn(0x563596cb41a0 msgr2=0x563596cc1c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:25.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 shutdown_connections 2024-05-20T20:15:25.503 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.500+0000 7f3c645f1740 1 -- 172.21.15.28:0/3295673237 wait complete. 2024-05-20T20:15:25.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.504+0000 7f3c645f1740 20 client.27455 trim_cache size 0 max 16384 2024-05-20T20:15:25.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:15:25.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:15:25.504 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:15:25.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.506+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c400b2a50 msgr2=0x7f3c400b4f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.506+0000 7f3c61800640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c400b2a50 0x7f3c400b4f10 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f3c4c007320 tx=0x7f3c4c000bc0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.506+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c5c10f540 msgr2=0x7f3c5c115220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:25.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.506+0000 7f3c61800640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c5c10f540 0x7f3c5c115220 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f3c4800e970 tx=0x7f3c4800ee30 comp rx=0 tx=0).stop 2024-05-20T20:15:25.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.510+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 shutdown_connections 2024-05-20T20:15:25.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.510+0000 7f3c61800640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c5c10a410 0x7f3c5c124500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.510+0000 7f3c61800640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c400b2a50 0x7f3c400b4f10 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.510+0000 7f3c61800640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c5c10f540 0x7f3c5c115220 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.510+0000 7f3c61800640 1 --2- 172.21.15.28:0/2322647247 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c5c109a50 0x7f3c5c114ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:25.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.510+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 >> 172.21.15.28:0/2322647247 conn(0x7f3c5c06db20 msgr2=0x7f3c5c10e7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:25.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.516+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 shutdown_connections 2024-05-20T20:15:25.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:25.516+0000 7f3c61800640 1 -- 172.21.15.28:0/2322647247 wait complete. 2024-05-20T20:15:25.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:25 smithi028 ceph-mon[31178]: pgmap v304: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 245 MiB/s wr, 2.45k op/s 2024-05-20T20:15:26.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.488+0000 7f89ea200700 1 -- 172.21.15.28:0/1165985645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e410aa80 msgr2=0x7f89e410af00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.488+0000 7f89ea200700 1 --2- 172.21.15.28:0/1165985645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e410aa80 0x7f89e410af00 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f89d4009990 tx=0x7f89d4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.489+0000 7f89ea200700 1 -- 172.21.15.28:0/1165985645 shutdown_connections 2024-05-20T20:15:26.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.489+0000 7f89ea200700 1 --2- 172.21.15.28:0/1165985645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89e411cd20 0x7f89e411f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.489+0000 7f89ea200700 1 --2- 172.21.15.28:0/1165985645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89e4111d20 0x7f89e410a540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.489+0000 7f89ea200700 1 --2- 172.21.15.28:0/1165985645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e410aa80 0x7f89e410af00 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.489+0000 7f89ea200700 1 -- 172.21.15.28:0/1165985645 >> 172.21.15.28:0/1165985645 conn(0x7f89e406e6b0 msgr2=0x7f89e406eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:26.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.492+0000 7f89ea200700 1 -- 172.21.15.28:0/1165985645 shutdown_connections 2024-05-20T20:15:26.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.492+0000 7f89ea200700 1 -- 172.21.15.28:0/1165985645 wait complete. 2024-05-20T20:15:26.498 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.494+0000 7f89ea200700 1 Processor -- start 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.495+0000 7f89ea200700 1 -- start start 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.495+0000 7f89ea200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89e410aa80 0x7f89e4116960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.495+0000 7f89ea200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89e4111d20 0x7f89e4116ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.495+0000 7f89ea200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e411cd20 0x7f89e41173e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.496+0000 7f89daa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89e4111d20 0x7f89e4116ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.497+0000 7f89e3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e411cd20 0x7f89e41173e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.497+0000 7f89e3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e411cd20 0x7f89e41173e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56160/0 (socket says 172.21.15.28:56160) 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.497+0000 7f89e3e00700 1 -- 172.21.15.28:0/1334068138 learned_addr learned my addr 172.21.15.28:0/1334068138 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.497+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f89e4113300 con 0x7f89e411cd20 2024-05-20T20:15:26.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.497+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f89e4113180 con 0x7f89e4111d20 2024-05-20T20:15:26.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.497+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f89e4113480 con 0x7f89e410aa80 2024-05-20T20:15:26.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.496+0000 7f89daa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89e4111d20 0x7f89e4116ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:40032/0 (socket says 172.21.15.28:40032) 2024-05-20T20:15:26.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.498+0000 7f89e3e00700 1 -- 172.21.15.28:0/1334068138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89e410aa80 msgr2=0x7f89e4116960 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:15:26.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.498+0000 7f89e3e00700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89e410aa80 0x7f89e4116960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.501 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.498+0000 7f89e3e00700 1 -- 172.21.15.28:0/1334068138 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89e4111d20 msgr2=0x7f89e4116ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.501 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.498+0000 7f89e3e00700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89e4111d20 0x7f89e4116ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.501 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.498+0000 7f89e3e00700 1 -- 172.21.15.28:0/1334068138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f89e41bd7c0 con 0x7f89e411cd20 2024-05-20T20:15:26.501 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.499+0000 7f89e3e00700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e411cd20 0x7f89e41173e0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f89dc00b7b0 tx=0x7f89dc00bb70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:26.501 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.500+0000 7f89e0c00700 1 -- 172.21.15.28:0/1334068138 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89dc00f840 con 0x7f89e411cd20 2024-05-20T20:15:26.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.500+0000 7f89e0c00700 1 -- 172.21.15.28:0/1334068138 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f89dc017400 con 0x7f89e411cd20 2024-05-20T20:15:26.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.501+0000 7f89e0c00700 1 -- 172.21.15.28:0/1334068138 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89dc00d610 con 0x7f89e411cd20 2024-05-20T20:15:26.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.501+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f89e41bdab0 con 0x7f89e411cd20 2024-05-20T20:15:26.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.501+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f89e41bdff0 con 0x7f89e411cd20 2024-05-20T20:15:26.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.512+0000 7f89f4af0b80 1 Processor -- start 2024-05-20T20:15:26.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.512+0000 7f89e0c00700 1 -- 172.21.15.28:0/1334068138 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f89dc0175b0 con 0x7f89e411cd20 2024-05-20T20:15:26.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.514+0000 7f89e0c00700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89c80b2b20 0x7f89c80b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.515+0000 7f89e0c00700 1 -- 172.21.15.28:0/1334068138 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f89dc0e2460 con 0x7f89e411cd20 2024-05-20T20:15:26.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.516+0000 7f89f4af0b80 1 -- start start 2024-05-20T20:15:26.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.516+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78524290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.516+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851e260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.516+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7851ece0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.517+0000 7f89daa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851e260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.517+0000 7f89daa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851e260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35772/0 (socket says 172.21.15.28:35772) 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.517+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bd7847ce40 con 0x55bd7851e880 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.517+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bd7846c9e0 con 0x55bd78523eb0 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.517+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bd78470a20 con 0x55bd785247d0 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.517+0000 7f89e3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7851ece0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78524290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7851ece0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56164/0 (socket says 172.21.15.28:56164) 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3e00700 1 -- 172.21.15.28:0/1001062281 learned_addr learned my addr 172.21.15.28:0/1001062281 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:26.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89daa00700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89c80b2b20 0x7f89c80b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3e00700 1 -- 172.21.15.28:0/1001062281 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 msgr2=0x55bd7851e260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3e00700 1 --2- 172.21.15.28:0/1001062281 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851e260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3e00700 1 -- 172.21.15.28:0/1001062281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 msgr2=0x55bd78524290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3e00700 1 --2- 172.21.15.28:0/1001062281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78524290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.518+0000 7f89e3e00700 1 -- 172.21.15.28:0/1001062281 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bd78435420 con 0x55bd7851e880 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.519+0000 7f89e3e00700 1 --2- 172.21.15.28:0/1001062281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7851ece0 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f89dc0099f0 tx=0x7f89dc0a9e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ec28f15ab75ad809 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.520+0000 7f89daa00700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89c80b2b20 0x7f89c80b4fe0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f89e4118120 tx=0x7f89d4009ca0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.520+0000 7f89d8c00700 1 -- 172.21.15.28:0/1001062281 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89dc0ada80 con 0x55bd7851e880 2024-05-20T20:15:26.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.520+0000 7f89d8c00700 1 -- 172.21.15.28:0/1001062281 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f89dc0b2530 con 0x55bd7851e880 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.521+0000 7f89f4af0b80 1 -- 172.21.15.28:0/1001062281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 msgr2=0x55bd7851ece0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.521+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/1001062281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7851ece0 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7f89dc0099f0 tx=0x7f89dc0a9e50 comp rx=0 tx=0).stop 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.520+0000 7f89d8c00700 1 -- 172.21.15.28:0/1001062281 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89dc0a9930 con 0x55bd7851e880 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.524+0000 7f89f4af0b80 1 -- 172.21.15.28:0/1001062281 shutdown_connections 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.524+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/1001062281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7851ece0 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.524+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/1001062281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78524290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.524+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/1001062281 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851e260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.524+0000 7f89f4af0b80 1 -- 172.21.15.28:0/1001062281 >> 172.21.15.28:0/1001062281 conn(0x55bd7844cbe0 msgr2=0x55bd78519010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.527+0000 7f89f4af0b80 1 -- 172.21.15.28:0/1001062281 shutdown_connections 2024-05-20T20:15:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.527+0000 7f89f4af0b80 1 -- 172.21.15.28:0/1001062281 wait complete. 2024-05-20T20:15:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.528+0000 7f89f4af0b80 1 -- start start 2024-05-20T20:15:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.528+0000 7f89f4af0b80 1 Processor -- start 2024-05-20T20:15:26.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.530+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7852a440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78527830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd78527d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bd78372fa0 con 0x55bd7851e880 2024-05-20T20:15:26.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bd7840d760 con 0x55bd78523eb0 2024-05-20T20:15:26.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bd7852b240 con 0x55bd785247d0 2024-05-20T20:15:26.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89f4af0b80 10 client.0 _ll_register_callbacks cb 0x55bd78416c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:26.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89e3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7852a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.531+0000 7f89e3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7852a440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56172/0 (socket says 172.21.15.28:56172) 2024-05-20T20:15:26.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.532+0000 7f89daa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78527830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.532+0000 7f89daa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78527830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:40052/0 (socket says 172.21.15.28:40052) 2024-05-20T20:15:26.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.532+0000 7f89daa00700 1 -- 172.21.15.28:0/3375526450 learned_addr learned my addr 172.21.15.28:0/3375526450 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:26.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.532+0000 7f89e3400700 1 -- 172.21.15.28:0/3375526450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 msgr2=0x55bd78527d70 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:15:26.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.532+0000 7f89e3400700 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd78527d70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.533+0000 7f89e3400700 1 -- 172.21.15.28:0/3375526450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 msgr2=0x55bd78527830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.533+0000 7f89e3400700 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78527830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.533+0000 7f89e3400700 1 -- 172.21.15.28:0/3375526450 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bd783bc8d0 con 0x55bd7851e880 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.533+0000 7f89daa00700 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78527830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.534+0000 7f89e3400700 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7852a440 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f89cc00e250 tx=0x7f89cc00e610 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e89bbfd770347b87 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.534+0000 7f89d1600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.534+0000 7f89d1600700 1 -- 172.21.15.28:0/3375526450 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89cc017070 con 0x55bd7851e880 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.534+0000 7f89d1600700 1 -- 172.21.15.28:0/3375526450 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f89cc013470 con 0x55bd7851e880 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.534+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f89dc0b2530 con 0x55bd7851e880 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.535+0000 7f89d1600700 1 -- 172.21.15.28:0/3375526450 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f89cc023050 con 0x55bd7851e880 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.535+0000 7f89f4af0b80 10 client.17001 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:26.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.535+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55bd7843d060 con 0x55bd7851e880 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89d1600700 1 -- 172.21.15.28:0/3375526450 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f89dc0b2530 con 0x55bd7851e880 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89d1600700 1 -- 172.21.15.28:0/3375526450 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f89cc0126f0 con 0x55bd7851e880 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89f4af0b80 10 client.17001 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89f4af0b80 10 client.17001 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89f4af0b80 20 client.17001 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd78537dc0 0x55bd7853a1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89f4af0b80 4 client.17001 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55bd7847ed90 con 0x55bd78537dc0 2024-05-20T20:15:26.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.536+0000 7f89daa00700 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd78537dc0 0x55bd7853a1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.537+0000 7f89daa00700 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd78537dc0 0x55bd7853a1f0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=5cbf98ebdaa45df4 server_cookie=8f42911af981214c in_seq=0 out_seq=0 2024-05-20T20:15:26.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.537+0000 7f89d1600700 10 client.17001 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:15:26.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.538+0000 7f89d1600700 1 -- 172.21.15.28:0/3375526450 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55bd7847ed90 con 0x55bd78537dc0 2024-05-20T20:15:26.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.538+0000 7f89d1600700 10 client.17001 handle_command_reply: tid=0 2024-05-20T20:15:26.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 client.17001 shutdown 2024-05-20T20:15:26.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 msgr2=0x55bd7852a440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7852a440 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7f89cc00e250 tx=0x7f89cc00e610 comp rx=0 tx=0).stop 2024-05-20T20:15:26.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 shutdown_connections 2024-05-20T20:15:26.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd78527d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd78537dc0 0x55bd7853a1f0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd7851e880 0x55bd7852a440 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3375526450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78523eb0 0x55bd78527830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89d1600700 0 client.17001 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:15:26.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.539+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 >> 172.21.15.28:0/3375526450 conn(0x55bd7844cbe0 msgr2=0x55bd7851d250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:26.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.540+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 shutdown_connections 2024-05-20T20:15:26.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.540+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3375526450 wait complete. 2024-05-20T20:15:26.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.540+0000 7f89f4af0b80 20 client.17001 trim_cache size 0 max 16384 2024-05-20T20:15:26.557 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 Processor -- start 2024-05-20T20:15:26.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 -- start start 2024-05-20T20:15:26.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd7851e260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851ec80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78526830 0x55bd785189e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f89e41c5660 con 0x55bd78523eb0 2024-05-20T20:15:26.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f89e41c71a0 con 0x55bd78526830 2024-05-20T20:15:26.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f89e41c5d30 con 0x55bd785247d0 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89daa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851ec80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89e3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd7851e260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.557+0000 7f89daa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851ec80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35784/0 (socket says 172.21.15.28:35784) 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89daa00700 1 -- 172.21.15.28:0/3308991981 learned_addr learned my addr 172.21.15.28:0/3308991981 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89e3e00700 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78526830 0x55bd785189e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89daa00700 1 -- 172.21.15.28:0/3308991981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78526830 msgr2=0x55bd785189e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89daa00700 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78526830 0x55bd785189e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89daa00700 1 -- 172.21.15.28:0/3308991981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 msgr2=0x55bd7851e260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89daa00700 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd7851e260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89daa00700 1 -- 172.21.15.28:0/3308991981 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bd78519d30 con 0x55bd785247d0 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89e3400700 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd7851e260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.558+0000 7f89daa00700 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851ec80 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f89d4009990 tx=0x7f89d4023000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8256cae516ec266f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:26.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89d3e00700 1 -- 172.21.15.28:0/3308991981 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89d4003880 con 0x55bd785247d0 2024-05-20T20:15:26.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89d3e00700 1 -- 172.21.15.28:0/3308991981 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f89d4032390 con 0x55bd785247d0 2024-05-20T20:15:26.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89d3e00700 1 -- 172.21.15.28:0/3308991981 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89d400f6c0 con 0x55bd785247d0 2024-05-20T20:15:26.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3308991981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 msgr2=0x55bd7851ec80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851ec80 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f89d4009990 tx=0x7f89d4023000 comp rx=0 tx=0).stop 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3308991981 shutdown_connections 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd78526830 0x55bd785189e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd7851e260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/3308991981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd785247d0 0x55bd7851ec80 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.559+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3308991981 >> 172.21.15.28:0/3308991981 conn(0x55bd7844cbe0 msgr2=0x55bd78526120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.560+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3308991981 shutdown_connections 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.560+0000 7f89f4af0b80 1 -- 172.21.15.28:0/3308991981 wait complete. 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.560+0000 7f89f4af0b80 1 -- start start 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.560+0000 7f89f4af0b80 1 Processor -- start 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd78530940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd785247d0 0x55bd7852d4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89f4af0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd78526830 0x55bd7852da30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f89e41c6ad0 con 0x55bd78523eb0 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f89e41c6400 con 0x55bd785247d0 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89f4af0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f89e41c3620 con 0x55bd78526830 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89f4af0b80 10 client.0 _ll_register_callbacks cb 0x55bd78416c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89daa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd785247d0 0x55bd7852d4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89e3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd78530940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89e3e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd78526830 0x55bd7852da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89daa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd785247d0 0x55bd7852d4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:40076/0 (socket says 172.21.15.28:40076) 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89e3400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd78530940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:56188/0 (socket says 172.21.15.28:56188) 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.561+0000 7f89daa00700 1 -- 172.21.15.28:0/18723419 learned_addr learned my addr 172.21.15.28:0/18723419 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89daa00700 1 -- 172.21.15.28:0/18723419 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd78526830 msgr2=0x55bd7852da30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:26.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89daa00700 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd78526830 0x55bd7852da30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89daa00700 1 -- 172.21.15.28:0/18723419 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 msgr2=0x55bd78530940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:26.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89daa00700 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd78530940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89daa00700 1 -- 172.21.15.28:0/18723419 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bd7852df70 con 0x55bd785247d0 2024-05-20T20:15:26.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89daa00700 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd785247d0 0x55bd7852d4f0 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f89d4003820 tx=0x7f89d400b770 comp rx=0 tx=0).ready entity=mon.2 client_cookie=489579dd7892e394 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:26.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89d1600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89d1600700 1 -- 172.21.15.28:0/18723419 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f89d4003a80 con 0x55bd785247d0 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f89d4032390 con 0x55bd785247d0 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.562+0000 7f89d1600700 1 -- 172.21.15.28:0/18723419 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f89d4003c20 con 0x55bd785247d0 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89d1600700 1 -- 172.21.15.28:0/18723419 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f89d4003e40 con 0x55bd785247d0 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 10 client.27470 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55bd785a36b0 con 0x55bd785247d0 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89d1600700 1 -- 172.21.15.28:0/18723419 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f89d4032390 con 0x55bd785247d0 2024-05-20T20:15:26.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89d1600700 1 -- 172.21.15.28:0/18723419 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f89d400f820 con 0x55bd785247d0 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 10 client.27470 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 10 client.27470 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 20 client.27470 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd785a3d00 0x55bd785a6120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 4 client.27470 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.563+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55bd78478570 con 0x55bd785a3d00 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.564+0000 7f89e3400700 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd785a3d00 0x55bd785a6120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.564+0000 7f89d1600700 10 client.27470 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.564+0000 7f89e3400700 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd785a3d00 0x55bd785a6120 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=33c65c507894992d server_cookie=82b400efe4852557 in_seq=0 out_seq=0 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.564+0000 7f89d1600700 1 -- 172.21.15.28:0/18723419 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55bd78478570 con 0x55bd785a3d00 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89d1600700 10 client.27470 handle_command_reply: tid=0 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 client.27470 shutdown 2024-05-20T20:15:26.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd785247d0 msgr2=0x55bd7852d4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd785247d0 0x55bd7852d4f0 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f89d4003820 tx=0x7f89d400b770 comp rx=0 tx=0).stop 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 shutdown_connections 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bd78526830 0x55bd7852da30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55bd785a3d00 0x55bd785a6120 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bd78523eb0 0x55bd78530940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 --2- 172.21.15.28:0/18723419 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bd785247d0 0x55bd7852d4f0 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89d1600700 0 client.27470 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 >> 172.21.15.28:0/18723419 conn(0x55bd7844cbe0 msgr2=0x55bd785189e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 shutdown_connections 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 1 -- 172.21.15.28:0/18723419 wait complete. 2024-05-20T20:15:26.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.565+0000 7f89f4af0b80 20 client.27470 trim_cache size 0 max 16384 2024-05-20T20:15:26.570 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:15:26.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89c80b2b20 msgr2=0x7f89c80b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89c80b2b20 0x7f89c80b4fe0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f89e4118120 tx=0x7f89d4009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.570 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e411cd20 msgr2=0x7f89e41173e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e411cd20 0x7f89e41173e0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f89dc00b7b0 tx=0x7f89dc00bb70 comp rx=0 tx=0).stop 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 shutdown_connections 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f89e411cd20 0x7f89e41173e0 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89e410aa80 0x7f89e4116960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f89c80b2b20 0x7f89c80b4fe0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 --2- 172.21.15.28:0/1334068138 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f89e4111d20 0x7f89e4116ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 >> 172.21.15.28:0/1334068138 conn(0x7f89e406e6b0 msgr2=0x7f89e411e4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 shutdown_connections 2024-05-20T20:15:26.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:26.567+0000 7f89ea200700 1 -- 172.21.15.28:0/1334068138 wait complete. 2024-05-20T20:15:27.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:27 smithi154 ceph-mon[36314]: pgmap v305: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 242 MiB/s wr, 2.40k op/s 2024-05-20T20:15:27.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:27 smithi033 ceph-mon[36179]: pgmap v305: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 242 MiB/s wr, 2.40k op/s 2024-05-20T20:15:27.395 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:27 smithi028 ceph-mon[31178]: pgmap v305: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 242 MiB/s wr, 2.40k op/s 2024-05-20T20:15:28.304 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:15:29.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:29 smithi033 ceph-mon[36179]: pgmap v306: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 233 MiB/s wr, 2.23k op/s 2024-05-20T20:15:29.434 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:29.585 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:29 smithi028 ceph-mon[31178]: pgmap v306: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 233 MiB/s wr, 2.23k op/s 2024-05-20T20:15:29.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:29 smithi154 ceph-mon[36314]: pgmap v306: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 233 MiB/s wr, 2.23k op/s 2024-05-20T20:15:31.459 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:31 smithi028 ceph-mon[31178]: pgmap v307: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 238 MiB/s wr, 1.97k op/s 2024-05-20T20:15:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:31 smithi154 ceph-mon[36314]: pgmap v307: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 238 MiB/s wr, 1.97k op/s 2024-05-20T20:15:31.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:31 smithi033 ceph-mon[36179]: pgmap v307: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 238 MiB/s wr, 1.97k op/s 2024-05-20T20:15:32.149 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 32.6 2024-05-20T20:15:32.150 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:15:33.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:33 smithi028 ceph-mon[31178]: pgmap v308: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 220 MiB/s wr, 1.80k op/s 2024-05-20T20:15:33.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:33 smithi154 ceph-mon[36314]: pgmap v308: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 220 MiB/s wr, 1.80k op/s 2024-05-20T20:15:33.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:33 smithi033 ceph-mon[36179]: pgmap v308: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 220 MiB/s wr, 1.80k op/s 2024-05-20T20:15:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:35 smithi028 ceph-mon[31178]: pgmap v309: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 302 MiB/s wr, 2.18k op/s 2024-05-20T20:15:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:35 smithi154 ceph-mon[36314]: pgmap v309: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 302 MiB/s wr, 2.18k op/s 2024-05-20T20:15:35.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:35 smithi033 ceph-mon[36179]: pgmap v309: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 302 MiB/s wr, 2.18k op/s 2024-05-20T20:15:36.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.665+0000 7fa7e4000700 1 -- 172.21.15.28:0/587272083 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc10a540 msgr2=0x7fa7dc10a920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.665+0000 7fa7e4000700 1 --2- 172.21.15.28:0/587272083 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc10a540 0x7fa7dc10a920 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7fa7c8009990 tx=0x7fa7c8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.665+0000 7fa7e4000700 1 -- 172.21.15.28:0/587272083 shutdown_connections 2024-05-20T20:15:36.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.665+0000 7fa7e4000700 1 --2- 172.21.15.28:0/587272083 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7dc116690 0x7fa7dc118a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.665+0000 7fa7e4000700 1 --2- 172.21.15.28:0/587272083 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc10a540 0x7fa7dc10a920 secure :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7fa7c8009990 tx=0x7fa7c8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.665+0000 7fa7e4000700 1 --2- 172.21.15.28:0/587272083 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7dc10ae60 0x7fa7dc116150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.665+0000 7fa7e4000700 1 -- 172.21.15.28:0/587272083 >> 172.21.15.28:0/587272083 conn(0x7fa7dc06e6b0 msgr2=0x7fa7dc06eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:36.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.673+0000 7fa7e4000700 1 -- 172.21.15.28:0/587272083 shutdown_connections 2024-05-20T20:15:36.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.673+0000 7fa7e4000700 1 -- 172.21.15.28:0/587272083 wait complete. 2024-05-20T20:15:36.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.677+0000 7fa7e4000700 1 Processor -- start 2024-05-20T20:15:36.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.681+0000 7fa7e4000700 1 -- start start 2024-05-20T20:15:36.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.682+0000 7fa7e4000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7dc10ae60 0x7fa7dc1a85a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.683+0000 7fa7e4000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7dc116690 0x7fa7dc1a8ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.683+0000 7fa7e1600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7dc10ae60 0x7fa7dc1a85a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.683+0000 7fa7e1600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7dc10ae60 0x7fa7dc1a85a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37998/0 (socket says 172.21.15.28:37998) 2024-05-20T20:15:36.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.685+0000 7fa7e4000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc1a2670 0x7fa7dc1a2b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.686+0000 7fa7e2000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc1a2670 0x7fa7dc1a2b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.686+0000 7fa7e2000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc1a2670 0x7fa7dc1a2b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52186/0 (socket says 172.21.15.28:52186) 2024-05-20T20:15:36.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.687+0000 7fa7e1600700 1 -- 172.21.15.28:0/2533396668 learned_addr learned my addr 172.21.15.28:0/2533396668 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:36.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.687+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc11c130 con 0x7fa7dc116690 2024-05-20T20:15:36.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.688+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc11bfb0 con 0x7fa7dc1a2670 2024-05-20T20:15:36.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.688+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc11c2b0 con 0x7fa7dc10ae60 2024-05-20T20:15:36.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.689+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7dc116690 0x7fa7dc1a8ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.689+0000 7fa7e2000700 1 -- 172.21.15.28:0/2533396668 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7dc10ae60 msgr2=0x7fa7dc1a85a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.689+0000 7fa7e2000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7dc10ae60 0x7fa7dc1a85a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.689+0000 7fa7e2000700 1 -- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7dc116690 msgr2=0x7fa7dc1a8ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.689+0000 7fa7e2000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7dc116690 0x7fa7dc1a8ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.689+0000 7fa7e2000700 1 -- 172.21.15.28:0/2533396668 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa7dc1a3290 con 0x7fa7dc1a2670 2024-05-20T20:15:36.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.690+0000 7fa7e2000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc1a2670 0x7fa7dc1a2b20 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7fa7d400be00 tx=0x7fa7d4009ed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:36.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.691+0000 7fa7da000700 1 -- 172.21.15.28:0/2533396668 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d4019070 con 0x7fa7dc1a2670 2024-05-20T20:15:36.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.691+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa7dc10bee0 con 0x7fa7dc1a2670 2024-05-20T20:15:36.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.691+0000 7fa7da000700 1 -- 172.21.15.28:0/2533396668 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa7d400cc10 con 0x7fa7dc1a2670 2024-05-20T20:15:36.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.692+0000 7fa7da000700 1 -- 172.21.15.28:0/2533396668 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d40125a0 con 0x7fa7dc1a2670 2024-05-20T20:15:36.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.693+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa7dc10c470 con 0x7fa7dc1a2670 2024-05-20T20:15:36.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.708+0000 7fa7da000700 1 -- 172.21.15.28:0/2533396668 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa7d401c070 con 0x7fa7dc1a2670 2024-05-20T20:15:36.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.716+0000 7fa7ee931b80 1 Processor -- start 2024-05-20T20:15:36.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.716+0000 7fa7ee931b80 1 -- start start 2024-05-20T20:15:36.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.717+0000 7fa7da000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa7c40b2a00 0x7fa7c40b4ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.719 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.719+0000 7fa7da000700 1 -- 172.21.15.28:0/2533396668 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fa7d40e1a90 con 0x7fa7dc1a2670 2024-05-20T20:15:36.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.721+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd6e20 0x561a10dd7200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10dd1330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd1950 0x561a10dd1dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7ee931b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561a10d26e40 con 0x561a10dd6e20 2024-05-20T20:15:36.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7ee931b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561a10d169e0 con 0x561a10dd7740 2024-05-20T20:15:36.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7ee931b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561a10d1aa20 con 0x561a10dd1950 2024-05-20T20:15:36.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa7c40b2a00 0x7fa7c40b4ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7e2000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd1950 0x561a10dd1dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7e2000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd1950 0x561a10dd1dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38012/0 (socket says 172.21.15.28:38012) 2024-05-20T20:15:36.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.722+0000 7fa7e2000700 1 -- 172.21.15.28:0/3985360754 learned_addr learned my addr 172.21.15.28:0/3985360754 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:36.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.723+0000 7fa7e1600700 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd6e20 0x561a10dd7200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.724+0000 7fa7e2000700 1 -- 172.21.15.28:0/3985360754 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 msgr2=0x561a10dd1330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7e2000700 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10dd1330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.725 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7e2000700 1 -- 172.21.15.28:0/3985360754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd6e20 msgr2=0x561a10dd7200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7e2000700 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd6e20 0x561a10dd7200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7e2000700 1 -- 172.21.15.28:0/3985360754 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561a10cdf420 con 0x561a10dd1950 2024-05-20T20:15:36.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7e2000700 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd1950 0x561a10dd1dd0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7fa7d400be30 tx=0x7fa7d40a8890 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4a88ef269df727ae server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:36.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7cea00700 1 -- 172.21.15.28:0/3985360754 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d40ad1c0 con 0x561a10dd1950 2024-05-20T20:15:36.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7cea00700 1 -- 172.21.15.28:0/3985360754 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa7d40b1c70 con 0x561a10dd1950 2024-05-20T20:15:36.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.725+0000 7fa7cea00700 1 -- 172.21.15.28:0/3985360754 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d40a4870 con 0x561a10dd1950 2024-05-20T20:15:36.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.726+0000 7fa7ee931b80 1 -- 172.21.15.28:0/3985360754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd1950 msgr2=0x561a10dd1dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.726+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd1950 0x561a10dd1dd0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7fa7d400be30 tx=0x7fa7d40a8890 comp rx=0 tx=0).stop 2024-05-20T20:15:36.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.727+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa7c40b2a00 0x7fa7c40b4ec0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fa7dc1a3e30 tx=0x7fa7d00093d0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:36.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.730+0000 7fa7ee931b80 1 -- 172.21.15.28:0/3985360754 shutdown_connections 2024-05-20T20:15:36.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.730+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd1950 0x561a10dd1dd0 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.730+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd6e20 0x561a10dd7200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.730 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.730+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/3985360754 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10dd1330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.731 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.730+0000 7fa7ee931b80 1 -- 172.21.15.28:0/3985360754 >> 172.21.15.28:0/3985360754 conn(0x561a10cf6be0 msgr2=0x561a10dc3690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.747+0000 7fa7ee931b80 1 -- 172.21.15.28:0/3985360754 shutdown_connections 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.747+0000 7fa7ee931b80 1 -- 172.21.15.28:0/3985360754 wait complete. 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.747+0000 7fa7ee931b80 1 -- start start 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.748+0000 7fa7ee931b80 1 Processor -- start 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10ddd470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dda810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10ddad50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7e0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dda810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7e0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dda810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38022/0 (socket says 172.21.15.28:38022) 2024-05-20T20:15:36.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7e0c00700 1 -- 172.21.15.28:0/580011870 learned_addr learned my addr 172.21.15.28:0/580011870 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561a10c1cfa0 con 0x561a10dd1950 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561a10cb7760 con 0x561a10dd7740 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561a10dde1f0 con 0x561a10dd6e20 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.749+0000 7fa7e2000700 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10ddad50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.750+0000 7fa7e0c00700 1 -- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 msgr2=0x561a10ddad50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.750+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10ddad50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.750+0000 7fa7e0c00700 1 -- 172.21.15.28:0/580011870 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 msgr2=0x561a10ddd470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.750+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10ddd470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.750+0000 7fa7e0c00700 1 -- 172.21.15.28:0/580011870 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561a10c668d0 con 0x561a10dd6e20 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.751+0000 7fa7ee931b80 10 client.0 _ll_register_callbacks cb 0x561a10cc0c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:36.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.751+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa7d40b1c70 con 0x561a10dd6e20 2024-05-20T20:15:36.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.753+0000 7fa7c3400700 10 client.27382 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:36.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.753+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dda810 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fa7d000dbd0 tx=0x7fa7d0004800 comp rx=0 tx=0).ready entity=mon.1 client_cookie=44ade29555370b1d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:36.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.755+0000 7fa7c3400700 1 -- 172.21.15.28:0/580011870 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d0024060 con 0x561a10dd6e20 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.756+0000 7fa7c3400700 1 -- 172.21.15.28:0/580011870 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fa7d000f040 con 0x561a10dd6e20 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.756+0000 7fa7c3400700 1 -- 172.21.15.28:0/580011870 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa7d0024060 con 0x561a10dd6e20 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.756+0000 7fa7ee931b80 10 client.27382 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.757+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x561a10ce7060 con 0x561a10dd6e20 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.758+0000 7fa7c3400700 1 -- 172.21.15.28:0/580011870 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa7d0013070 con 0x561a10dd6e20 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.758+0000 7fa7ee931b80 10 client.27382 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.758+0000 7fa7ee931b80 10 client.27382 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.758+0000 7fa7ee931b80 20 client.27382 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.758+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10deb050 0x561a10ded510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.758+0000 7fa7ee931b80 4 client.27382 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.758+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561a10d28d90 con 0x561a10deb050 2024-05-20T20:15:36.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.762+0000 7fa7e1600700 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10deb050 0x561a10ded510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.763+0000 7fa7c3400700 10 client.27382 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:15:36.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.763+0000 7fa7e1600700 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10deb050 0x561a10ded510 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=39f70d1cad206024 server_cookie=1341d217bf895858 in_seq=0 out_seq=0 2024-05-20T20:15:36.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.764+0000 7fa7c3400700 1 -- 172.21.15.28:0/580011870 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561a10d28d90 con 0x561a10deb050 2024-05-20T20:15:36.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.764+0000 7fa7c3400700 10 client.27382 handle_command_reply: tid=0 2024-05-20T20:15:36.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.764+0000 7fa7ee931b80 1 client.27382 shutdown 2024-05-20T20:15:36.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 msgr2=0x561a10dda810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dda810 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fa7d000dbd0 tx=0x7fa7d0004800 comp rx=0 tx=0).stop 2024-05-20T20:15:36.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 shutdown_connections 2024-05-20T20:15:36.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10deb050 0x561a10ded510 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7c3400700 0 client.27382 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:15:36.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10ddad50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10ddd470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/580011870 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dda810 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.771 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.765+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 >> 172.21.15.28:0/580011870 conn(0x561a10cf6be0 msgr2=0x561a10dc2d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:36.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.767+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 shutdown_connections 2024-05-20T20:15:36.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.767+0000 7fa7ee931b80 1 -- 172.21.15.28:0/580011870 wait complete. 2024-05-20T20:15:36.772 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.767+0000 7fa7ee931b80 20 client.27382 trim_cache size 0 max 16384 2024-05-20T20:15:36.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.806+0000 7fa7ee931b80 1 Processor -- start 2024-05-20T20:15:36.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.807+0000 7fa7ee931b80 1 -- start start 2024-05-20T20:15:36.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.807+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10dd1330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.807+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dd9990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.807+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10dc34f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.807+0000 7fa7ee931b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc1b49d0 con 0x561a10dd1950 2024-05-20T20:15:36.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.807+0000 7fa7ee931b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc1b6510 con 0x561a10dd7740 2024-05-20T20:15:36.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.807+0000 7fa7ee931b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc1b50a0 con 0x561a10dd6e20 2024-05-20T20:15:36.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.808+0000 7fa7e0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dd9990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.809+0000 7fa7e0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dd9990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38030/0 (socket says 172.21.15.28:38030) 2024-05-20T20:15:36.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.809+0000 7fa7e1600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10dd1330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.809+0000 7fa7e0c00700 1 -- 172.21.15.28:0/2427230159 learned_addr learned my addr 172.21.15.28:0/2427230159 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:36.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.809+0000 7fa7e2000700 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10dc34f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.809+0000 7fa7e0c00700 1 -- 172.21.15.28:0/2427230159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 msgr2=0x561a10dc34f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.810+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10dc34f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.810+0000 7fa7e0c00700 1 -- 172.21.15.28:0/2427230159 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 msgr2=0x561a10dd1330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.810+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10dd1330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.810+0000 7fa7e0c00700 1 -- 172.21.15.28:0/2427230159 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561a10dedb90 con 0x561a10dd6e20 2024-05-20T20:15:36.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.813+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dd9990 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fa7d00096e0 tx=0x7fa7d001d000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1d6153124ba4daa3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:36.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.813+0000 7fa7ce000700 1 -- 172.21.15.28:0/2427230159 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d0036070 con 0x561a10dd6e20 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.815+0000 7fa7ce000700 1 -- 172.21.15.28:0/2427230159 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa7d40b1c70 con 0x561a10dd6e20 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.816+0000 7fa7ce000700 1 -- 172.21.15.28:0/2427230159 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d0006790 con 0x561a10dd6e20 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.816+0000 7fa7ee931b80 1 -- 172.21.15.28:0/2427230159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 msgr2=0x561a10dd9990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.816+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dd9990 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7fa7d00096e0 tx=0x7fa7d001d000 comp rx=0 tx=0).stop 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.817+0000 7fa7ee931b80 1 -- 172.21.15.28:0/2427230159 shutdown_connections 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.817+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10dd7740 0x561a10dc34f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.817+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10dd1330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.817+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/2427230159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd6e20 0x561a10dd9990 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.817+0000 7fa7ee931b80 1 -- 172.21.15.28:0/2427230159 >> 172.21.15.28:0/2427230159 conn(0x561a10cf6be0 msgr2=0x561a10dd81d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:36.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.822+0000 7fa7ee931b80 1 -- 172.21.15.28:0/2427230159 shutdown_connections 2024-05-20T20:15:36.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.825+0000 7fa7ee931b80 1 -- 172.21.15.28:0/2427230159 wait complete. 2024-05-20T20:15:36.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.825+0000 7fa7ee931b80 1 -- start start 2024-05-20T20:15:36.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.827+0000 7fa7ee931b80 1 Processor -- start 2024-05-20T20:15:36.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.828+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10de36c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.828+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd7740 0x561a10de3c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.828+0000 7fa7ee931b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10de04b0 0x561a10de0930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.828+0000 7fa7e0c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd7740 0x561a10de3c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7e2000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10de04b0 0x561a10de0930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7e2000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10de04b0 0x561a10de0930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:52236/0 (socket says 172.21.15.28:52236) 2024-05-20T20:15:36.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7e2000700 1 -- 172.21.15.28:0/45179159 learned_addr learned my addr 172.21.15.28:0/45179159 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:36.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7e1600700 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10de36c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc1b5e40 con 0x561a10dd1950 2024-05-20T20:15:36.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc1b5770 con 0x561a10de04b0 2024-05-20T20:15:36.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa7dc1b29b0 con 0x561a10dd7740 2024-05-20T20:15:36.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.829+0000 7fa7ee931b80 10 client.0 _ll_register_callbacks cb 0x561a10cc0c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:36.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.830+0000 7fa7e2000700 1 -- 172.21.15.28:0/45179159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd7740 msgr2=0x561a10de3c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.831+0000 7fa7e2000700 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd7740 0x561a10de3c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.831+0000 7fa7e2000700 1 -- 172.21.15.28:0/45179159 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 msgr2=0x561a10de36c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:36.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.831+0000 7fa7e2000700 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10de36c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.831+0000 7fa7e2000700 1 -- 172.21.15.28:0/45179159 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561a10de1030 con 0x561a10de04b0 2024-05-20T20:15:36.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.831+0000 7fa7e0c00700 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd7740 0x561a10de3c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:15:36.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.832+0000 7fa7e2000700 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10de04b0 0x561a10de0930 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7fa7d40a4e90 tx=0x7fa7d40a8ed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=36d22342e0ca35e6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:36.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.832+0000 7fa7c3400700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:36.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.832+0000 7fa7c3400700 1 -- 172.21.15.28:0/45179159 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa7d40a8280 con 0x561a10de04b0 2024-05-20T20:15:36.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.833+0000 7fa7c3400700 1 -- 172.21.15.28:0/45179159 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fa7d40a5400 con 0x561a10de04b0 2024-05-20T20:15:36.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.833+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa7d40b1c70 con 0x561a10de04b0 2024-05-20T20:15:36.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.833+0000 7fa7c3400700 1 -- 172.21.15.28:0/45179159 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa7d40a4780 con 0x561a10de04b0 2024-05-20T20:15:36.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.833+0000 7fa7ee931b80 10 client.27485 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:36.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.833+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561a10e56b70 con 0x561a10de04b0 2024-05-20T20:15:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7c3400700 1 -- 172.21.15.28:0/45179159 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa7d40b1c70 con 0x561a10de04b0 2024-05-20T20:15:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7c3400700 1 -- 172.21.15.28:0/45179159 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa7d40a7ae0 con 0x561a10de04b0 2024-05-20T20:15:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7ee931b80 10 client.27485 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7ee931b80 10 client.27485 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:15:36.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7ee931b80 20 client.27485 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10e56e50 0x561a10e59240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7ee931b80 4 client.27485 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:15:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x561a10d22570 con 0x561a10e56e50 2024-05-20T20:15:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.834+0000 7fa7e1600700 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10e56e50 0x561a10e59240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.836+0000 7fa7c3400700 10 client.27485 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:15:36.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.836+0000 7fa7e1600700 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10e56e50 0x561a10e59240 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f5eb0e0cde268525 server_cookie=978f2657900242cc in_seq=0 out_seq=0 2024-05-20T20:15:36.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.836+0000 7fa7c3400700 1 -- 172.21.15.28:0/45179159 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x561a10d22570 con 0x561a10e56e50 2024-05-20T20:15:36.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.836+0000 7fa7c3400700 10 client.27485 handle_command_reply: tid=0 2024-05-20T20:15:36.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.836+0000 7fa7ee931b80 1 client.27485 shutdown 2024-05-20T20:15:36.838 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10de04b0 msgr2=0x561a10de0930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10de04b0 0x561a10de0930 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7fa7d40a4e90 tx=0x7fa7d40a8ed0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 shutdown_connections 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x561a10e56e50 0x561a10e59240 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7c3400700 0 client.27485 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a10de04b0 0x561a10de0930 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a10dd1950 0x561a10de36c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 --2- 172.21.15.28:0/45179159 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a10dd7740 0x561a10de3c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.839+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 >> 172.21.15.28:0/45179159 conn(0x561a10cf6be0 msgr2=0x561a10dd9990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:36.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.841+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 shutdown_connections 2024-05-20T20:15:36.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.841+0000 7fa7ee931b80 1 -- 172.21.15.28:0/45179159 wait complete. 2024-05-20T20:15:36.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.841+0000 7fa7ee931b80 20 client.27485 trim_cache size 0 max 16384 2024-05-20T20:15:36.843 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:15:36.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.842+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa7c40b2a00 msgr2=0x7fa7c40b4ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.843+0000 7fa7e4000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa7c40b2a00 0x7fa7c40b4ec0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fa7dc1a3e30 tx=0x7fa7d00093d0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.843+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc1a2670 msgr2=0x7fa7dc1a2b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:36.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.843+0000 7fa7e4000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc1a2670 0x7fa7dc1a2b20 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7fa7d400be00 tx=0x7fa7d4009ed0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.844+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 shutdown_connections 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.844+0000 7fa7e4000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa7c40b2a00 0x7fa7c40b4ec0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.844+0000 7fa7e4000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa7dc1a2670 0x7fa7dc1a2b20 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.844+0000 7fa7e4000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa7dc10ae60 0x7fa7dc1a85a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.844+0000 7fa7e4000700 1 --2- 172.21.15.28:0/2533396668 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa7dc116690 0x7fa7dc1a8ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.844+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 >> 172.21.15.28:0/2533396668 conn(0x7fa7dc06e6b0 msgr2=0x7fa7dc0723c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.845+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 shutdown_connections 2024-05-20T20:15:36.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:36.845+0000 7fa7e4000700 1 -- 172.21.15.28:0/2533396668 wait complete. 2024-05-20T20:15:37.525 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:37 smithi028 ceph-mon[31178]: pgmap v310: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 247 MiB/s wr, 1.56k op/s 2024-05-20T20:15:37.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:37 smithi154 ceph-mon[36314]: pgmap v310: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 247 MiB/s wr, 1.56k op/s 2024-05-20T20:15:37.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:37 smithi033 ceph-mon[36179]: pgmap v310: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 247 MiB/s wr, 1.56k op/s 2024-05-20T20:15:39.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.294+0000 7fb3b7c00640 1 -- 172.21.15.28:0/26358954 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0103420 msgr2=0x7fb3b010f930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.294+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/26358954 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0103420 0x7fb3b010f930 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7fb39c009930 tx=0x7fb39c02a480 comp rx=0 tx=0).stop 2024-05-20T20:15:39.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.294+0000 7fb3b7c00640 1 -- 172.21.15.28:0/26358954 shutdown_connections 2024-05-20T20:15:39.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.294+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/26358954 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0103420 0x7fb3b010f930 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.294+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/26358954 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 0x7fb3b0102ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.294+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/26358954 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3b0108a70 0x7fb3b0108e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.294+0000 7fb3b7c00640 1 -- 172.21.15.28:0/26358954 >> 172.21.15.28:0/26358954 conn(0x7fb3b0079ee0 msgr2=0x7fb3b007a2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:39.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.295+0000 7fb3b7c00640 1 -- 172.21.15.28:0/26358954 shutdown_connections 2024-05-20T20:15:39.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.295+0000 7fb3b7c00640 1 -- 172.21.15.28:0/26358954 wait complete. 2024-05-20T20:15:39.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.296+0000 7fb3b7c00640 1 Processor -- start 2024-05-20T20:15:39.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b7c00640 1 -- start start 2024-05-20T20:15:39.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b7c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 0x7fb3b0196f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b7c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3b0103420 0x7fb3b01974c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b7c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0108a70 0x7fb3b019b840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b7c00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb3b0110b40 con 0x7fb3b0108a70 2024-05-20T20:15:39.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b7c00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb3b01109c0 con 0x7fb3b0103420 2024-05-20T20:15:39.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b7c00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb3b0110cc0 con 0x7fb3b0102a60 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.297+0000 7fb3b5200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 0x7fb3b0196f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 0x7fb3b0196f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54482/0 (socket says 172.21.15.28:54482) 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 -- 172.21.15.28:0/4095360889 learned_addr learned my addr 172.21.15.28:0/4095360889 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3afe00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3b0103420 0x7fb3b01974c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0108a70 0x7fb3b019b840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 -- 172.21.15.28:0/4095360889 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3b0103420 msgr2=0x7fb3b01974c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3b0103420 0x7fb3b01974c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 -- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0108a70 msgr2=0x7fb3b019b840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0108a70 0x7fb3b019b840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.298 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 -- 172.21.15.28:0/4095360889 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb3b019bf70 con 0x7fb3b0102a60 2024-05-20T20:15:39.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.298+0000 7fb3b5200640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 0x7fb3b0196f80 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7fb3a800b6d0 tx=0x7fb3a800bb90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:39.300 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.300+0000 7fb3ad600640 1 -- 172.21.15.28:0/4095360889 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3a8023070 con 0x7fb3b0102a60 2024-05-20T20:15:39.301 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.300+0000 7fb3ad600640 1 -- 172.21.15.28:0/4095360889 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3a8008cc0 con 0x7fb3b0102a60 2024-05-20T20:15:39.301 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.300+0000 7fb3ad600640 1 -- 172.21.15.28:0/4095360889 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3a801eb40 con 0x7fb3b0102a60 2024-05-20T20:15:39.301 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.300+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb3b019c260 con 0x7fb3b0102a60 2024-05-20T20:15:39.301 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.300+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb3b01a3af0 con 0x7fb3b0102a60 2024-05-20T20:15:39.305 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.305+0000 7fb3ad600640 1 -- 172.21.15.28:0/4095360889 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb3a8026070 con 0x7fb3b0102a60 2024-05-20T20:15:39.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.309+0000 7fb3ad600640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3940b2ae0 0x7fb3940b4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.312 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.309+0000 7fb3ad600640 1 -- 172.21.15.28:0/4095360889 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fb3a80eb3b0 con 0x7fb3b0102a60 2024-05-20T20:15:39.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.313+0000 7fb3afe00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3940b2ae0 0x7fb3940b4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.321+0000 7fb3afe00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3940b2ae0 0x7fb3940b4fa0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fb3b0102210 tx=0x7fb3a000b390 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:39.321 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.322+0000 7fb3babbd740 1 Processor -- start 2024-05-20T20:15:39.322 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.322+0000 7fb3babbd740 1 -- start start 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f7c90 0x55e4df4f8070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df502930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df505260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3babbd740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e4df468b90 con 0x55e4df4f85b0 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3babbd740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e4df458730 con 0x55e4df4f7c90 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3babbd740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e4df45c770 con 0x55e4df502e70 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3b5200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f7c90 0x55e4df4f8070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3b5200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f7c90 0x55e4df4f8070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37894/0 (socket says 172.21.15.28:37894) 2024-05-20T20:15:39.324 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.323+0000 7fb3b5200640 1 -- 172.21.15.28:0/2961719958 learned_addr learned my addr 172.21.15.28:0/2961719958 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df505260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3afe00640 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df502930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3b5200640 1 -- 172.21.15.28:0/2961719958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 msgr2=0x55e4df505260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3b5200640 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df505260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3b5200640 1 -- 172.21.15.28:0/2961719958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 msgr2=0x55e4df502930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3b5200640 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df502930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3b5200640 1 -- 172.21.15.28:0/2961719958 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e4df3f18b0 con 0x55e4df4f7c90 2024-05-20T20:15:39.325 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3b5200640 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f7c90 0x55e4df4f8070 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fb3a8008600 tx=0x7fb3a80b22a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2fd381140e314023 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.324+0000 7fb3a6000640 1 -- 172.21.15.28:0/2961719958 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3a80bb480 con 0x55e4df4f7c90 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.325+0000 7fb3a6000640 1 -- 172.21.15.28:0/2961719958 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3a80e8d10 con 0x55e4df4f7c90 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.325+0000 7fb3a6000640 1 -- 172.21.15.28:0/2961719958 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3a801e670 con 0x55e4df4f7c90 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.325+0000 7fb3babbd740 1 -- 172.21.15.28:0/2961719958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f7c90 msgr2=0x55e4df4f8070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.325+0000 7fb3babbd740 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f7c90 0x55e4df4f8070 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fb3a8008600 tx=0x7fb3a80b22a0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 -- 172.21.15.28:0/2961719958 shutdown_connections 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df505260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df502930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.327 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 --2- 172.21.15.28:0/2961719958 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f7c90 0x55e4df4f8070 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 -- 172.21.15.28:0/2961719958 >> 172.21.15.28:0/2961719958 conn(0x55e4df4f41a0 msgr2=0x55e4df4f4580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 -- 172.21.15.28:0/2961719958 shutdown_connections 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 -- 172.21.15.28:0/2961719958 wait complete. 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 -- start start 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.326+0000 7fb3babbd740 1 Processor -- start 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df502880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 0x55e4df4ffb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df500180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3babbd740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e4df13e790 con 0x55e4df4f7c90 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3babbd740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e4df400bc0 con 0x55e4df4f85b0 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3babbd740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e4df505aa0 con 0x55e4df502e70 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3babbd740 10 client.0 _ll_register_callbacks cb 0x55e4df3de400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:39.328 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3b5c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df500180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3b5c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df500180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54502/0 (socket says 172.21.15.28:54502) 2024-05-20T20:15:39.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3971200452 learned_addr learned my addr 172.21.15.28:0/3971200452 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:39.329 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.327+0000 7fb3b5200640 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df502880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.328+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3971200452 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 msgr2=0x55e4df4ffb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.328+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 0x55e4df4ffb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.328+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3971200452 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 msgr2=0x55e4df502880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.328+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df502880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.328+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3971200452 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e4df16d210 con 0x55e4df502e70 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.328+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df500180 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7fb39c02aa70 tx=0x7fb39c005270 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7d17c598d2fe31c9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.328+0000 7fb39aa00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:39.330 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.329+0000 7fb39aa00640 1 -- 172.21.15.28:0/3971200452 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb39c035040 con 0x55e4df502e70 2024-05-20T20:15:39.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.329+0000 7fb39aa00640 1 -- 172.21.15.28:0/3971200452 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fb39c0048c0 con 0x55e4df502e70 2024-05-20T20:15:39.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.329+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3a80e8d10 con 0x55e4df502e70 2024-05-20T20:15:39.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.329+0000 7fb39aa00640 1 -- 172.21.15.28:0/3971200452 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb39c004ac0 con 0x55e4df502e70 2024-05-20T20:15:39.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.329+0000 7fb3babbd740 10 client.27400 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:39.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.329+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e4df239620 con 0x55e4df502e70 2024-05-20T20:15:39.331 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.329+0000 7fb39aa00640 1 -- 172.21.15.28:0/3971200452 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3a80e8d10 con 0x55e4df502e70 2024-05-20T20:15:39.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.333+0000 7fb39aa00640 1 -- 172.21.15.28:0/3971200452 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb39c045470 con 0x55e4df502e70 2024-05-20T20:15:39.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.333+0000 7fb3babbd740 10 client.27400 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:39.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.333+0000 7fb3babbd740 10 client.27400 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:39.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.333+0000 7fb3babbd740 20 client.27400 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.333+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df512940 0x55e4df514dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.333+0000 7fb3babbd740 4 client.27400 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.333+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e4df46aae0 con 0x55e4df512940 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.335+0000 7fb3b5200640 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df512940 0x55e4df514dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.335+0000 7fb39aa00640 10 client.27400 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.335+0000 7fb3b5200640 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df512940 0x55e4df514dd0 crc :-1 s=READY pgs=213 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=961f8e490af0e1a2 server_cookie=4b7055744a346382 in_seq=0 out_seq=0 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.336+0000 7fb39aa00640 1 -- 172.21.15.28:0/3971200452 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55e4df46aae0 con 0x55e4df512940 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.336+0000 7fb39aa00640 10 client.27400 handle_command_reply: tid=0 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.336+0000 7fb3babbd740 1 client.27400 shutdown 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 msgr2=0x55e4df500180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df500180 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7fb39c02aa70 tx=0x7fb39c005270 comp rx=0 tx=0).stop 2024-05-20T20:15:39.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 shutdown_connections 2024-05-20T20:15:39.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df512940 0x55e4df514dd0 crc :-1 s=READY pgs=213 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb39aa00640 0 client.27400 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:39.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 0x55e4df4ffb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df500180 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3971200452 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df502880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.337+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 >> 172.21.15.28:0/3971200452 conn(0x55e4df4f41a0 msgr2=0x55e4df4f9f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:39.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.338+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 shutdown_connections 2024-05-20T20:15:39.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.338+0000 7fb3babbd740 1 -- 172.21.15.28:0/3971200452 wait complete. 2024-05-20T20:15:39.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.338+0000 7fb3babbd740 20 client.27400 trim_cache size 0 max 16384 2024-05-20T20:15:39.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.380+0000 7fb3babbd740 1 Processor -- start 2024-05-20T20:15:39.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.380+0000 7fb3babbd740 1 -- start start 2024-05-20T20:15:39.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.381+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df4f7c90 0x55e4df50eb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.381+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df5023b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.381+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df502e70 0x55e4df51a6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.381+0000 7fb3babbd740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb39000e7e0 con 0x55e4df4f85b0 2024-05-20T20:15:39.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.381+0000 7fb3babbd740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb390010320 con 0x55e4df502e70 2024-05-20T20:15:39.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.381+0000 7fb3babbd740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb39000eeb0 con 0x55e4df4f7c90 2024-05-20T20:15:39.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.383+0000 7fb3afe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df5023b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.383+0000 7fb3b5200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df4f7c90 0x55e4df50eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.383+0000 7fb3afe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df5023b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:43934/0 (socket says 172.21.15.28:43934) 2024-05-20T20:15:39.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.383+0000 7fb3afe00640 1 -- 172.21.15.28:0/3638535985 learned_addr learned my addr 172.21.15.28:0/3638535985 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:39.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.384+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df502e70 0x55e4df51a6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.385+0000 7fb3afe00640 1 -- 172.21.15.28:0/3638535985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df4f7c90 msgr2=0x55e4df50eb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.385+0000 7fb3afe00640 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df4f7c90 0x55e4df50eb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.385+0000 7fb3afe00640 1 -- 172.21.15.28:0/3638535985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df502e70 msgr2=0x55e4df51a6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.385+0000 7fb3afe00640 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df502e70 0x55e4df51a6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.385+0000 7fb3afe00640 1 -- 172.21.15.28:0/3638535985 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e4df500760 con 0x55e4df4f85b0 2024-05-20T20:15:39.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.385+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df502e70 0x55e4df51a6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:39.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.386+0000 7fb3b5200640 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df4f7c90 0x55e4df50eb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:39.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.386+0000 7fb3afe00640 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df5023b0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fb3a0010fb0 tx=0x7fb3a0013070 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2e7d45600ab6e0bc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:39.387 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.387+0000 7fb3a5600640 1 -- 172.21.15.28:0/3638535985 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3a0005220 con 0x55e4df4f85b0 2024-05-20T20:15:39.387 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.388+0000 7fb3a5600640 1 -- 172.21.15.28:0/3638535985 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3a0023070 con 0x55e4df4f85b0 2024-05-20T20:15:39.387 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.388+0000 7fb3a5600640 1 -- 172.21.15.28:0/3638535985 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb3a0005220 con 0x55e4df4f85b0 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 -- 172.21.15.28:0/3638535985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 msgr2=0x55e4df5023b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df5023b0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7fb3a0010fb0 tx=0x7fb3a0013070 comp rx=0 tx=0).stop 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 -- 172.21.15.28:0/3638535985 shutdown_connections 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df502e70 0x55e4df51a6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f85b0 0x55e4df5023b0 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3638535985 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df4f7c90 0x55e4df50eb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 -- 172.21.15.28:0/3638535985 >> 172.21.15.28:0/3638535985 conn(0x55e4df4f41a0 msgr2=0x55e4df4fb160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:39.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 -- 172.21.15.28:0/3638535985 shutdown_connections 2024-05-20T20:15:39.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.389+0000 7fb3babbd740 1 -- 172.21.15.28:0/3638535985 wait complete. 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.390+0000 7fb3babbd740 1 -- start start 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.390+0000 7fb3babbd740 1 Processor -- start 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.390+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df50b430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 0x55e4df5068a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3babbd740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df506de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3babbd740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb39000fc50 con 0x55e4df4f7c90 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3babbd740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb39000f580 con 0x55e4df4f85b0 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3babbd740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb39000c7e0 con 0x55e4df502e70 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3babbd740 10 client.0 _ll_register_callbacks cb 0x7fb3a800d7b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:39.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3afe00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 0x55e4df5068a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3b5200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df50b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3b5c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df506de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3b5c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df506de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:54528/0 (socket says 172.21.15.28:54528) 2024-05-20T20:15:39.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.391+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3619955369 learned_addr learned my addr 172.21.15.28:0/3619955369 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.392+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3619955369 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 msgr2=0x55e4df5068a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.392+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 0x55e4df5068a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.392+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3619955369 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 msgr2=0x55e4df50b430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.392+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df50b430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.392+0000 7fb3b5c00640 1 -- 172.21.15.28:0/3619955369 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e4df5075e0 con 0x55e4df502e70 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.392+0000 7fb3b5c00640 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df506de0 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7fb39c009800 tx=0x7fb39c005970 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f7e1b29ba5a7457f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.392+0000 7fb39aa00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.393+0000 7fb39aa00640 1 -- 172.21.15.28:0/3619955369 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb39c051070 con 0x55e4df502e70 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.393+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb3a0023070 con 0x55e4df502e70 2024-05-20T20:15:39.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.393+0000 7fb39aa00640 1 -- 172.21.15.28:0/3619955369 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fb39c0048c0 con 0x55e4df502e70 2024-05-20T20:15:39.394 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.393+0000 7fb39aa00640 1 -- 172.21.15.28:0/3619955369 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb39c05d050 con 0x55e4df502e70 2024-05-20T20:15:39.394 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.393+0000 7fb3babbd740 10 client.27406 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.393+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e4df57d650 con 0x55e4df502e70 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.394+0000 7fb39aa00640 1 -- 172.21.15.28:0/3619955369 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb3a0023070 con 0x55e4df502e70 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.394+0000 7fb39aa00640 1 -- 172.21.15.28:0/3619955369 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb39c003aa0 con 0x55e4df502e70 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.394+0000 7fb3babbd740 10 client.27406 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.394+0000 7fb3babbd740 10 client.27406 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.394+0000 7fb3babbd740 20 client.27406 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.394+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df57de50 0x55e4df580270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.395+0000 7fb3babbd740 4 client.27406 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.395+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55e4df4642c0 con 0x55e4df57de50 2024-05-20T20:15:39.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.395+0000 7fb3b5200640 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df57de50 0x55e4df580270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:39.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.396+0000 7fb39aa00640 10 client.27406 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:39.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.396+0000 7fb3b5200640 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df57de50 0x55e4df580270 crc :-1 s=READY pgs=214 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fc7e41f01840f627 server_cookie=7bfe39946ef569db in_seq=0 out_seq=0 2024-05-20T20:15:39.414 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb39aa00640 1 -- 172.21.15.28:0/3619955369 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55e4df4642c0 con 0x55e4df57de50 2024-05-20T20:15:39.414 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb39aa00640 10 client.27406 handle_command_reply: tid=0 2024-05-20T20:15:39.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 client.27406 shutdown 2024-05-20T20:15:39.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 msgr2=0x55e4df506de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df506de0 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7fb39c009800 tx=0x7fb39c005970 comp rx=0 tx=0).stop 2024-05-20T20:15:39.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 shutdown_connections 2024-05-20T20:15:39.415 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e4df57de50 0x55e4df580270 crc :-1 s=READY pgs=214 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e4df4f85b0 0x55e4df5068a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e4df502e70 0x55e4df506de0 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 --2- 172.21.15.28:0/3619955369 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e4df4f7c90 0x55e4df50b430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 >> 172.21.15.28:0/3619955369 conn(0x55e4df4f41a0 msgr2=0x55e4df50daf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb39aa00640 0 client.27406 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.415+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 shutdown_connections 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.416+0000 7fb3babbd740 1 -- 172.21.15.28:0/3619955369 wait complete. 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:15:39.416 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:15:39.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:15:39.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.416+0000 7fb3babbd740 20 client.27406 trim_cache size 0 max 16384 2024-05-20T20:15:39.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.417+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3940b2ae0 msgr2=0x7fb3940b4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.417+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3940b2ae0 0x7fb3940b4fa0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fb3b0102210 tx=0x7fb3a000b390 comp rx=0 tx=0).stop 2024-05-20T20:15:39.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.417+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 msgr2=0x7fb3b0196f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:39.417 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.417+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 0x7fb3b0196f80 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7fb3a800b6d0 tx=0x7fb3a800bb90 comp rx=0 tx=0).stop 2024-05-20T20:15:39.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.420+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 shutdown_connections 2024-05-20T20:15:39.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.420+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb3b0103420 0x7fb3b01974c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.419 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.420+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb3940b2ae0 0x7fb3940b4fa0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.420+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb3b0108a70 0x7fb3b019b840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.420+0000 7fb3b7c00640 1 --2- 172.21.15.28:0/4095360889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb3b0102a60 0x7fb3b0196f80 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:39.420 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.420+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 >> 172.21.15.28:0/4095360889 conn(0x7fb3b0079ee0 msgr2=0x7fb3b0100260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:39.421 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.422+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 shutdown_connections 2024-05-20T20:15:39.422 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:39.422+0000 7fb3b7c00640 1 -- 172.21.15.28:0/4095360889 wait complete. 2024-05-20T20:15:39.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:39 smithi028 ceph-mon[31178]: pgmap v311: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 258 MiB/s wr, 1.55k op/s 2024-05-20T20:15:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:39 smithi154 ceph-mon[36314]: pgmap v311: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 258 MiB/s wr, 1.55k op/s 2024-05-20T20:15:39.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:39 smithi033 ceph-mon[36179]: pgmap v311: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 258 MiB/s wr, 1.55k op/s 2024-05-20T20:15:40.965 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:15:41.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:41 smithi028 ceph-mon[31178]: pgmap v312: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 312 MiB/s wr, 1.73k op/s 2024-05-20T20:15:41.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:41 smithi154 ceph-mon[36314]: pgmap v312: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 312 MiB/s wr, 1.73k op/s 2024-05-20T20:15:41.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:41 smithi033 ceph-mon[36179]: pgmap v312: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 312 MiB/s wr, 1.73k op/s 2024-05-20T20:15:43.507 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:43 smithi028 ceph-mon[31178]: pgmap v313: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 252 MiB/s wr, 1.26k op/s 2024-05-20T20:15:43.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:43 smithi154 ceph-mon[36314]: pgmap v313: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 252 MiB/s wr, 1.26k op/s 2024-05-20T20:15:43.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:43 smithi033 ceph-mon[36179]: pgmap v313: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 252 MiB/s wr, 1.26k op/s 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.505+0000 7f1334000700 1 -- 172.21.15.28:0/2589453089 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 msgr2=0x7f132c1028c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.505+0000 7f1334000700 1 --2- 172.21.15.28:0/2589453089 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 0x7f132c1028c0 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f1318009990 tx=0x7f1318009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.506+0000 7f1334000700 1 -- 172.21.15.28:0/2589453089 shutdown_connections 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.506+0000 7f1334000700 1 --2- 172.21.15.28:0/2589453089 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f132c102e00 0x7f132c10f2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.506+0000 7f1334000700 1 --2- 172.21.15.28:0/2589453089 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c108520 0x7f132c108900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.506+0000 7f1334000700 1 --2- 172.21.15.28:0/2589453089 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 0x7f132c1028c0 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.506+0000 7f1334000700 1 -- 172.21.15.28:0/2589453089 >> 172.21.15.28:0/2589453089 conn(0x7f132c0793f0 msgr2=0x7f132c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:43.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.507+0000 7f1334000700 1 -- 172.21.15.28:0/2589453089 shutdown_connections 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.507+0000 7f1334000700 1 -- 172.21.15.28:0/2589453089 wait complete. 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.509+0000 7f1334000700 1 Processor -- start 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.509+0000 7f1334000700 1 -- start start 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1334000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 0x7f132c197930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1334000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c102e00 0x7f132c197e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1334000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f132c108520 0x7f132c1983b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1334000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f132c1115f0 con 0x7f132c108520 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1334000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f132c111470 con 0x7f132c102e00 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1334000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f132c111770 con 0x7f132c102440 2024-05-20T20:15:43.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1330c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c102e00 0x7f132c197e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1330c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c102e00 0x7f132c197e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:37952/0 (socket says 172.21.15.28:37952) 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.510+0000 7f1330c00700 1 -- 172.21.15.28:0/78675741 learned_addr learned my addr 172.21.15.28:0/78675741 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.511+0000 7f1330c00700 1 -- 172.21.15.28:0/78675741 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 msgr2=0x7f132c197930 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.511+0000 7f1331600700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 0x7f132c197930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.511+0000 7f1330c00700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 0x7f132c197930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.511+0000 7f1330c00700 1 -- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f132c108520 msgr2=0x7f132c1983b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.511+0000 7f1330c00700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f132c108520 0x7f132c1983b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.511+0000 7f1330c00700 1 -- 172.21.15.28:0/78675741 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f132c19bd70 con 0x7f132c102e00 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.511+0000 7f1331600700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 0x7f132c197930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.512+0000 7f1330c00700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c102e00 0x7f132c197e70 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f13180098e0 tx=0x7f131800ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.512+0000 7f132a000700 1 -- 172.21.15.28:0/78675741 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1318023070 con 0x7f132c102e00 2024-05-20T20:15:43.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.513+0000 7f132a000700 1 -- 172.21.15.28:0/78675741 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1318015450 con 0x7f132c102e00 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.513+0000 7f132a000700 1 -- 172.21.15.28:0/78675741 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f131801a030 con 0x7f132c102e00 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.513+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f132c19c000 con 0x7f132c102e00 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.513+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f132c19c490 con 0x7f132c102e00 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.516+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f132c103b60 con 0x7f132c102e00 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.520+0000 7f132a000700 1 -- 172.21.15.28:0/78675741 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1318026070 con 0x7f132c102e00 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.523+0000 7f132a000700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13140b2b60 0x7f13140b5020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.523+0000 7f132a000700 1 -- 172.21.15.28:0/78675741 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f13180e26a0 con 0x7f132c102e00 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.524+0000 7f1331600700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13140b2b60 0x7f13140b5020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.526+0000 7f1331600700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13140b2b60 0x7f13140b5020 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1320006fd0 tx=0x7f1320008040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:43.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:43.526+0000 7f132a000700 1 -- 172.21.15.28:0/78675741 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f13180aa0a0 con 0x7f132c102e00 2024-05-20T20:15:43.782 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:44.029 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:15:44.029 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:15:44.029 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.021+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f132c198e10 con 0x7f132c102e00 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.022+0000 7f132a000700 1 -- 172.21.15.28:0/78675741 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f13180a99a0 con 0x7f132c102e00 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13140b2b60 msgr2=0x7f13140b5020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13140b2b60 0x7f13140b5020 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1320006fd0 tx=0x7f1320008040 comp rx=0 tx=0).stop 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c102e00 msgr2=0x7f132c197e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c102e00 0x7f132c197e70 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7f13180098e0 tx=0x7f131800ecb0 comp rx=0 tx=0).stop 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 shutdown_connections 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f132c108520 0x7f132c1983b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f132c102440 0x7f132c197930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f13140b2b60 0x7f13140b5020 secure :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1320006fd0 tx=0x7f1320008040 comp rx=0 tx=0).stop 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 --2- 172.21.15.28:0/78675741 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f132c102e00 0x7f132c197e70 unknown :-1 s=CLOSED pgs=527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:44.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.028+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 >> 172.21.15.28:0/78675741 conn(0x7f132c0793f0 msgr2=0x7f132c100530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:44.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.030+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 shutdown_connections 2024-05-20T20:15:44.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:44.030+0000 7f1334000700 1 -- 172.21.15.28:0/78675741 wait complete. 2024-05-20T20:15:44.043 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:15:44.278 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:44 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/78675741' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:15:44.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:44 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/78675741' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:15:44.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:44 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/78675741' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:15:45.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:45 smithi028 ceph-mon[31178]: pgmap v314: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 347 MiB/s wr, 1.60k op/s 2024-05-20T20:15:45.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:45 smithi154 ceph-mon[36314]: pgmap v314: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 347 MiB/s wr, 1.60k op/s 2024-05-20T20:15:45.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:45 smithi033 ceph-mon[36179]: pgmap v314: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 347 MiB/s wr, 1.60k op/s 2024-05-20T20:15:46.065 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 152.6 --await-for 45.0' 2024-05-20T20:15:46.065 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 152.6 --await-for 45.0 2024-05-20T20:15:46.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:46 smithi028 ceph-mon[31178]: pgmap v315: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 268 MiB/s wr, 1.13k op/s 2024-05-20T20:15:46.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:46 smithi154 ceph-mon[36314]: pgmap v315: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 268 MiB/s wr, 1.13k op/s 2024-05-20T20:15:46.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:46 smithi033 ceph-mon[36179]: pgmap v315: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 268 MiB/s wr, 1.13k op/s 2024-05-20T20:15:48.158 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:49.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:49 smithi154 ceph-mon[36314]: pgmap v316: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 271 MiB/s wr, 1.13k op/s 2024-05-20T20:15:49.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:49 smithi033 ceph-mon[36179]: pgmap v316: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 271 MiB/s wr, 1.13k op/s 2024-05-20T20:15:49.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:49 smithi028 ceph-mon[31178]: pgmap v316: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 271 MiB/s wr, 1.13k op/s 2024-05-20T20:15:51.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:51 smithi028 ceph-mon[31178]: pgmap v317: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 326 MiB/s wr, 1.39k op/s 2024-05-20T20:15:51.606 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.684+0000 7fdd13600700 1 -- 172.21.15.28:0/2852723033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c075770 msgr2=0x7fdd0c075b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.684+0000 7fdd13600700 1 --2- 172.21.15.28:0/2852723033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c075770 0x7fdd0c075b50 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7fdd00009990 tx=0x7fdd00009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.685+0000 7fdd13600700 1 -- 172.21.15.28:0/2852723033 shutdown_connections 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.685+0000 7fdd13600700 1 --2- 172.21.15.28:0/2852723033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd0c10ff40 0x7fdd0c112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.685+0000 7fdd13600700 1 --2- 172.21.15.28:0/2852723033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c075770 0x7fdd0c075b50 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.685+0000 7fdd13600700 1 --2- 172.21.15.28:0/2852723033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 0x7fdd0c10fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.685+0000 7fdd13600700 1 -- 172.21.15.28:0/2852723033 >> 172.21.15.28:0/2852723033 conn(0x7fdd0c07a6a0 msgr2=0x7fdd0c07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.685+0000 7fdd13600700 1 -- 172.21.15.28:0/2852723033 shutdown_connections 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.685+0000 7fdd13600700 1 -- 172.21.15.28:0/2852723033 wait complete. 2024-05-20T20:15:51.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.686+0000 7fdd13600700 1 Processor -- start 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd13600700 1 -- start start 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd13600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd0c075770 0x7fdd0c19be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd13600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 0x7fdd0c19c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd13600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c10ff40 0x7fdd0c1a06e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd13600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c115940 con 0x7fdd0c075770 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd13600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c1157c0 con 0x7fdd0c10ff40 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd13600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c115ac0 con 0x7fdd0c076090 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd0be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 0x7fdd0c19c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd0be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 0x7fdd0c19c350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55072/0 (socket says 172.21.15.28:55072) 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd0be00700 1 -- 172.21.15.28:0/863215178 learned_addr learned my addr 172.21.15.28:0/863215178 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd10c00700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd0c075770 0x7fdd0c19be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.687+0000 7fdd11600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c10ff40 0x7fdd0c1a06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.688+0000 7fdd0be00700 1 -- 172.21.15.28:0/863215178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c10ff40 msgr2=0x7fdd0c1a06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.688+0000 7fdd0be00700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c10ff40 0x7fdd0c1a06e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.688+0000 7fdd0be00700 1 -- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd0c075770 msgr2=0x7fdd0c19be10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.688+0000 7fdd0be00700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd0c075770 0x7fdd0c19be10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.688+0000 7fdd0be00700 1 -- 172.21.15.28:0/863215178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdd0c1a0e20 con 0x7fdd0c076090 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.688+0000 7fdd0be00700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 0x7fdd0c19c350 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7fdcf800ea50 tx=0x7fdcf800ed60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.689+0000 7fdd09600700 1 -- 172.21.15.28:0/863215178 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdcf800cbd0 con 0x7fdd0c076090 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.689+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdd0c1a1110 con 0x7fdd0c076090 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.689+0000 7fdd09600700 1 -- 172.21.15.28:0/863215178 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdcf800cd70 con 0x7fdd0c076090 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.689+0000 7fdd09600700 1 -- 172.21.15.28:0/863215178 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdcf8018680 con 0x7fdd0c076090 2024-05-20T20:15:51.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.689+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdd0c1a8990 con 0x7fdd0c076090 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.693+0000 7fdd09600700 1 -- 172.21.15.28:0/863215178 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fdcf801f020 con 0x7fdd0c076090 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.694+0000 7fdd09600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdcf40b29f0 0x7fdcf40b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.694+0000 7fdd09600700 1 -- 172.21.15.28:0/863215178 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fdcf80e2620 con 0x7fdd0c076090 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.695+0000 7fdd10c00700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdcf40b29f0 0x7fdcf40b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.695+0000 7fdd10c00700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdcf40b29f0 0x7fdcf40b4eb0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fdd00006010 tx=0x7fdd0000b260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.698+0000 7fdd1dfceb80 1 Processor -- start 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd1dfceb80 1 -- start start 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358139f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 0x55a53580db20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a53580e5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a53576be40 con 0x55a53580e140 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a53575b9e0 con 0x55a535813610 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a53575fa20 con 0x55a535813f30 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.699+0000 7fdd10c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358139f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd10c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358139f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60576/0 (socket says 172.21.15.28:60576) 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd10c00700 1 -- 172.21.15.28:0/2187920056 learned_addr learned my addr 172.21.15.28:0/2187920056 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd0be00700 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 0x55a53580db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd10c00700 1 -- 172.21.15.28:0/2187920056 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 msgr2=0x55a53580db20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd10c00700 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 0x55a53580db20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd10c00700 1 -- 172.21.15.28:0/2187920056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 msgr2=0x55a53580e5c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd11600700 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a53580e5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd10c00700 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a53580e5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.700+0000 7fdd10c00700 1 -- 172.21.15.28:0/2187920056 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a535724420 con 0x55a535813610 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.701+0000 7fdd11600700 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a53580e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.701+0000 7fdd10c00700 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358139f0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7fdd000056e0 tx=0x7fdd0001f000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5bb25a8245e055e1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.701+0000 7fdcfe000700 1 -- 172.21.15.28:0/2187920056 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd0000bdb0 con 0x55a535813610 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.701+0000 7fdcfe000700 1 -- 172.21.15.28:0/2187920056 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd000095c0 con 0x55a535813610 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.701+0000 7fdcfe000700 1 -- 172.21.15.28:0/2187920056 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd00030970 con 0x55a535813610 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.702+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/2187920056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 msgr2=0x55a5358139f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.702+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358139f0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7fdd000056e0 tx=0x7fdd0001f000 comp rx=0 tx=0).stop 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.705+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/2187920056 shutdown_connections 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.705+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a53580e5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.705+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358139f0 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.705+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/2187920056 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 0x55a53580db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.705+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/2187920056 >> 172.21.15.28:0/2187920056 conn(0x55a53573bbe0 msgr2=0x55a535808690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.705+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/2187920056 shutdown_connections 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.705+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/2187920056 wait complete. 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.706+0000 7fdd1dfceb80 1 -- start start 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.706+0000 7fdd1dfceb80 1 Processor -- start 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a535819d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358170c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813f30 0x55a535817600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a535661fa0 con 0x55a535813f30 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a5356fc760 con 0x55a535813610 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a53581a980 con 0x55a53580e140 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd1dfceb80 10 client.0 _ll_register_callbacks cb 0x55a535705c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:51.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd10c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a535819d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd10c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a535819d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55078/0 (socket says 172.21.15.28:55078) 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.707+0000 7fdd10c00700 1 -- 172.21.15.28:0/3305399533 learned_addr learned my addr 172.21.15.28:0/3305399533 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdd10c00700 1 -- 172.21.15.28:0/3305399533 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 msgr2=0x55a5358170c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdd10c00700 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358170c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdd10c00700 1 -- 172.21.15.28:0/3305399533 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813f30 msgr2=0x55a535817600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdd10c00700 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813f30 0x55a535817600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdd10c00700 1 -- 172.21.15.28:0/3305399533 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a5356ab8d0 con 0x55a53580e140 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdd10c00700 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a535819d20 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fdd00000c00 tx=0x7fdd0000be90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f9eae0ff6f0b9ad8 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdcf2a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:51.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdcf2a00700 1 -- 172.21.15.28:0/3305399533 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd000309e0 con 0x55a53580e140 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdcf2a00700 1 -- 172.21.15.28:0/3305399533 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fdd00003680 con 0x55a53580e140 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.708+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55a53576dd90 con 0x55a53580e140 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.709+0000 7fdcf2a00700 1 -- 172.21.15.28:0/3305399533 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fdd00003880 con 0x55a53580e140 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.709+0000 7fdd1dfceb80 10 client.27421 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.709+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a53572c060 con 0x55a53580e140 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.709+0000 7fdcf2a00700 1 -- 172.21.15.28:0/3305399533 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55a53576dd90 con 0x55a53580e140 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdcf2a00700 1 -- 172.21.15.28:0/3305399533 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fdd00026430 con 0x55a53580e140 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdd1dfceb80 10 client.27421 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdd1dfceb80 10 client.27421 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdd1dfceb80 20 client.27421 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53582fb80 0x55a535831f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdd1dfceb80 4 client.27421 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a535767570 con 0x55a53582fb80 2024-05-20T20:15:51.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.710+0000 7fdd0be00700 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53582fb80 0x55a535831f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.711+0000 7fdcf2a00700 10 client.27421 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.711+0000 7fdd0be00700 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53582fb80 0x55a535831f70 crc :-1 s=READY pgs=215 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d24a23edb9d494fe server_cookie=d63a67554fc60b73 in_seq=0 out_seq=0 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdcf2a00700 1 -- 172.21.15.28:0/3305399533 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a535767570 con 0x55a53582fb80 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdcf2a00700 10 client.27421 handle_command_reply: tid=0 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 client.27421 shutdown 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 msgr2=0x55a535819d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a535819d20 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7fdd00000c00 tx=0x7fdd0000be90 comp rx=0 tx=0).stop 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 shutdown_connections 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53582fb80 0x55a535831f70 crc :-1 s=READY pgs=215 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813f30 0x55a535817600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a535819d20 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdcf2a00700 0 client.27421 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3305399533 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a5358170c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 >> 172.21.15.28:0/3305399533 conn(0x55a53573bbe0 msgr2=0x55a535807d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 shutdown_connections 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.712+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3305399533 wait complete. 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.713+0000 7fdd1dfceb80 20 client.27421 trim_cache size 0 max 16384 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.731+0000 7fdd1dfceb80 1 Processor -- start 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.731+0000 7fdd1dfceb80 1 -- start start 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a53580db20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813610 0x55a535811790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813f30 0x55a535814ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c1abcb0 con 0x55a535813610 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c1ad7f0 con 0x55a535813f30 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c1ac380 con 0x55a53580e140 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd10c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a53580db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd10c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a53580db20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55086/0 (socket says 172.21.15.28:55086) 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.732+0000 7fdd10c00700 1 -- 172.21.15.28:0/3874315675 learned_addr learned my addr 172.21.15.28:0/3874315675 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.733+0000 7fdd10c00700 1 -- 172.21.15.28:0/3874315675 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813f30 msgr2=0x55a535814ba0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.733+0000 7fdd10c00700 1 --2- 172.21.15.28:0/3874315675 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813f30 0x55a535814ba0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.733+0000 7fdd10c00700 1 -- 172.21.15.28:0/3874315675 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813610 msgr2=0x55a535811790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.733+0000 7fdd10c00700 1 --2- 172.21.15.28:0/3874315675 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813610 0x55a535811790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.733+0000 7fdd10c00700 1 -- 172.21.15.28:0/3874315675 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a535817a30 con 0x55a53580e140 2024-05-20T20:15:51.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.733+0000 7fdd10c00700 1 --2- 172.21.15.28:0/3874315675 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a53580db20 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7fdd00009990 tx=0x7fdd000094b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=11f59005d9af07cf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.736+0000 7fdcfd600700 1 -- 172.21.15.28:0/3874315675 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd00003680 con 0x55a53580e140 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.736+0000 7fdcfd600700 1 -- 172.21.15.28:0/3874315675 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd000309e0 con 0x55a53580e140 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.736+0000 7fdcfd600700 1 -- 172.21.15.28:0/3874315675 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd00041920 con 0x55a53580e140 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.737+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3874315675 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 msgr2=0x55a53580db20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.737+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3874315675 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a53580db20 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7fdd00009990 tx=0x7fdd000094b0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.738+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3874315675 shutdown_connections 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.738+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3874315675 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813f30 0x55a535814ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.738+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3874315675 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a53580e140 0x55a53580db20 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.738+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/3874315675 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a535813610 0x55a535811790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.738+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3874315675 >> 172.21.15.28:0/3874315675 conn(0x55a53573bbe0 msgr2=0x55a535812200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.738+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3874315675 shutdown_connections 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.738+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/3874315675 wait complete. 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.739+0000 7fdd1dfceb80 1 -- start start 2024-05-20T20:15:51.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.739+0000 7fdd1dfceb80 1 Processor -- start 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a535891710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a535891c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd1dfceb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 0x55a535832550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c1ad120 con 0x55a53580e140 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c1aca50 con 0x55a535813610 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd1dfceb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdd0c1a9d70 con 0x55a535813f30 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd1dfceb80 10 client.0 _ll_register_callbacks cb 0x55a535705c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd0be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a535891c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd0be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a535891c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60602/0 (socket says 172.21.15.28:60602) 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.740+0000 7fdd0be00700 1 -- 172.21.15.28:0/1774634907 learned_addr learned my addr 172.21.15.28:0/1774634907 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.741+0000 7fdd10c00700 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a535891710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.741+0000 7fdd0be00700 1 -- 172.21.15.28:0/1774634907 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 msgr2=0x55a535832550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:51.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.741+0000 7fdd0be00700 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 0x55a535832550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.741+0000 7fdd0be00700 1 -- 172.21.15.28:0/1774634907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 msgr2=0x55a535891710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.741+0000 7fdd0be00700 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a535891710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.741+0000 7fdd0be00700 1 -- 172.21.15.28:0/1774634907 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a535832d80 con 0x55a535813610 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.741+0000 7fdd10c00700 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a535891710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.742+0000 7fdd0be00700 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a535891c50 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7fdcf80a97d0 tx=0x7fdcf80a9660 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c6467e97e32b7a29 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.742+0000 7fdcf2a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.743+0000 7fdcf2a00700 1 -- 172.21.15.28:0/1774634907 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdcf80a93a0 con 0x55a535813610 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.743+0000 7fdcf2a00700 1 -- 172.21.15.28:0/1774634907 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fdcf80a9d00 con 0x55a535813610 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.743+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdd000309e0 con 0x55a535813610 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.743+0000 7fdcf2a00700 1 -- 172.21.15.28:0/1774634907 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fdcf80a9f90 con 0x55a535813610 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.743+0000 7fdd1dfceb80 10 client.27515 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.743+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a535899ed0 con 0x55a535813610 2024-05-20T20:15:51.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.744+0000 7fdcf2a00700 1 -- 172.21.15.28:0/1774634907 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdcf80c5070 con 0x55a535813610 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.744+0000 7fdcf2a00700 1 -- 172.21.15.28:0/1774634907 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fdcf80c7070 con 0x55a535813610 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.744+0000 7fdd1dfceb80 10 client.27515 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.744+0000 7fdd1dfceb80 10 client.27515 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.744+0000 7fdd1dfceb80 20 client.27515 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.744+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53589a5c0 0x55a53589c9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.745+0000 7fdd1dfceb80 4 client.27515 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.745+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55a5355de250 con 0x55a53589a5c0 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.745+0000 7fdd10c00700 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53589a5c0 0x55a53589c9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:51.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.745+0000 7fdd10c00700 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53589a5c0 0x55a53589c9e0 crc :-1 s=READY pgs=216 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a822e4d67532a329 server_cookie=488320028f49d933 in_seq=0 out_seq=0 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.745+0000 7fdcf2a00700 10 client.27515 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.745+0000 7fdcf2a00700 1 -- 172.21.15.28:0/1774634907 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55a5355de250 con 0x55a53589a5c0 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.745+0000 7fdcf2a00700 10 client.27515 handle_command_reply: tid=0 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 client.27515 shutdown 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 msgr2=0x55a535891c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a535891c50 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7fdcf80a97d0 tx=0x7fdcf80a9660 comp rx=0 tx=0).stop 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 shutdown_connections 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a53589a5c0 0x55a53589c9e0 crc :-1 s=READY pgs=216 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a535813f30 0x55a535832550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a53580e140 0x55a535891710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 --2- 172.21.15.28:0/1774634907 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a535813610 0x55a535891c50 unknown :-1 s=CLOSED pgs=530 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 >> 172.21.15.28:0/1774634907 conn(0x55a53573bbe0 msgr2=0x55a535814310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.746+0000 7fdcf2a00700 0 client.27515 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.747+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 shutdown_connections 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.747+0000 7fdd1dfceb80 1 -- 172.21.15.28:0/1774634907 wait complete. 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.747+0000 7fdd1dfceb80 20 client.27515 trim_cache size 0 max 16384 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdcf40b29f0 msgr2=0x7fdcf40b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdcf40b29f0 0x7fdcf40b4eb0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fdd00006010 tx=0x7fdd0000b260 comp rx=0 tx=0).stop 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 msgr2=0x7fdd0c19c350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 0x7fdd0c19c350 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7fdcf800ea50 tx=0x7fdcf800ed60 comp rx=0 tx=0).stop 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 shutdown_connections 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdcf40b29f0 0x7fdcf40b4eb0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd0c10ff40 0x7fdd0c1a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd0c075770 0x7fdd0c19be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:51.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 --2- 172.21.15.28:0/863215178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd0c076090 0x7fdd0c19c350 secure :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7fdcf800ea50 tx=0x7fdcf800ed60 comp rx=0 tx=0).stop 2024-05-20T20:15:51.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.748+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 >> 172.21.15.28:0/863215178 conn(0x7fdd0c07a6a0 msgr2=0x7fdd0c100710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:51.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.751+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 shutdown_connections 2024-05-20T20:15:51.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:50.751+0000 7fdd13600700 1 -- 172.21.15.28:0/863215178 wait complete. 2024-05-20T20:15:51.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:51 smithi154 ceph-mon[36314]: pgmap v317: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 326 MiB/s wr, 1.39k op/s 2024-05-20T20:15:51.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:51 smithi033 ceph-mon[36179]: pgmap v317: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 326 MiB/s wr, 1.39k op/s 2024-05-20T20:15:52.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.825+0000 7f8a2aa00640 1 -- 172.21.15.28:0/1084303851 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a241063e0 msgr2=0x7f8a2410dfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.825+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/1084303851 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a241063e0 0x7f8a2410dfb0 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f8a14009900 tx=0x7f8a1402a420 comp rx=0 tx=0).stop 2024-05-20T20:15:52.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.825+0000 7f8a2aa00640 1 -- 172.21.15.28:0/1084303851 shutdown_connections 2024-05-20T20:15:52.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.825+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/1084303851 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a241063e0 0x7f8a2410dfb0 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.825+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/1084303851 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a24101cc0 0x7f8a24105d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.825+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/1084303851 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a24101310 0x7f8a241016f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.825+0000 7f8a2aa00640 1 -- 172.21.15.28:0/1084303851 >> 172.21.15.28:0/1084303851 conn(0x7f8a24077ee0 msgr2=0x7f8a240ff260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:52.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.826+0000 7f8a2aa00640 1 -- 172.21.15.28:0/1084303851 shutdown_connections 2024-05-20T20:15:52.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.826+0000 7f8a2aa00640 1 -- 172.21.15.28:0/1084303851 wait complete. 2024-05-20T20:15:52.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.827+0000 7f8a2aa00640 1 Processor -- start 2024-05-20T20:15:52.828 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.828+0000 7f8a2aa00640 1 -- start start 2024-05-20T20:15:52.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.829+0000 7f8a2aa00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a24101310 0x7f8a24074230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.829+0000 7f8a2aa00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a241063e0 0x7f8a2406d300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.829+0000 7f8a23e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a24101310 0x7f8a24074230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.829+0000 7f8a23e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a24101310 0x7f8a24074230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60622/0 (socket says 172.21.15.28:60622) 2024-05-20T20:15:52.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.830+0000 7f8a23e00640 1 -- 172.21.15.28:0/292309212 learned_addr learned my addr 172.21.15.28:0/292309212 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:52.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.830+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a24074880 0x7f8a2406d8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.830+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8a24112dd0 con 0x7f8a241063e0 2024-05-20T20:15:52.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.830+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8a24112c50 con 0x7f8a24101310 2024-05-20T20:15:52.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.830+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8a24112f50 con 0x7f8a24074880 2024-05-20T20:15:52.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.830+0000 7f8a28a00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a24074880 0x7f8a2406d8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.832 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.831+0000 7f8a23400640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a241063e0 0x7f8a2406d300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.831+0000 7f8a23400640 1 -- 172.21.15.28:0/292309212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a24074880 msgr2=0x7f8a2406d8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.831+0000 7f8a23400640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a24074880 0x7f8a2406d8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.831+0000 7f8a23400640 1 -- 172.21.15.28:0/292309212 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a24101310 msgr2=0x7f8a24074230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.831+0000 7f8a23400640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a24101310 0x7f8a24074230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.831+0000 7f8a23400640 1 -- 172.21.15.28:0/292309212 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8a24071d50 con 0x7f8a241063e0 2024-05-20T20:15:52.837 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.836+0000 7f8a23400640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a241063e0 0x7f8a2406d300 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c00bd40 tx=0x7f8a1c00be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:52.837 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.837+0000 7f8a20c00640 1 -- 172.21.15.28:0/292309212 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c019070 con 0x7f8a241063e0 2024-05-20T20:15:52.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.837+0000 7f8a20c00640 1 -- 172.21.15.28:0/292309212 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8a1c0148f0 con 0x7f8a241063e0 2024-05-20T20:15:52.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.837+0000 7f8a20c00640 1 -- 172.21.15.28:0/292309212 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c0128b0 con 0x7f8a241063e0 2024-05-20T20:15:52.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.837+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8a24072040 con 0x7f8a241063e0 2024-05-20T20:15:52.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.837+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8a24072540 con 0x7f8a241063e0 2024-05-20T20:15:52.853 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.846+0000 7f8a2d9b9740 1 Processor -- start 2024-05-20T20:15:52.854 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.853+0000 7f8a2d9b9740 1 -- start start 2024-05-20T20:15:52.854 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.853+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c5cfb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.854 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.853+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c70140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.855 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.853+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c72a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.855 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.853+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564857bcdb90 con 0x564857c5d4f0 2024-05-20T20:15:52.855 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.853+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564857bbd730 con 0x564857c70680 2024-05-20T20:15:52.855 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.854+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564857bc1770 con 0x564857c5cbd0 2024-05-20T20:15:52.856 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.854+0000 7f8a28a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c5cfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.856 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.854+0000 7f8a28a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c5cfb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55144/0 (socket says 172.21.15.28:55144) 2024-05-20T20:15:52.856 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.854+0000 7f8a28a00640 1 -- 172.21.15.28:0/3232033541 learned_addr learned my addr 172.21.15.28:0/3232033541 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.855+0000 7f8a23e00640 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c70140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.855+0000 7f8a28a00640 1 -- 172.21.15.28:0/3232033541 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 msgr2=0x564857c72a70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.855+0000 7f8a28a00640 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c72a70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.855+0000 7f8a28a00640 1 -- 172.21.15.28:0/3232033541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 msgr2=0x564857c70140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.855+0000 7f8a28a00640 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c70140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.855+0000 7f8a28a00640 1 -- 172.21.15.28:0/3232033541 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564857b568b0 con 0x564857c5cbd0 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.855+0000 7f8a23e00640 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c70140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:52.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.856+0000 7f8a28a00640 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c5cfb0 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7f8a14009900 tx=0x7f8a14004410 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2effd70200fae016 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:52.858 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.856+0000 7f8a19200640 1 -- 172.21.15.28:0/3232033541 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a14040070 con 0x564857c5cbd0 2024-05-20T20:15:52.858 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.856+0000 7f8a20c00640 1 -- 172.21.15.28:0/292309212 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f8a1c01c070 con 0x7f8a241063e0 2024-05-20T20:15:52.858 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.857+0000 7f8a19200640 1 -- 172.21.15.28:0/3232033541 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8a140047a0 con 0x564857c5cbd0 2024-05-20T20:15:52.858 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.857+0000 7f8a19200640 1 -- 172.21.15.28:0/3232033541 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a14035040 con 0x564857c5cbd0 2024-05-20T20:15:52.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.858+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/3232033541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 msgr2=0x564857c5cfb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.858+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c5cfb0 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7f8a14009900 tx=0x7f8a14004410 comp rx=0 tx=0).stop 2024-05-20T20:15:52.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.859+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/3232033541 shutdown_connections 2024-05-20T20:15:52.860 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.859+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c72a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.860 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.859+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c70140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.860 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.860+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/3232033541 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c5cfb0 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.860 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.860+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/3232033541 >> 172.21.15.28:0/3232033541 conn(0x564857c591a0 msgr2=0x564857c59580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:52.860 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.860+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/3232033541 shutdown_connections 2024-05-20T20:15:52.862 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.860+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/3232033541 wait complete. 2024-05-20T20:15:52.862 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.861+0000 7f8a2d9b9740 1 -- start start 2024-05-20T20:15:52.862 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.861+0000 7f8a2d9b9740 1 Processor -- start 2024-05-20T20:15:52.863 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.861+0000 7f8a20c00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8a080b2b10 0x7f8a080b4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.863 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.862+0000 7f8a20c00640 1 -- 172.21.15.28:0/292309212 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f8a1c0e1dd0 con 0x7f8a241063e0 2024-05-20T20:15:52.863 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.863+0000 7f8a28a00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8a080b2b10 0x7f8a080b4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.864 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.864+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c69de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.864+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6a320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.864+0000 7f8a23e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c69de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.865 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.864+0000 7f8a23e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c69de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55156/0 (socket says 172.21.15.28:55156) 2024-05-20T20:15:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a23e00640 1 -- 172.21.15.28:0/1200072285 learned_addr learned my addr 172.21.15.28:0/1200072285 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c6fb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5648578a3790 con 0x564857c5d4f0 2024-05-20T20:15:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564857b65bc0 con 0x564857c70680 2024-05-20T20:15:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564857c5fb10 con 0x564857c5cbd0 2024-05-20T20:15:52.866 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a23e00640 1 -- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 msgr2=0x564857c6fb00 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:15:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a23400640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6a320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a28a00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8a080b2b10 0x7f8a080b4fd0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f8a140042a0 tx=0x7f8a1402a420 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a28a00640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c6fb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a2d9b9740 10 client.0 _ll_register_callbacks cb 0x564857b43400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a23e00640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c6fb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.867 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a23e00640 1 -- 172.21.15.28:0/1200072285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 msgr2=0x564857c6a320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a23e00640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6a320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.865+0000 7f8a23e00640 1 -- 172.21.15.28:0/1200072285 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5648578d2210 con 0x564857c5cbd0 2024-05-20T20:15:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.866+0000 7f8a23400640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6a320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.866+0000 7f8a28a00640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c6fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:52.868 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.866+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8a140047a0 con 0x564857c5cbd0 2024-05-20T20:15:52.869 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.868+0000 7f8a23e00640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c69de0 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c00f210 tx=0x7f8a0c00f6d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d69772410741b665 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:52.869 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.868+0000 7f8a11600640 10 client.27433 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:15:52.869 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.868+0000 7f8a11600640 1 -- 172.21.15.28:0/1200072285 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a0c018020 con 0x564857c5cbd0 2024-05-20T20:15:52.869 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.868+0000 7f8a11600640 1 -- 172.21.15.28:0/1200072285 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f8a0c014870 con 0x564857c5cbd0 2024-05-20T20:15:52.869 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.868+0000 7f8a11600640 1 -- 172.21.15.28:0/1200072285 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8a0c014a90 con 0x564857c5cbd0 2024-05-20T20:15:52.869 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.868+0000 7f8a2d9b9740 10 client.27433 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:52.869 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.868+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x56485799e620 con 0x564857c5cbd0 2024-05-20T20:15:52.871 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.869+0000 7f8a11600640 1 -- 172.21.15.28:0/1200072285 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8a0c025020 con 0x564857c5cbd0 2024-05-20T20:15:52.871 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.869+0000 7f8a2d9b9740 10 client.27433 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:52.871 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.869+0000 7f8a2d9b9740 10 client.27433 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:52.871 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.869+0000 7f8a2d9b9740 20 client.27433 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:52.871 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.869+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857c887a0 0x564857c8ab90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.871 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.869+0000 7f8a2d9b9740 4 client.27433 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:52.872 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.869+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564857bcfae0 con 0x564857c887a0 2024-05-20T20:15:52.872 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.870+0000 7f8a23400640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857c887a0 0x564857c8ab90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.872 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.870+0000 7f8a23400640 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857c887a0 0x564857c8ab90 crc :-1 s=READY pgs=217 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a2cc1999001b6568 server_cookie=5aab0f343e3ba647 in_seq=0 out_seq=0 2024-05-20T20:15:52.872 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.870+0000 7f8a11600640 10 client.27433 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:52.873 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.871+0000 7f8a11600640 1 -- 172.21.15.28:0/1200072285 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x564857bcfae0 con 0x564857c887a0 2024-05-20T20:15:52.873 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.871+0000 7f8a11600640 10 client.27433 handle_command_reply: tid=0 2024-05-20T20:15:52.873 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.871+0000 7f8a2d9b9740 1 client.27433 shutdown 2024-05-20T20:15:52.873 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.871+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 msgr2=0x564857c69de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.873 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.871+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c69de0 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7f8a0c00f210 tx=0x7f8a0c00f6d0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.874 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 shutdown_connections 2024-05-20T20:15:52.874 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857c887a0 0x564857c8ab90 crc :-1 s=READY pgs=217 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.874 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c6fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.874 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6a320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.874 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/1200072285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c69de0 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.874 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 >> 172.21.15.28:0/1200072285 conn(0x564857c591a0 msgr2=0x564857c5ef20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:52.877 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a11600640 0 client.27433 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:52.877 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 shutdown_connections 2024-05-20T20:15:52.877 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/1200072285 wait complete. 2024-05-20T20:15:52.877 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.872+0000 7f8a2d9b9740 20 client.27433 trim_cache size 0 max 16384 2024-05-20T20:15:52.889 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 Processor -- start 2024-05-20T20:15:52.889 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 -- start start 2024-05-20T20:15:52.889 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c5cbd0 0x564857c64cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6fa40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c70680 0x564857c90430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8a241b4c50 con 0x564857c5d4f0 2024-05-20T20:15:52.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8a241b6790 con 0x564857c5cbd0 2024-05-20T20:15:52.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.888+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8a241b5320 con 0x564857c70680 2024-05-20T20:15:52.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c70680 0x564857c90430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.890 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a23400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6fa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.891 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c70680 0x564857c90430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55164/0 (socket says 172.21.15.28:55164) 2024-05-20T20:15:52.891 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 -- 172.21.15.28:0/2166170022 learned_addr learned my addr 172.21.15.28:0/2166170022 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:52.891 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a23e00640 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c5cbd0 0x564857c64cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 -- 172.21.15.28:0/2166170022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c5cbd0 msgr2=0x564857c64cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c5cbd0 0x564857c64cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 -- 172.21.15.28:0/2166170022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 msgr2=0x564857c6fa40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6fa40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a28a00640 1 -- 172.21.15.28:0/2166170022 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564857c63690 con 0x564857c70680 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.889+0000 7f8a23400640 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6fa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.890+0000 7f8a28a00640 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c70680 0x564857c90430 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f8a14043000 tx=0x7f8a14049000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7d5503b03b782b53 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.890+0000 7f8a13e00640 1 -- 172.21.15.28:0/2166170022 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a14040040 con 0x564857c70680 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.890+0000 7f8a13e00640 1 -- 172.21.15.28:0/2166170022 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8a14003ec0 con 0x564857c70680 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.890+0000 7f8a13e00640 1 -- 172.21.15.28:0/2166170022 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a1403d7f0 con 0x564857c70680 2024-05-20T20:15:52.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/2166170022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c70680 msgr2=0x564857c90430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c70680 0x564857c90430 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7f8a14043000 tx=0x7f8a14049000 comp rx=0 tx=0).stop 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/2166170022 shutdown_connections 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c70680 0x564857c90430 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c6fa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/2166170022 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c5cbd0 0x564857c64cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/2166170022 >> 172.21.15.28:0/2166170022 conn(0x564857c591a0 msgr2=0x564857c58310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/2166170022 shutdown_connections 2024-05-20T20:15:52.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/2166170022 wait complete. 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 -- start start 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.891+0000 7f8a2d9b9740 1 Processor -- start 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c9c780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c97af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a2d9b9740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c98030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8a241b60c0 con 0x564857c5d4f0 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8a241b59f0 con 0x564857c70680 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a2d9b9740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8a241b2c50 con 0x564857c5cbd0 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a2d9b9740 10 client.0 _ll_register_callbacks cb 0x7f8a0c007320 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:52.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a28a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c98030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a23e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c9c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a28a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c98030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60638/0 (socket says 172.21.15.28:60638) 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a23e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c9c780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55176/0 (socket says 172.21.15.28:55176) 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a23e00640 1 -- 172.21.15.28:0/4013432798 learned_addr learned my addr 172.21.15.28:0/4013432798 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.892+0000 7f8a23400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c97af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a28a00640 1 -- 172.21.15.28:0/4013432798 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 msgr2=0x564857c9c780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a28a00640 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c9c780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a28a00640 1 -- 172.21.15.28:0/4013432798 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 msgr2=0x564857c97af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a28a00640 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c97af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a28a00640 1 -- 172.21.15.28:0/4013432798 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564857c98860 con 0x564857c70680 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a23e00640 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c9c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:15:52.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a28a00640 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c98030 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f8a14000c00 tx=0x7f8a14031750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d534a0d58f872966 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a11600640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a11600640 1 -- 172.21.15.28:0/4013432798 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8a14040040 con 0x564857c70680 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8a14003ec0 con 0x564857c70680 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.893+0000 7f8a11600640 1 -- 172.21.15.28:0/4013432798 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f8a14031b50 con 0x564857c70680 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.894+0000 7f8a11600640 1 -- 172.21.15.28:0/4013432798 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f8a14031d50 con 0x564857c70680 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.894+0000 7f8a2d9b9740 10 client.27527 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.894+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564857cf3170 con 0x564857c70680 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.894+0000 7f8a11600640 1 -- 172.21.15.28:0/4013432798 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8a14003ec0 con 0x564857c70680 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.894+0000 7f8a11600640 1 -- 172.21.15.28:0/4013432798 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f8a1405f410 con 0x564857c70680 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.895+0000 7f8a2d9b9740 10 client.27527 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.895+0000 7f8a2d9b9740 10 client.27527 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.895+0000 7f8a2d9b9740 20 client.27527 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:52.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.895+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857cf3400 0x564857cf57f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:52.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.895+0000 7f8a2d9b9740 4 client.27527 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:15:52.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.895+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x564857bc92c0 con 0x564857cf3400 2024-05-20T20:15:52.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.895+0000 7f8a23e00640 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857cf3400 0x564857cf57f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:52.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.897+0000 7f8a11600640 10 client.27527 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:52.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.897+0000 7f8a23e00640 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857cf3400 0x564857cf57f0 crc :-1 s=READY pgs=218 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5d535d8ca2fa1b7 server_cookie=1db0699a5b08f644 in_seq=0 out_seq=0 2024-05-20T20:15:52.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.918+0000 7f8a11600640 1 -- 172.21.15.28:0/4013432798 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x564857bc92c0 con 0x564857cf3400 2024-05-20T20:15:52.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.918+0000 7f8a11600640 10 client.27527 handle_command_reply: tid=0 2024-05-20T20:15:52.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.918+0000 7f8a2d9b9740 1 client.27527 shutdown 2024-05-20T20:15:52.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.918+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 msgr2=0x564857c98030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.918+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c98030 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f8a14000c00 tx=0x7f8a14031750 comp rx=0 tx=0).stop 2024-05-20T20:15:52.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.918+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 shutdown_connections 2024-05-20T20:15:52.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.918+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564857cf3400 0x564857cf57f0 crc :-1 s=READY pgs=218 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.921+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564857c70680 0x564857c98030 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.921+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564857c5d4f0 0x564857c97af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.921+0000 7f8a2d9b9740 1 --2- 172.21.15.28:0/4013432798 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564857c5cbd0 0x564857c9c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.921+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 >> 172.21.15.28:0/4013432798 conn(0x564857c591a0 msgr2=0x564857c57a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:52.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.921+0000 7f8a11600640 0 client.27527 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:52.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.923+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 shutdown_connections 2024-05-20T20:15:52.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.923+0000 7f8a2d9b9740 1 -- 172.21.15.28:0/4013432798 wait complete. 2024-05-20T20:15:52.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.923+0000 7f8a2d9b9740 20 client.27527 trim_cache size 0 max 16384 2024-05-20T20:15:52.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:15:52.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:15:52.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:15:52.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8a080b2b10 msgr2=0x7f8a080b4fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8a080b2b10 0x7f8a080b4fd0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f8a140042a0 tx=0x7f8a1402a420 comp rx=0 tx=0).stop 2024-05-20T20:15:52.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a241063e0 msgr2=0x7f8a2406d300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:52.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a241063e0 0x7f8a2406d300 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c00bd40 tx=0x7f8a1c00be30 comp rx=0 tx=0).stop 2024-05-20T20:15:52.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 shutdown_connections 2024-05-20T20:15:52.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8a080b2b10 0x7f8a080b4fd0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8a24074880 0x7f8a2406d8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8a241063e0 0x7f8a2406d300 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 --2- 172.21.15.28:0/292309212 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8a24101310 0x7f8a24074230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:52.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.924+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 >> 172.21.15.28:0/292309212 conn(0x7f8a24077ee0 msgr2=0x7f8a240ff260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:52.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.927+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 shutdown_connections 2024-05-20T20:15:52.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:15:52.927+0000 7f8a2aa00640 1 -- 172.21.15.28:0/292309212 wait complete. 2024-05-20T20:15:53.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:53 smithi028 ceph-mon[31178]: pgmap v318: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 265 MiB/s wr, 1.12k op/s 2024-05-20T20:15:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:53 smithi154 ceph-mon[36314]: pgmap v318: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 265 MiB/s wr, 1.12k op/s 2024-05-20T20:15:53.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:53 smithi033 ceph-mon[36179]: pgmap v318: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 265 MiB/s wr, 1.12k op/s 2024-05-20T20:15:54.183 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:15:54.183 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:15:55.089 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:15:55.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.115+0000 7f8852200700 1 -- 172.21.15.28:0/2111480777 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 msgr2=0x7f884c076130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.115+0000 7f8852200700 1 --2- 172.21.15.28:0/2111480777 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 0x7f884c076130 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7f8834009990 tx=0x7f8834009ca0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.116+0000 7f8852200700 1 -- 172.21.15.28:0/2111480777 shutdown_connections 2024-05-20T20:15:55.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.116+0000 7f8852200700 1 --2- 172.21.15.28:0/2111480777 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f884c114230 0x7f884c116620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.116+0000 7f8852200700 1 --2- 172.21.15.28:0/2111480777 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f884c077560 0x7f884c075770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.116+0000 7f8852200700 1 --2- 172.21.15.28:0/2111480777 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 0x7f884c076130 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.116+0000 7f8852200700 1 -- 172.21.15.28:0/2111480777 >> 172.21.15.28:0/2111480777 conn(0x7f884c07a610 msgr2=0x7f884c07aa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:55.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.129+0000 7f8852200700 1 -- 172.21.15.28:0/2111480777 shutdown_connections 2024-05-20T20:15:55.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.131+0000 7f8852200700 1 -- 172.21.15.28:0/2111480777 wait complete. 2024-05-20T20:15:55.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.133+0000 7f8852200700 1 Processor -- start 2024-05-20T20:15:55.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.134+0000 7f8852200700 1 -- start start 2024-05-20T20:15:55.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.137+0000 7f8852200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 0x7f884c074c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.138+0000 7f8852200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f884c077560 0x7f884c075180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.138+0000 7f8852200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f884c114230 0x7f884c06de40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.139+0000 7f8852200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f884c119e50 con 0x7f884c114230 2024-05-20T20:15:55.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.139+0000 7f8852200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f884c119cd0 con 0x7f884c077560 2024-05-20T20:15:55.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.139+0000 7f8852200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f884c119fd0 con 0x7f884c075cb0 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.140+0000 7f884b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 0x7f884c074c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.140+0000 7f884b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 0x7f884c074c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55182/0 (socket says 172.21.15.28:55182) 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.140+0000 7f884b400700 1 -- 172.21.15.28:0/901260178 learned_addr learned my addr 172.21.15.28:0/901260178 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.140+0000 7f884be00700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f884c114230 0x7f884c06de40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.141+0000 7f884aa00700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f884c077560 0x7f884c075180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.142+0000 7f884aa00700 1 -- 172.21.15.28:0/901260178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 msgr2=0x7f884c074c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.142+0000 7f884aa00700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 0x7f884c074c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.142+0000 7f884aa00700 1 -- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f884c114230 msgr2=0x7f884c06de40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.142+0000 7f884aa00700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f884c114230 0x7f884c06de40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.142+0000 7f884aa00700 1 -- 172.21.15.28:0/901260178 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f884c06e740 con 0x7f884c077560 2024-05-20T20:15:55.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.143+0000 7f884aa00700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f884c077560 0x7f884c075180 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7f8834000c00 tx=0x7f8834005640 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:55.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.145+0000 7f8843e00700 1 -- 172.21.15.28:0/901260178 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8834026070 con 0x7f884c077560 2024-05-20T20:15:55.155 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:55 smithi028 ceph-mon[31178]: pgmap v319: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 350 MiB/s wr, 1.52k op/s 2024-05-20T20:15:55.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.146+0000 7f8843e00700 1 -- 172.21.15.28:0/901260178 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8834003fb0 con 0x7f884c077560 2024-05-20T20:15:55.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.146+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f884c06e9d0 con 0x7f884c077560 2024-05-20T20:15:55.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.146+0000 7f8843e00700 1 -- 172.21.15.28:0/901260178 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f883400fc60 con 0x7f884c077560 2024-05-20T20:15:55.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.146+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f884c072940 con 0x7f884c077560 2024-05-20T20:15:55.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.163+0000 7f8843e00700 1 -- 172.21.15.28:0/901260178 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f883400f460 con 0x7f884c077560 2024-05-20T20:15:55.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.166+0000 7f885cb6bb80 1 Processor -- start 2024-05-20T20:15:55.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.166+0000 7f8843e00700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f88300b2cb0 0x7f88300b5170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.166+0000 7f885cb6bb80 1 -- start start 2024-05-20T20:15:55.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.166+0000 7f8843e00700 1 -- 172.21.15.28:0/901260178 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f88340e3310 con 0x7f884c077560 2024-05-20T20:15:55.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.167+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bbe850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.167+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bb89f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.167+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bb9450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.167+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562be2b16fd0 con 0x562be2bbed90 2024-05-20T20:15:55.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.167+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562be2b06b70 con 0x562be2bb9010 2024-05-20T20:15:55.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.167+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562be2b0abb0 con 0x562be2bbe470 2024-05-20T20:15:55.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.167+0000 7f884b400700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f88300b2cb0 0x7f88300b5170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.168+0000 7f884b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bbe850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.168+0000 7f884aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bb89f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.168+0000 7f884be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bb9450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.168+0000 7f884be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bb9450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60666/0 (socket says 172.21.15.28:60666) 2024-05-20T20:15:55.170 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.168+0000 7f884be00700 1 -- 172.21.15.28:0/1221485156 learned_addr learned my addr 172.21.15.28:0/1221485156 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:55.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.170+0000 7f884b400700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f88300b2cb0 0x7f88300b5170 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f883c0060b0 tx=0x7f883c008e20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:55.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.170+0000 7f884b400700 1 -- 172.21.15.28:0/1221485156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 msgr2=0x562be2bb9450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.170+0000 7f884b400700 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bb9450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.171+0000 7f884b400700 1 -- 172.21.15.28:0/1221485156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 msgr2=0x562be2bb89f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.171+0000 7f884b400700 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bb89f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.171+0000 7f884b400700 1 -- 172.21.15.28:0/1221485156 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562be2acf5b0 con 0x562be2bbe470 2024-05-20T20:15:55.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.172+0000 7f884aa00700 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bb89f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:15:55.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.175+0000 7f884b400700 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bbe850 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f883c0154e0 tx=0x7f883c01c000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a4659caa848d3e9b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:55.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.177+0000 7f8840c00700 1 -- 172.21.15.28:0/1221485156 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f883c028070 con 0x562be2bbe470 2024-05-20T20:15:55.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.178+0000 7f8840c00700 1 -- 172.21.15.28:0/1221485156 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f883c01f070 con 0x562be2bbe470 2024-05-20T20:15:55.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.178+0000 7f8840c00700 1 -- 172.21.15.28:0/1221485156 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f883c013040 con 0x562be2bbe470 2024-05-20T20:15:55.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.180+0000 7f885cb6bb80 1 -- 172.21.15.28:0/1221485156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 msgr2=0x562be2bbe850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.180+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bbe850 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7f883c0154e0 tx=0x7f883c01c000 comp rx=0 tx=0).stop 2024-05-20T20:15:55.183 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.181+0000 7f885cb6bb80 1 -- 172.21.15.28:0/1221485156 shutdown_connections 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.181+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bb9450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.181+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bbe850 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.181+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/1221485156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bb89f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.181+0000 7f885cb6bb80 1 -- 172.21.15.28:0/1221485156 >> 172.21.15.28:0/1221485156 conn(0x562be2ae6d70 msgr2=0x562be2bb34b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.181+0000 7f885cb6bb80 1 -- 172.21.15.28:0/1221485156 shutdown_connections 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.181+0000 7f885cb6bb80 1 -- 172.21.15.28:0/1221485156 wait complete. 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.182+0000 7f885cb6bb80 1 -- start start 2024-05-20T20:15:55.184 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.183+0000 7f885cb6bb80 1 Processor -- start 2024-05-20T20:15:55.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.184+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc4a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.185 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bc1e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f884b400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc4a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.186 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bc23a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f884b400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc4a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60676/0 (socket says 172.21.15.28:60676) 2024-05-20T20:15:55.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f884b400700 1 -- 172.21.15.28:0/78507894 learned_addr learned my addr 172.21.15.28:0/78507894 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:55.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562be2a0d130 con 0x562be2bbed90 2024-05-20T20:15:55.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562be2aa78f0 con 0x562be2bb9010 2024-05-20T20:15:55.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562be2bc5810 con 0x562be2bbe470 2024-05-20T20:15:55.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.185+0000 7f885cb6bb80 10 client.0 _ll_register_callbacks cb 0x562be2ab0e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:55.187 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.186+0000 7f884aa00700 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bc1e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.186+0000 7f884be00700 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bc23a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.186+0000 7f884b400700 1 -- 172.21.15.28:0/78507894 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 msgr2=0x562be2bc1e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.188 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.186+0000 7f884b400700 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bc1e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.186+0000 7f884b400700 1 -- 172.21.15.28:0/78507894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 msgr2=0x562be2bc23a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.186+0000 7f884b400700 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bc23a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.186+0000 7f884b400700 1 -- 172.21.15.28:0/78507894 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562be2a56a60 con 0x562be2bb9010 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.187+0000 7f884b400700 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc4a70 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f883c0235b0 tx=0x7f883c023460 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5ce509588dfea49d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.187+0000 7f8839600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.187+0000 7f8839600700 1 -- 172.21.15.28:0/78507894 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f883c028070 con 0x562be2bb9010 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.187+0000 7f8839600700 1 -- 172.21.15.28:0/78507894 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f883c01b770 con 0x562be2bb9010 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.187+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f883c01f070 con 0x562be2bb9010 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.188+0000 7f8839600700 1 -- 172.21.15.28:0/78507894 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f883c004e60 con 0x562be2bb9010 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.188+0000 7f885cb6bb80 10 client.27536 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.188+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562be2ad71f0 con 0x562be2bb9010 2024-05-20T20:15:55.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.188+0000 7f8839600700 1 -- 172.21.15.28:0/78507894 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f883c01f070 con 0x562be2bb9010 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.189+0000 7f8839600700 1 -- 172.21.15.28:0/78507894 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f883c035030 con 0x562be2bb9010 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.189+0000 7f885cb6bb80 10 client.27536 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.189+0000 7f885cb6bb80 10 client.27536 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.189+0000 7f885cb6bb80 20 client.27536 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.189+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2bd22a0 0x562be2bd4690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.189+0000 7f885cb6bb80 4 client.27536 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.189+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562be2b18f20 con 0x562be2bd22a0 2024-05-20T20:15:55.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.190+0000 7f884aa00700 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2bd22a0 0x562be2bd4690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.190+0000 7f8839600700 10 client.27536 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:55.191 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.190+0000 7f884aa00700 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2bd22a0 0x562be2bd4690 crc :-1 s=READY pgs=219 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ccfab98c8d6feadc server_cookie=e5b991dedd4cc397 in_seq=0 out_seq=0 2024-05-20T20:15:55.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.191+0000 7f8839600700 1 -- 172.21.15.28:0/78507894 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562be2b18f20 con 0x562be2bd22a0 2024-05-20T20:15:55.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.191+0000 7f8839600700 10 client.27536 handle_command_reply: tid=0 2024-05-20T20:15:55.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.191+0000 7f885cb6bb80 1 client.27536 shutdown 2024-05-20T20:15:55.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 msgr2=0x562be2bc4a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc4a70 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f883c0235b0 tx=0x7f883c023460 comp rx=0 tx=0).stop 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 shutdown_connections 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2bd22a0 0x562be2bd4690 crc :-1 s=READY pgs=219 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbed90 0x562be2bc23a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc4a70 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/78507894 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbe470 0x562be2bc1e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 >> 172.21.15.28:0/78507894 conn(0x562be2ae6d70 msgr2=0x562be2bb79e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:55.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.192+0000 7f8839600700 0 client.27536 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:55.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.193+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 shutdown_connections 2024-05-20T20:15:55.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.193+0000 7f885cb6bb80 1 -- 172.21.15.28:0/78507894 wait complete. 2024-05-20T20:15:55.194 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.193+0000 7f885cb6bb80 20 client.27536 trim_cache size 0 max 16384 2024-05-20T20:15:55.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.268+0000 7f885cb6bb80 1 Processor -- start 2024-05-20T20:15:55.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.269+0000 7f885cb6bb80 1 -- start start 2024-05-20T20:15:55.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.269+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bbfa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.269+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbe470 0x562be2bb8940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.269+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bad520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.269+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f882800d630 con 0x562be2bbe470 2024-05-20T20:15:55.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.269+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f882800f170 con 0x562be2bb9010 2024-05-20T20:15:55.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.269+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f882800dd00 con 0x562be2bbed90 2024-05-20T20:15:55.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.270+0000 7f884be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bad520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.274+0000 7f884be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bad520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55204/0 (socket says 172.21.15.28:55204) 2024-05-20T20:15:55.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.274+0000 7f884be00700 1 -- 172.21.15.28:0/491913290 learned_addr learned my addr 172.21.15.28:0/491913290 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:55.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.274+0000 7f884b400700 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bbfa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.276 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.275+0000 7f884be00700 1 -- 172.21.15.28:0/491913290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 msgr2=0x562be2bbfa00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.275+0000 7f884be00700 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bbfa00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.275+0000 7f884be00700 1 -- 172.21.15.28:0/491913290 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbe470 msgr2=0x562be2bb8940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.275+0000 7f884be00700 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbe470 0x562be2bb8940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.276+0000 7f884be00700 1 -- 172.21.15.28:0/491913290 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562be2bc2500 con 0x562be2bbed90 2024-05-20T20:15:55.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.276+0000 7f884be00700 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bad520 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f884400b9a0 tx=0x7f884400bcb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=58ca7fe549fcca84 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:55.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.277+0000 7f883be00700 1 -- 172.21.15.28:0/491913290 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8844019030 con 0x562be2bbed90 2024-05-20T20:15:55.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.277+0000 7f883be00700 1 -- 172.21.15.28:0/491913290 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8844012c40 con 0x562be2bbed90 2024-05-20T20:15:55.279 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.277+0000 7f883be00700 1 -- 172.21.15.28:0/491913290 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8844015b20 con 0x562be2bbed90 2024-05-20T20:15:55.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.282+0000 7f885cb6bb80 1 -- 172.21.15.28:0/491913290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 msgr2=0x562be2bad520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.282+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bad520 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7f884400b9a0 tx=0x7f884400bcb0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.282+0000 7f885cb6bb80 1 -- 172.21.15.28:0/491913290 shutdown_connections 2024-05-20T20:15:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.282+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bad520 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.282+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bbfa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.282+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/491913290 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bbe470 0x562be2bb8940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.282+0000 7f885cb6bb80 1 -- 172.21.15.28:0/491913290 >> 172.21.15.28:0/491913290 conn(0x562be2ae6d70 msgr2=0x562be2bace10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.284+0000 7f885cb6bb80 1 -- 172.21.15.28:0/491913290 shutdown_connections 2024-05-20T20:15:55.287 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.284+0000 7f885cb6bb80 1 -- 172.21.15.28:0/491913290 wait complete. 2024-05-20T20:15:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.290+0000 7f885cb6bb80 1 -- start start 2024-05-20T20:15:55.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.291+0000 7f885cb6bb80 1 Processor -- start 2024-05-20T20:15:55.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.306+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc7b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.309+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bc8040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.309+0000 7f885cb6bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bc8580 0x562be2c3ccb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.323 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f882800eaa0 con 0x562be2bc8580 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f882800e3d0 con 0x562be2bb9010 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f885cb6bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f882800b5f0 con 0x562be2bbed90 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f884aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bc8040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f885cb6bb80 10 client.0 _ll_register_callbacks cb 0x562be2ab0e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f884aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bc8040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:55218/0 (socket says 172.21.15.28:55218) 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f884aa00700 1 -- 172.21.15.28:0/2538370591 learned_addr learned my addr 172.21.15.28:0/2538370591 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.310+0000 7f884b400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc7b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.313+0000 7f884b400700 1 -- 172.21.15.28:0/2538370591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 msgr2=0x562be2bc8040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.313+0000 7f884b400700 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bc8040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.313+0000 7f884b400700 1 -- 172.21.15.28:0/2538370591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bc8580 msgr2=0x562be2c3ccb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:55.324 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.313+0000 7f884b400700 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bc8580 0x562be2c3ccb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.313+0000 7f884b400700 1 -- 172.21.15.28:0/2538370591 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562be2c3d1f0 con 0x562be2bb9010 2024-05-20T20:15:55.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.314+0000 7f884b400700 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc7b00 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f883c023e80 tx=0x7f883c0154e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cf24873091c04a7c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:55.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.317+0000 7f8839600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:55.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.318+0000 7f8839600700 1 -- 172.21.15.28:0/2538370591 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f883c044060 con 0x562be2bb9010 2024-05-20T20:15:55.325 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.322+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8844012c40 con 0x562be2bb9010 2024-05-20T20:15:55.338 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.326+0000 7f8839600700 1 -- 172.21.15.28:0/2538370591 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f883c01b780 con 0x562be2bb9010 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.328+0000 7f8839600700 1 -- 172.21.15.28:0/2538370591 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f883c044060 con 0x562be2bb9010 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.328+0000 7f885cb6bb80 10 client.27542 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.328+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562be2c3e2d0 con 0x562be2bb9010 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.329+0000 7f8839600700 1 -- 172.21.15.28:0/2538370591 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8844012c40 con 0x562be2bb9010 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.329+0000 7f8839600700 1 -- 172.21.15.28:0/2538370591 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f883c03e030 con 0x562be2bb9010 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.330+0000 7f885cb6bb80 10 client.27542 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.330+0000 7f885cb6bb80 10 client.27542 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.330+0000 7f885cb6bb80 20 client.27542 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.330+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2c3e990 0x562be2c40db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:55.339 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.330+0000 7f885cb6bb80 4 client.27542 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 152.6, "await_for": 45.0}] 2024-05-20T20:15:55.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.330+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 152.6, "await_for": 45.0}) -- 0x562be2b12700 con 0x562be2c3e990 2024-05-20T20:15:55.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.333+0000 7f884aa00700 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2c3e990 0x562be2c40db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:55.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.335+0000 7f8839600700 10 client.27542 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:55.340 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.335+0000 7f884aa00700 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2c3e990 0x562be2c40db0 crc :-1 s=READY pgs=220 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d631b35959281092 server_cookie=9fe2ad30b62892b6 in_seq=0 out_seq=0 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.437+0000 7f8839600700 1 -- 172.21.15.28:0/2538370591 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x562be2b12700 con 0x562be2c3e990 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.437+0000 7f8839600700 10 client.27542 handle_command_reply: tid=0 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.438+0000 7f885cb6bb80 1 client.27542 shutdown 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.438+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 msgr2=0x562be2bc7b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.438+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc7b00 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f883c023e80 tx=0x7f883c0154e0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.438+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 shutdown_connections 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.439+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562be2c3e990 0x562be2c40db0 crc :-1 s=READY pgs=220 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.439+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562be2bc8580 0x562be2c3ccb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.439+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562be2bb9010 0x562be2bc7b00 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.439+0000 7f885cb6bb80 1 --2- 172.21.15.28:0/2538370591 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562be2bbed90 0x562be2bc8040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.439+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 >> 172.21.15.28:0/2538370591 conn(0x562be2ae6d70 msgr2=0x562be2bc0750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:55.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.439+0000 7f8839600700 0 client.27542 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:55.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.440+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 shutdown_connections 2024-05-20T20:15:55.441 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.440+0000 7f885cb6bb80 1 -- 172.21.15.28:0/2538370591 wait complete. 2024-05-20T20:15:55.442 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.440+0000 7f885cb6bb80 20 client.27542 trim_cache size 0 max 16384 2024-05-20T20:15:55.442 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:15:55.442 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:15:55.442 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:15:55.442 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 26, 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "7ee8534d": { 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 26, 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 152.6, 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:15:55.443 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:15:55.444 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f88300b2cb0 msgr2=0x7f88300b5170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f88300b2cb0 0x7f88300b5170 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f883c0060b0 tx=0x7f883c008e20 comp rx=0 tx=0).stop 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f884c077560 msgr2=0x7f884c075180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f884c077560 0x7f884c075180 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7f8834000c00 tx=0x7f8834005640 comp rx=0 tx=0).stop 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 shutdown_connections 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f884c114230 0x7f884c06de40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f884c075cb0 0x7f884c074c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f88300b2cb0 0x7f88300b5170 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 --2- 172.21.15.28:0/901260178 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f884c077560 0x7f884c075180 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:55.445 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.442+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 >> 172.21.15.28:0/901260178 conn(0x7f884c07a610 msgr2=0x7f884c112ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:55.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.445+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 shutdown_connections 2024-05-20T20:15:55.446 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:55.445+0000 7f8852200700 1 -- 172.21.15.28:0/901260178 wait complete. 2024-05-20T20:15:55.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:55 smithi154 ceph-mon[36314]: pgmap v319: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 350 MiB/s wr, 1.52k op/s 2024-05-20T20:15:55.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:55 smithi033 ceph-mon[36179]: pgmap v319: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 350 MiB/s wr, 1.52k op/s 2024-05-20T20:15:57.053 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:56 smithi028 ceph-mon[31178]: pgmap v320: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 261 MiB/s wr, 1.19k op/s 2024-05-20T20:15:57.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:56 smithi154 ceph-mon[36314]: pgmap v320: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 261 MiB/s wr, 1.19k op/s 2024-05-20T20:15:57.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:56 smithi033 ceph-mon[36179]: pgmap v320: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 261 MiB/s wr, 1.19k op/s 2024-05-20T20:15:59.023 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set '7ee8534d', quiesce took 26.9 seconds. Will release after: 5.700000000000003 2024-05-20T20:15:59.175 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:15:59 smithi028 ceph-mon[31178]: pgmap v321: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 270 MiB/s wr, 1.22k op/s 2024-05-20T20:15:59.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:15:59 smithi154 ceph-mon[36314]: pgmap v321: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 270 MiB/s wr, 1.22k op/s 2024-05-20T20:15:59.376 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:15:59 smithi033 ceph-mon[36179]: pgmap v321: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 270 MiB/s wr, 1.22k op/s 2024-05-20T20:15:59.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.792+0000 7f697c800700 1 -- 172.21.15.28:0/3842292403 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410f2d0 msgr2=0x7f6974116ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.792+0000 7f697c800700 1 --2- 172.21.15.28:0/3842292403 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410f2d0 0x7f6974116ea0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f6970009f90 tx=0x7f697000b330 comp rx=0 tx=0).stop 2024-05-20T20:15:59.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.792+0000 7f697c800700 1 -- 172.21.15.28:0/3842292403 shutdown_connections 2024-05-20T20:15:59.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.792+0000 7f697c800700 1 --2- 172.21.15.28:0/3842292403 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410f2d0 0x7f6974116ea0 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.792+0000 7f697c800700 1 --2- 172.21.15.28:0/3842292403 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f697410a7a0 0x7f697410ab80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.792+0000 7f697c800700 1 --2- 172.21.15.28:0/3842292403 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f697410b150 0x7f697410ebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.792+0000 7f697c800700 1 -- 172.21.15.28:0/3842292403 >> 172.21.15.28:0/3842292403 conn(0x7f697406e6b0 msgr2=0x7f697406eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:59.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.793+0000 7f697c800700 1 -- 172.21.15.28:0/3842292403 shutdown_connections 2024-05-20T20:15:59.795 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.793+0000 7f697c800700 1 -- 172.21.15.28:0/3842292403 wait complete. 2024-05-20T20:15:59.796 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.794+0000 7f697c800700 1 Processor -- start 2024-05-20T20:15:59.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.794+0000 7f697c800700 1 -- start start 2024-05-20T20:15:59.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.795+0000 7f697c800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410a7a0 0x7f69741ae6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.795+0000 7f697c800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f697410b150 0x7f69741aec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.797 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.795+0000 7f697c800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f697410f2d0 0x7f6974116040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.795+0000 7f6979e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410a7a0 0x7f69741ae6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697c800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f697411bfb0 con 0x7f697410f2d0 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697c800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f697411be30 con 0x7f697410b150 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697c800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f697411c130 con 0x7f697410a7a0 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697a800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f697410f2d0 0x7f6974116040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f6979400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f697410b150 0x7f69741aec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f6979400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f697410b150 0x7f69741aec00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56530/0 (socket says 172.21.15.28:56530) 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f6979400700 1 -- 172.21.15.28:0/2392129646 learned_addr learned my addr 172.21.15.28:0/2392129646 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:59.798 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697a800700 1 -- 172.21.15.28:0/2392129646 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410a7a0 msgr2=0x7f69741ae6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697a800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410a7a0 0x7f69741ae6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697a800700 1 -- 172.21.15.28:0/2392129646 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f697410b150 msgr2=0x7f69741aec00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697a800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f697410b150 0x7f69741aec00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.796+0000 7f697a800700 1 -- 172.21.15.28:0/2392129646 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6974116820 con 0x7f697410f2d0 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.797+0000 7f697a800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f697410f2d0 0x7f6974116040 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f697000b6c0 tx=0x7f6970009510 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.797+0000 7f696ea00700 1 -- 172.21.15.28:0/2392129646 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6970021070 con 0x7f697410f2d0 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.797+0000 7f696ea00700 1 -- 172.21.15.28:0/2392129646 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6970007ac0 con 0x7f697410f2d0 2024-05-20T20:15:59.799 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.797+0000 7f696ea00700 1 -- 172.21.15.28:0/2392129646 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f69700052a0 con 0x7f697410f2d0 2024-05-20T20:15:59.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.797+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6974116ab0 con 0x7f697410f2d0 2024-05-20T20:15:59.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.797+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f69741b5520 con 0x7f697410f2d0 2024-05-20T20:15:59.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.805+0000 7f696ea00700 1 -- 172.21.15.28:0/2392129646 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f6970024070 con 0x7f697410f2d0 2024-05-20T20:15:59.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.806+0000 7f696ea00700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f695c0b29f0 0x7f695c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.806+0000 7f696ea00700 1 -- 172.21.15.28:0/2392129646 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f69700e3640 con 0x7f697410f2d0 2024-05-20T20:15:59.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.807+0000 7f6979e00700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f695c0b29f0 0x7f695c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.808+0000 7f6979e00700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f695c0b29f0 0x7f695c0b4eb0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f6968005950 tx=0x7f696800b410 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:59.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.809+0000 7f69870bcb80 1 Processor -- start 2024-05-20T20:15:59.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.809+0000 7f69870bcb80 1 -- start start 2024-05-20T20:15:59.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.810+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89690 0x55d0f5b89a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.810+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b83ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.810+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b841c0 0x55d0f5b84640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.810+0000 7f69870bcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d0f5ae1ec0 con 0x55d0f5b89fb0 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.810+0000 7f69870bcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d0f5ad1a60 con 0x55d0f5b841c0 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.810+0000 7f69870bcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d0f5ad5aa0 con 0x55d0f5b89690 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b83ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b83ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42064/0 (socket says 172.21.15.28:42064) 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979400700 1 -- 172.21.15.28:0/1982402294 learned_addr learned my addr 172.21.15.28:0/1982402294 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:59.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979e00700 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89690 0x55d0f5b89a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979e00700 1 -- 172.21.15.28:0/1982402294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b841c0 msgr2=0x55d0f5b84640 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:15:59.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979e00700 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b841c0 0x55d0f5b84640 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979e00700 1 -- 172.21.15.28:0/1982402294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 msgr2=0x55d0f5b83ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:59.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979e00700 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b83ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.811+0000 7f6979e00700 1 -- 172.21.15.28:0/1982402294 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d0f5a9a4a0 con 0x55d0f5b89690 2024-05-20T20:15:59.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.812+0000 7f6979e00700 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89690 0x55d0f5b89a70 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f6968005d90 tx=0x7f6968013000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2c6824d2a0f30d32 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:59.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.812+0000 7f6967400700 1 -- 172.21.15.28:0/1982402294 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6968011b80 con 0x55d0f5b89690 2024-05-20T20:15:59.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.812+0000 7f6967400700 1 -- 172.21.15.28:0/1982402294 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6968011d20 con 0x55d0f5b89690 2024-05-20T20:15:59.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.812+0000 7f6967400700 1 -- 172.21.15.28:0/1982402294 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f696801e420 con 0x55d0f5b89690 2024-05-20T20:15:59.814 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 -- 172.21.15.28:0/1982402294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89690 msgr2=0x55d0f5b89a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89690 0x55d0f5b89a70 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f6968005d90 tx=0x7f6968013000 comp rx=0 tx=0).stop 2024-05-20T20:15:59.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 -- 172.21.15.28:0/1982402294 shutdown_connections 2024-05-20T20:15:59.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b841c0 0x55d0f5b84640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89690 0x55d0f5b89a70 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1982402294 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b83ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 -- 172.21.15.28:0/1982402294 >> 172.21.15.28:0/1982402294 conn(0x55d0f5ab1c60 msgr2=0x55d0f5b7e710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:59.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.814+0000 7f69870bcb80 1 -- 172.21.15.28:0/1982402294 shutdown_connections 2024-05-20T20:15:59.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.815+0000 7f69870bcb80 1 -- 172.21.15.28:0/1982402294 wait complete. 2024-05-20T20:15:59.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.816+0000 7f69870bcb80 1 -- start start 2024-05-20T20:15:59.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.817+0000 7f69870bcb80 1 Processor -- start 2024-05-20T20:15:59.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b841c0 0x55d0f5b8fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b8d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b8d710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f69870bcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d0f59d8020 con 0x55d0f5b89fb0 2024-05-20T20:15:59.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f69870bcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d0f5a727e0 con 0x55d0f5b89690 2024-05-20T20:15:59.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f69870bcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d0f5b90a60 con 0x55d0f5b841c0 2024-05-20T20:15:59.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f69870bcb80 10 client.0 _ll_register_callbacks cb 0x55d0f5a7bcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:59.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f6979400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b8d1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f6979400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b8d1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56558/0 (socket says 172.21.15.28:56558) 2024-05-20T20:15:59.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f6979400700 1 -- 172.21.15.28:0/1120964493 learned_addr learned my addr 172.21.15.28:0/1120964493 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:59.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.818+0000 7f6979e00700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b841c0 0x55d0f5b8fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.819+0000 7f6979400700 1 -- 172.21.15.28:0/1120964493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b841c0 msgr2=0x55d0f5b8fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:59.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.819+0000 7f697a800700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b8d710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.819+0000 7f6979400700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b841c0 0x55d0f5b8fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.820+0000 7f6979400700 1 -- 172.21.15.28:0/1120964493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 msgr2=0x55d0f5b8d710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:59.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.820+0000 7f6979400700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b8d710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.820+0000 7f6979400700 1 -- 172.21.15.28:0/1120964493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d0f5a21950 con 0x55d0f5b89690 2024-05-20T20:15:59.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.820+0000 7f697a800700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b8d710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:15:59.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.820+0000 7f6979e00700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b841c0 0x55d0f5b8fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:15:59.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.820+0000 7f6979400700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b8d1d0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f6960012790 tx=0x7f6960012b50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e3c48740e808d5e9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:59.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.820+0000 7f695be00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:15:59.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.821+0000 7f695be00700 1 -- 172.21.15.28:0/1120964493 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6960011850 con 0x55d0f5b89690 2024-05-20T20:15:59.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.821+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6968011d20 con 0x55d0f5b89690 2024-05-20T20:15:59.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.822+0000 7f695be00700 1 -- 172.21.15.28:0/1120964493 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f69600119f0 con 0x55d0f5b89690 2024-05-20T20:15:59.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.822+0000 7f695be00700 1 -- 172.21.15.28:0/1120964493 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6960011bf0 con 0x55d0f5b89690 2024-05-20T20:15:59.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.823+0000 7f69870bcb80 10 client.27551 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:59.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.823+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d0f5aa20e0 con 0x55d0f5b89690 2024-05-20T20:15:59.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.823+0000 7f695be00700 1 -- 172.21.15.28:0/1120964493 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6968011d20 con 0x55d0f5b89690 2024-05-20T20:15:59.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.825+0000 7f695be00700 1 -- 172.21.15.28:0/1120964493 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f6960004750 con 0x55d0f5b89690 2024-05-20T20:15:59.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.825+0000 7f69870bcb80 10 client.27551 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:59.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.825+0000 7f69870bcb80 10 client.27551 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:59.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.826+0000 7f69870bcb80 20 client.27551 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:59.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.826+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5baea00 0x55d0f5bb0e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.826+0000 7f69870bcb80 4 client.27551 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:15:59.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.827+0000 7f6979e00700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5baea00 0x55d0f5bb0e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.827+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d0f5ae3e10 con 0x55d0f5baea00 2024-05-20T20:15:59.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.828+0000 7f6979e00700 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5baea00 0x55d0f5bb0e60 crc :-1 s=READY pgs=221 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e1c3a5bb606052c4 server_cookie=cdb0763cc185c19 in_seq=0 out_seq=0 2024-05-20T20:15:59.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.828+0000 7f695be00700 10 client.27551 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:59.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.830+0000 7f695be00700 1 -- 172.21.15.28:0/1120964493 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55d0f5ae3e10 con 0x55d0f5baea00 2024-05-20T20:15:59.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.830+0000 7f695be00700 10 client.27551 handle_command_reply: tid=0 2024-05-20T20:15:59.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.830+0000 7f69870bcb80 1 client.27551 shutdown 2024-05-20T20:15:59.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 msgr2=0x55d0f5b8d1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b8d1d0 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f6960012790 tx=0x7f6960012b50 comp rx=0 tx=0).stop 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 shutdown_connections 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5baea00 0x55d0f5bb0e60 crc :-1 s=READY pgs=221 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b8d710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b841c0 0x55d0f5b8fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 --2- 172.21.15.28:0/1120964493 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b8d1d0 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 >> 172.21.15.28:0/1120964493 conn(0x55d0f5ab1c60 msgr2=0x55d0f5b7ddf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f695be00700 0 client.27551 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 shutdown_connections 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.833+0000 7f69870bcb80 1 -- 172.21.15.28:0/1120964493 wait complete. 2024-05-20T20:15:59.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.834+0000 7f69870bcb80 20 client.27551 trim_cache size 0 max 16384 2024-05-20T20:15:59.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.876+0000 7f69870bcb80 1 Processor -- start 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.876+0000 7f69870bcb80 1 -- start start 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.876+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5b87810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.876+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b88b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.876+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b7e570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f69870bcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f69741a8940 con 0x55d0f5b841c0 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f69870bcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f69741b5fb0 con 0x55d0f5b89690 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f69870bcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f69741a9010 con 0x55d0f5b89fb0 2024-05-20T20:15:59.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f6979e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5b87810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f6979e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5b87810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42090/0 (socket says 172.21.15.28:42090) 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f6979e00700 1 -- 172.21.15.28:0/2318397320 learned_addr learned my addr 172.21.15.28:0/2318397320 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f6979400700 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b88b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.877+0000 7f697a800700 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b7e570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.878+0000 7f6979e00700 1 -- 172.21.15.28:0/2318397320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 msgr2=0x55d0f5b7e570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.878+0000 7f6979e00700 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b7e570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.878+0000 7f6979e00700 1 -- 172.21.15.28:0/2318397320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 msgr2=0x55d0f5b88b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.878+0000 7f6979e00700 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b88b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.878+0000 7f6979e00700 1 -- 172.21.15.28:0/2318397320 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d0f5b8b740 con 0x55d0f5b841c0 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.878+0000 7f6979e00700 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5b87810 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f6968005350 tx=0x7f6968011c40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5df14eb12306fac5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.879+0000 7f6966a00700 1 -- 172.21.15.28:0/2318397320 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6968026440 con 0x55d0f5b841c0 2024-05-20T20:15:59.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.879+0000 7f6966a00700 1 -- 172.21.15.28:0/2318397320 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6968026a90 con 0x55d0f5b841c0 2024-05-20T20:15:59.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.879+0000 7f6966a00700 1 -- 172.21.15.28:0/2318397320 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6968004b70 con 0x55d0f5b841c0 2024-05-20T20:15:59.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.879+0000 7f69870bcb80 1 -- 172.21.15.28:0/2318397320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 msgr2=0x55d0f5b87810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.879+0000 7f69870bcb80 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5b87810 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f6968005350 tx=0x7f6968011c40 comp rx=0 tx=0).stop 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 -- 172.21.15.28:0/2318397320 shutdown_connections 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 0x55d0f5b7e570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5b87810 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 --2- 172.21.15.28:0/2318397320 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5b88b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 -- 172.21.15.28:0/2318397320 >> 172.21.15.28:0/2318397320 conn(0x55d0f5ab1c60 msgr2=0x55d0f5b98d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 -- 172.21.15.28:0/2318397320 shutdown_connections 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 -- 172.21.15.28:0/2318397320 wait complete. 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.880+0000 7f69870bcb80 1 -- start start 2024-05-20T20:15:59.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 1 Processor -- start 2024-05-20T20:15:59.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5bb4b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5bb50c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 0x55d0f5c185c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f69741b58e0 con 0x55d0f5b841c0 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f69741a96e0 con 0x55d0f5b89690 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f69741a6880 con 0x55d0f5b89fb0 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.881+0000 7f69870bcb80 10 client.0 _ll_register_callbacks cb 0x55d0f5a7bcf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.882+0000 7f6979e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5bb4b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.882+0000 7f6979e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5bb4b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42092/0 (socket says 172.21.15.28:42092) 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.882+0000 7f6979e00700 1 -- 172.21.15.28:0/3099873120 learned_addr learned my addr 172.21.15.28:0/3099873120 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:15:59.884 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.882+0000 7f6979400700 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5bb50c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.882+0000 7f6979e00700 1 -- 172.21.15.28:0/3099873120 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 msgr2=0x55d0f5c185c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:15:59.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.882+0000 7f6979e00700 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 0x55d0f5c185c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.883+0000 7f6979e00700 1 -- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 msgr2=0x55d0f5bb50c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:15:59.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.883+0000 7f6979e00700 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5bb50c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.883+0000 7f6979e00700 1 -- 172.21.15.28:0/3099873120 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d0f5c18b00 con 0x55d0f5b841c0 2024-05-20T20:15:59.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.884+0000 7f6979e00700 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5bb4b80 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f6968005a70 tx=0x7f6968005610 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b051284f7f191787 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:15:59.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.884+0000 7f695be00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:15:59.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.885+0000 7f695be00700 1 -- 172.21.15.28:0/3099873120 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6968026e00 con 0x55d0f5b841c0 2024-05-20T20:15:59.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.885+0000 7f695be00700 1 -- 172.21.15.28:0/3099873120 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7f6968026650 con 0x55d0f5b841c0 2024-05-20T20:15:59.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.885+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6968026a90 con 0x55d0f5b841c0 2024-05-20T20:15:59.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.885+0000 7f695be00700 1 -- 172.21.15.28:0/3099873120 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6968026870 con 0x55d0f5b841c0 2024-05-20T20:15:59.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.885+0000 7f69870bcb80 10 client.17082 fetch_fsmap learned FSMap version 29 2024-05-20T20:15:59.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.885+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d0f5c19e40 con 0x55d0f5b841c0 2024-05-20T20:15:59.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.886+0000 7f695be00700 1 -- 172.21.15.28:0/3099873120 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f69680488d0 con 0x55d0f5b841c0 2024-05-20T20:15:59.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.887+0000 7f695be00700 1 -- 172.21.15.28:0/3099873120 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f696801e6f0 con 0x55d0f5b841c0 2024-05-20T20:15:59.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.887+0000 7f69870bcb80 10 client.17082 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:15:59.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.887+0000 7f69870bcb80 10 client.17082 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:15:59.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.887+0000 7f69870bcb80 20 client.17082 populate_metadata read hostname 'smithi028' 2024-05-20T20:15:59.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.887+0000 7f69870bcb80 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5c1a2a0 0x55d0f5c1c690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:15:59.888 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.887+0000 7f69870bcb80 4 client.17082 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:15:59.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.887+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x7f6968026a90 con 0x55d0f5c1a2a0 2024-05-20T20:15:59.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.888+0000 7f6979400700 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5c1a2a0 0x55d0f5c1c690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:15:59.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.888+0000 7f6979400700 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5c1a2a0 0x55d0f5c1c690 crc :-1 s=READY pgs=222 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=923e7ac7bee6375e server_cookie=e0d32bc88c75fa35 in_seq=0 out_seq=0 2024-05-20T20:15:59.889 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.888+0000 7f695be00700 10 client.17082 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:59.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.889+0000 7f695be00700 1 -- 172.21.15.28:0/3099873120 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x7f6968026a90 con 0x55d0f5c1a2a0 2024-05-20T20:15:59.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.889+0000 7f695be00700 10 client.17082 handle_command_reply: tid=0 2024-05-20T20:15:59.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.889+0000 7f69870bcb80 1 client.17082 shutdown 2024-05-20T20:15:59.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.889+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 msgr2=0x55d0f5bb4b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.889+0000 7f69870bcb80 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5bb4b80 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f6968005a70 tx=0x7f6968005610 comp rx=0 tx=0).stop 2024-05-20T20:15:59.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 shutdown_connections 2024-05-20T20:15:59.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d0f5c1a2a0 0x55d0f5c1c690 crc :-1 s=READY pgs=222 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f695be00700 0 client.17082 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:15:59.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d0f5b89fb0 0x55d0f5c185c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d0f5b841c0 0x55d0f5bb4b80 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 --2- 172.21.15.28:0/3099873120 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d0f5b89690 0x55d0f5bb50c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 >> 172.21.15.28:0/3099873120 conn(0x55d0f5ab1c60 msgr2=0x55d0f5b98d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 shutdown_connections 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 1 -- 172.21.15.28:0/3099873120 wait complete. 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.890+0000 7f69870bcb80 20 client.17082 trim_cache size 0 max 16384 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:15:59.892 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "de8cabc1-8202-44ad-be51-742e95f335d6", 2024-05-20T20:15:59.893 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:15:59.893 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:15:59.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f695c0b29f0 msgr2=0x7f695c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f695c0b29f0 0x7f695c0b4eb0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f6968005950 tx=0x7f696800b410 comp rx=0 tx=0).stop 2024-05-20T20:15:59.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f697410f2d0 msgr2=0x7f6974116040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:15:59.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f697410f2d0 0x7f6974116040 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f697000b6c0 tx=0x7f6970009510 comp rx=0 tx=0).stop 2024-05-20T20:15:59.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 shutdown_connections 2024-05-20T20:15:59.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f697410f2d0 0x7f6974116040 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f697410a7a0 0x7f69741ae6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f695c0b29f0 0x7f695c0b4eb0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 --2- 172.21.15.28:0/2392129646 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f697410b150 0x7f69741aec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:15:59.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 >> 172.21.15.28:0/2392129646 conn(0x7f697406e6b0 msgr2=0x7f6974073550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:15:59.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.895+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 shutdown_connections 2024-05-20T20:15:59.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:15:59.896+0000 7f697c800700 1 -- 172.21.15.28:0/2392129646 wait complete. 2024-05-20T20:16:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:01 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:16:01.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:01 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:16:01.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:01 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:16:01.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:01 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:16:01.376 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:01 smithi154 ceph-mon[36314]: pgmap v322: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 332 MiB/s wr, 1.49k op/s 2024-05-20T20:16:01.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:01 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:16:01.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:01 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:16:01.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:01 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:16:01.377 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:01 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:16:01.378 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:01 smithi033 ceph-mon[36179]: pgmap v322: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 332 MiB/s wr, 1.49k op/s 2024-05-20T20:16:01.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:01 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:16:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:01 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:16:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:01 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:16:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:01 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:16:01.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:01 smithi028 ceph-mon[31178]: pgmap v322: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 332 MiB/s wr, 1.49k op/s 2024-05-20T20:16:02.081 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:16:02.797 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:03.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:03 smithi028 ceph-mon[31178]: pgmap v323: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 262 MiB/s wr, 1.17k op/s 2024-05-20T20:16:03.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:03 smithi154 ceph-mon[36314]: pgmap v323: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 262 MiB/s wr, 1.17k op/s 2024-05-20T20:16:03.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:03 smithi033 ceph-mon[36179]: pgmap v323: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 262 MiB/s wr, 1.17k op/s 2024-05-20T20:16:04.306 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:04 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:16:04.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:04 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:16:04.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:04 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:16:04.724 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:16:05.216 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:05 smithi028 ceph-mon[31178]: pgmap v324: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 362 MiB/s wr, 1.76k op/s 2024-05-20T20:16:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:05 smithi154 ceph-mon[36314]: pgmap v324: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 362 MiB/s wr, 1.76k op/s 2024-05-20T20:16:05.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:05 smithi033 ceph-mon[36179]: pgmap v324: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 362 MiB/s wr, 1.76k op/s 2024-05-20T20:16:05.885 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:06.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 -- 172.21.15.28:0/547186138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd45010e380 msgr2=0x7fd450115f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 --2- 172.21.15.28:0/547186138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd45010e380 0x7fd450115f50 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7fd448009be0 tx=0x7fd44802a6c0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 -- 172.21.15.28:0/547186138 shutdown_connections 2024-05-20T20:16:06.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 --2- 172.21.15.28:0/547186138 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd45010e380 0x7fd450115f50 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 --2- 172.21.15.28:0/547186138 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd450109c60 0x7fd45010dca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.159 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 --2- 172.21.15.28:0/547186138 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd4501092b0 0x7fd450109690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 -- 172.21.15.28:0/547186138 >> 172.21.15.28:0/547186138 conn(0x7fd450076fe0 msgr2=0x7fd450079400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:06.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 -- 172.21.15.28:0/547186138 shutdown_connections 2024-05-20T20:16:06.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.159+0000 7fd457800640 1 -- 172.21.15.28:0/547186138 wait complete. 2024-05-20T20:16:06.160 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.161+0000 7fd457800640 1 Processor -- start 2024-05-20T20:16:06.161 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.161+0000 7fd457800640 1 -- start start 2024-05-20T20:16:06.162 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd457800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4501092b0 0x7fd45019f750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.162 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd457800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd45010e380 0x7fd45019fc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.162 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd457800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4501a4010 0x7fd4501a44c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.162 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd457800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd4501192d0 con 0x7fd4501a4010 2024-05-20T20:16:06.162 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd457800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd450119150 con 0x7fd45010e380 2024-05-20T20:16:06.162 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd457800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd450119450 con 0x7fd4501092b0 2024-05-20T20:16:06.162 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd454e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4501092b0 0x7fd45019f750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.163 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd454e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4501092b0 0x7fd45019f750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53636/0 (socket says 172.21.15.28:53636) 2024-05-20T20:16:06.163 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd454e00640 1 -- 172.21.15.28:0/1225169458 learned_addr learned my addr 172.21.15.28:0/1225169458 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:06.163 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.162+0000 7fd455800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4501a4010 0x7fd4501a44c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.163 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.163+0000 7fd454e00640 1 -- 172.21.15.28:0/1225169458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd45010e380 msgr2=0x7fd45019fc90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.163 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.163+0000 7fd454e00640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd45010e380 0x7fd45019fc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.164 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.163+0000 7fd454e00640 1 -- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4501a4010 msgr2=0x7fd4501a44c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.164 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.163+0000 7fd454e00640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4501a4010 0x7fd4501a44c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.164 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.163+0000 7fd454e00640 1 -- 172.21.15.28:0/1225169458 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd4501a4b00 con 0x7fd4501092b0 2024-05-20T20:16:06.164 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.164+0000 7fd454e00640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4501092b0 0x7fd45019f750 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7fd43c00d950 tx=0x7fd43c00de10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:06.165 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.164+0000 7fd44d600640 1 -- 172.21.15.28:0/1225169458 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd43c017030 con 0x7fd4501092b0 2024-05-20T20:16:06.165 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.164+0000 7fd44d600640 1 -- 172.21.15.28:0/1225169458 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd43c004020 con 0x7fd4501092b0 2024-05-20T20:16:06.165 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.164+0000 7fd44d600640 1 -- 172.21.15.28:0/1225169458 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd43c0047a0 con 0x7fd4501092b0 2024-05-20T20:16:06.165 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.165+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd4501a4df0 con 0x7fd4501092b0 2024-05-20T20:16:06.165 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.165+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd4501ac590 con 0x7fd4501092b0 2024-05-20T20:16:06.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.170+0000 7fd45a628740 1 Processor -- start 2024-05-20T20:16:06.170 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.171+0000 7fd45a628740 1 -- start start 2024-05-20T20:16:06.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd45a628740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce40070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.172 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd45a628740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce405b0 0x557ebce4a930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd45a628740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce4d260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd45a628740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557ebcdb0b90 con 0x557ebce405b0 2024-05-20T20:16:06.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd45a628740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557ebcda0730 con 0x557ebce3fc90 2024-05-20T20:16:06.173 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd45a628740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557ebcda4770 con 0x557ebce4ae70 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce4d260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd455800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce40070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce4d260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53650/0 (socket says 172.21.15.28:53650) 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 -- 172.21.15.28:0/3060759523 learned_addr learned my addr 172.21.15.28:0/3060759523 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 -- 172.21.15.28:0/3060759523 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 msgr2=0x557ebce40070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 --2- 172.21.15.28:0/3060759523 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce40070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 -- 172.21.15.28:0/3060759523 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce405b0 msgr2=0x557ebce4a930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 --2- 172.21.15.28:0/3060759523 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce405b0 0x557ebce4a930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.174 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.173+0000 7fd44fe00640 1 -- 172.21.15.28:0/3060759523 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557ebcd398b0 con 0x557ebce4ae70 2024-05-20T20:16:06.175 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.174+0000 7fd44fe00640 1 --2- 172.21.15.28:0/3060759523 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce4d260 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7fd444009b50 tx=0x7fd44400e720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b5b9a024849dbfa5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:06.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.174+0000 7fd442000640 1 -- 172.21.15.28:0/3060759523 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd444009e40 con 0x557ebce4ae70 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.174+0000 7fd442000640 1 -- 172.21.15.28:0/3060759523 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd44400ee70 con 0x557ebce4ae70 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.174+0000 7fd442000640 1 -- 172.21.15.28:0/3060759523 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd444018920 con 0x557ebce4ae70 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.174+0000 7fd44d600640 1 -- 172.21.15.28:0/1225169458 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd43c009a30 con 0x7fd4501092b0 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 -- 172.21.15.28:0/3060759523 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 msgr2=0x557ebce4d260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 --2- 172.21.15.28:0/3060759523 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce4d260 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7fd444009b50 tx=0x7fd44400e720 comp rx=0 tx=0).stop 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 -- 172.21.15.28:0/3060759523 shutdown_connections 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 --2- 172.21.15.28:0/3060759523 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce4d260 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 --2- 172.21.15.28:0/3060759523 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce405b0 0x557ebce4a930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 --2- 172.21.15.28:0/3060759523 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce40070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 -- 172.21.15.28:0/3060759523 >> 172.21.15.28:0/3060759523 conn(0x557ebce3c1a0 msgr2=0x557ebce3c580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:06.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 -- 172.21.15.28:0/3060759523 shutdown_connections 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.175+0000 7fd45a628740 1 -- 172.21.15.28:0/3060759523 wait complete. 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.176+0000 7fd45a628740 1 -- start start 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.176+0000 7fd45a628740 1 Processor -- start 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd45a628740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce47c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd45a628740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce481d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd45a628740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce48710 0x557ebce56d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd45a628740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557ebca86790 con 0x557ebce48710 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd45a628740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557ebcd48bc0 con 0x557ebce3fc90 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd45a628740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557ebce4daa0 con 0x557ebce4ae70 2024-05-20T20:16:06.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd45a628740 10 client.0 _ll_register_callbacks cb 0x557ebcd26400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd454e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce481d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd454e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce481d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53654/0 (socket says 172.21.15.28:53654) 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd454e00640 1 -- 172.21.15.28:0/2681549866 learned_addr learned my addr 172.21.15.28:0/2681549866 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.177+0000 7fd44fe00640 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce48710 0x557ebce56d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.178+0000 7fd44fe00640 1 -- 172.21.15.28:0/2681549866 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 msgr2=0x557ebce481d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.178+0000 7fd44fe00640 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce481d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.178+0000 7fd44fe00640 1 -- 172.21.15.28:0/2681549866 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 msgr2=0x557ebce47c90 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:06.180 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.178+0000 7fd44fe00640 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce47c90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.178+0000 7fd44fe00640 1 -- 172.21.15.28:0/2681549866 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557ebcab5210 con 0x557ebce48710 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.178+0000 7fd454e00640 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce481d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.179+0000 7fd44fe00640 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce48710 0x557ebce56d50 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fd444000c00 tx=0x7fd4440095f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2224f2fbc7059585 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.179+0000 7fd436a00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.179+0000 7fd436a00640 1 -- 172.21.15.28:0/2681549866 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd444027020 con 0x557ebce48710 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.179+0000 7fd436a00640 1 -- 172.21.15.28:0/2681549866 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fd444005600 con 0x557ebce48710 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.179+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd44400ee70 con 0x557ebce48710 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.179+0000 7fd44d600640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4380b29a0 0x7fd4380b4e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.180+0000 7fd436a00640 1 -- 172.21.15.28:0/2681549866 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd444032050 con 0x557ebce48710 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.180+0000 7fd45a628740 10 client.17091 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.180+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557ebcb81620 con 0x557ebce48710 2024-05-20T20:16:06.181 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.180+0000 7fd44d600640 1 -- 172.21.15.28:0/1225169458 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fd43c0e20c0 con 0x7fd4501092b0 2024-05-20T20:16:06.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.180+0000 7fd455800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4380b29a0 0x7fd4380b4e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd455800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4380b29a0 0x7fd4380b4e60 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fd448006010 tx=0x7fd448004a40 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:06.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd436a00640 1 -- 172.21.15.28:0/2681549866 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd44400ee70 con 0x557ebce48710 2024-05-20T20:16:06.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd436a00640 1 -- 172.21.15.28:0/2681549866 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd444018af0 con 0x557ebce48710 2024-05-20T20:16:06.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd45a628740 10 client.17091 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:06.183 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd45a628740 10 client.17091 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd45a628740 20 client.17091 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd45a628740 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebce5ace0 0x557ebce5d0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd45a628740 4 client.17091 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.181+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557ebcdb2ae0 con 0x557ebce5ace0 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.182+0000 7fd454e00640 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebce5ace0 0x557ebce5d0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.183+0000 7fd454e00640 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebce5ace0 0x557ebce5d0d0 crc :-1 s=READY pgs=223 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1d87937ce9a9f913 server_cookie=ae9187f5eea6ef5d in_seq=0 out_seq=0 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.183+0000 7fd436a00640 10 client.17091 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.183+0000 7fd436a00640 1 -- 172.21.15.28:0/2681549866 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x557ebcdb2ae0 con 0x557ebce5ace0 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.183+0000 7fd436a00640 10 client.17091 handle_command_reply: tid=0 2024-05-20T20:16:06.184 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 client.17091 shutdown 2024-05-20T20:16:06.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce48710 msgr2=0x557ebce56d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce48710 0x557ebce56d50 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7fd444000c00 tx=0x7fd4440095f0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 shutdown_connections 2024-05-20T20:16:06.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebce5ace0 0x557ebce5d0d0 crc :-1 s=READY pgs=223 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce48710 0x557ebce56d50 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce481d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 --2- 172.21.15.28:0/2681549866 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce47c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.184+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 >> 172.21.15.28:0/2681549866 conn(0x557ebce3c1a0 msgr2=0x557ebce41f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.185+0000 7fd436a00640 0 client.17091 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.185+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 shutdown_connections 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.185+0000 7fd45a628740 1 -- 172.21.15.28:0/2681549866 wait complete. 2024-05-20T20:16:06.186 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.186+0000 7fd45a628740 20 client.17091 trim_cache size 0 max 16384 2024-05-20T20:16:06.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.217+0000 7fd45a628740 1 Processor -- start 2024-05-20T20:16:06.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.217+0000 7fd45a628740 1 -- start start 2024-05-20T20:16:06.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.218+0000 7fd45a628740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce3b190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.218+0000 7fd45a628740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce3b6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.218+0000 7fd45a628740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce62780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.218+0000 7fd45a628740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd4501b4ce0 con 0x557ebce3c1a0 2024-05-20T20:16:06.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.218+0000 7fd45a628740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd4501b6820 con 0x557ebce3fc90 2024-05-20T20:16:06.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.218+0000 7fd45a628740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd4501b53b0 con 0x557ebce4ae70 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce62780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce62780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:53660/0 (socket says 172.21.15.28:53660) 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 -- 172.21.15.28:0/2644310332 learned_addr learned my addr 172.21.15.28:0/2644310332 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd454e00640 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce3b190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 -- 172.21.15.28:0/2644310332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 msgr2=0x557ebce3b6d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce3b6d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 -- 172.21.15.28:0/2644310332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 msgr2=0x557ebce3b190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce3b190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.223+0000 7fd455800640 1 -- 172.21.15.28:0/2644310332 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557ebce3a6b0 con 0x557ebce4ae70 2024-05-20T20:16:06.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.224+0000 7fd455800640 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce62780 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7fd448009be0 tx=0x7fd44803f000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3b3507d54057443f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:06.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.224+0000 7fd441600640 1 -- 172.21.15.28:0/2644310332 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd448036040 con 0x557ebce4ae70 2024-05-20T20:16:06.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.225+0000 7fd441600640 1 -- 172.21.15.28:0/2644310332 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd448050070 con 0x557ebce4ae70 2024-05-20T20:16:06.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.225+0000 7fd441600640 1 -- 172.21.15.28:0/2644310332 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd44804b480 con 0x557ebce4ae70 2024-05-20T20:16:06.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.226+0000 7fd45a628740 1 -- 172.21.15.28:0/2644310332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 msgr2=0x557ebce62780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.226+0000 7fd45a628740 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce62780 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7fd448009be0 tx=0x7fd44803f000 comp rx=0 tx=0).stop 2024-05-20T20:16:06.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.226+0000 7fd45a628740 1 -- 172.21.15.28:0/2644310332 shutdown_connections 2024-05-20T20:16:06.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.226+0000 7fd45a628740 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce3fc90 0x557ebce3b6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.226 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.226+0000 7fd45a628740 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce4ae70 0x557ebce62780 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.226+0000 7fd45a628740 1 --2- 172.21.15.28:0/2644310332 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce3b190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.227+0000 7fd45a628740 1 -- 172.21.15.28:0/2644310332 >> 172.21.15.28:0/2644310332 conn(0x557ebce48330 msgr2=0x557ebce3aa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:06.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.227+0000 7fd45a628740 1 -- 172.21.15.28:0/2644310332 shutdown_connections 2024-05-20T20:16:06.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.227+0000 7fd45a628740 1 -- 172.21.15.28:0/2644310332 wait complete. 2024-05-20T20:16:06.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.227+0000 7fd45a628740 1 -- start start 2024-05-20T20:16:06.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.228+0000 7fd45a628740 1 Processor -- start 2024-05-20T20:16:06.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.228+0000 7fd45a628740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce50360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.228+0000 7fd45a628740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce3fc90 0x557ebce508a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd45a628740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce4ae70 0x557ebcebd700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd45a628740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd4501b6150 con 0x557ebce3c1a0 2024-05-20T20:16:06.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd45a628740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd4501b5a80 con 0x557ebce4ae70 2024-05-20T20:16:06.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd45a628740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd4501b2ce0 con 0x557ebce3fc90 2024-05-20T20:16:06.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd45a628740 10 client.0 _ll_register_callbacks cb 0x7fd448034070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:06.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd454e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce50360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd454e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce50360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42150/0 (socket says 172.21.15.28:42150) 2024-05-20T20:16:06.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd454e00640 1 -- 172.21.15.28:0/1244802645 learned_addr learned my addr 172.21.15.28:0/1244802645 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:06.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.229+0000 7fd44fe00640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce3fc90 0x557ebce508a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd455800640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce4ae70 0x557ebcebd700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd454e00640 1 -- 172.21.15.28:0/1244802645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce3fc90 msgr2=0x557ebce508a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:06.230 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd454e00640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce3fc90 0x557ebce508a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd454e00640 1 -- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce4ae70 msgr2=0x557ebcebd700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:06.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd454e00640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce4ae70 0x557ebcebd700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd454e00640 1 -- 172.21.15.28:0/1244802645 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557ebcebdd40 con 0x557ebce3c1a0 2024-05-20T20:16:06.231 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd455800640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce4ae70 0x557ebcebd700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:06.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.230+0000 7fd44fe00640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce3fc90 0x557ebce508a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:06.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.231+0000 7fd454e00640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce50360 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7fd43c0a9090 tx=0x7fd43c0a9160 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3aec5621f8650b91 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:06.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.231+0000 7fd436a00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:16:06.232 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.231+0000 7fd436a00640 1 -- 172.21.15.28:0/1244802645 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd43c0a8ea0 con 0x557ebce3c1a0 2024-05-20T20:16:06.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.231+0000 7fd436a00640 1 -- 172.21.15.28:0/1244802645 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(89..89 src has 1..89) ==== 8542+0+0 (secure 0 0 0) 0x7fd43c0e5cf0 con 0x557ebce3c1a0 2024-05-20T20:16:06.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.231+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd448050070 con 0x557ebce3c1a0 2024-05-20T20:16:06.233 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.234+0000 7fd436a00640 1 -- 172.21.15.28:0/1244802645 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd43c0a9a90 con 0x557ebce3c1a0 2024-05-20T20:16:06.234 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.234+0000 7fd45a628740 10 client.17097 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:06.235 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.234+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557ebcec5120 con 0x557ebce3c1a0 2024-05-20T20:16:06.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.236+0000 7fd436a00640 1 -- 172.21.15.28:0/1244802645 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd448050070 con 0x557ebce3c1a0 2024-05-20T20:16:06.236 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.236+0000 7fd436a00640 1 -- 172.21.15.28:0/1244802645 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd43c0a5620 con 0x557ebce3c1a0 2024-05-20T20:16:06.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.237+0000 7fd45a628740 10 client.17097 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:06.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.237+0000 7fd45a628740 10 client.17097 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:06.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.237+0000 7fd45a628740 20 client.17097 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:06.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.237+0000 7fd45a628740 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebcec57a0 0x557ebcec7b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:06.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.237+0000 7fd45a628740 4 client.17097 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:16:06.237 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.237+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x557ebcdac2c0 con 0x557ebcec57a0 2024-05-20T20:16:06.238 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.238+0000 7fd44fe00640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebcec57a0 0x557ebcec7b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:06.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.239+0000 7fd44fe00640 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebcec57a0 0x557ebcec7b90 crc :-1 s=READY pgs=224 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c0dfd32ec9c72d45 server_cookie=f855e0aabace434c in_seq=0 out_seq=0 2024-05-20T20:16:06.240 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.239+0000 7fd436a00640 10 client.17097 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:06.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.272+0000 7fd436a00640 1 -- 172.21.15.28:0/1244802645 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x557ebcdac2c0 con 0x557ebcec57a0 2024-05-20T20:16:06.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.272+0000 7fd436a00640 10 client.17097 handle_command_reply: tid=0 2024-05-20T20:16:06.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.272+0000 7fd45a628740 1 client.17097 shutdown 2024-05-20T20:16:06.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 msgr2=0x557ebce50360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce50360 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7fd43c0a9090 tx=0x7fd43c0a9160 comp rx=0 tx=0).stop 2024-05-20T20:16:06.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 shutdown_connections 2024-05-20T20:16:06.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557ebcec57a0 0x557ebcec7b90 crc :-1 s=READY pgs=224 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557ebce4ae70 0x557ebcebd700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557ebce3fc90 0x557ebce508a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 --2- 172.21.15.28:0/1244802645 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557ebce3c1a0 0x557ebce50360 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 >> 172.21.15.28:0/1244802645 conn(0x557ebce48330 msgr2=0x557ebce3b310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:06.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.274+0000 7fd436a00640 0 client.17097 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:06.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.275+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 shutdown_connections 2024-05-20T20:16:06.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.275+0000 7fd45a628740 1 -- 172.21.15.28:0/1244802645 wait complete. 2024-05-20T20:16:06.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.276+0000 7fd45a628740 20 client.17097 trim_cache size 0 max 16384 2024-05-20T20:16:06.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:16:06.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:16:06.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:16:06.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4380b29a0 msgr2=0x7fd4380b4e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4380b29a0 0x7fd4380b4e60 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fd448006010 tx=0x7fd448004a40 comp rx=0 tx=0).stop 2024-05-20T20:16:06.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4501092b0 msgr2=0x7fd45019f750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:06.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4501092b0 0x7fd45019f750 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7fd43c00d950 tx=0x7fd43c00de10 comp rx=0 tx=0).stop 2024-05-20T20:16:06.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 shutdown_connections 2024-05-20T20:16:06.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd45010e380 0x7fd45019fc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4380b29a0 0x7fd4380b4e60 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd4501a4010 0x7fd4501a44c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 --2- 172.21.15.28:0/1225169458 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4501092b0 0x7fd45019f750 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:06.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.278+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 >> 172.21.15.28:0/1225169458 conn(0x7fd450076fe0 msgr2=0x7fd4500792b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:06.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.279+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 shutdown_connections 2024-05-20T20:16:06.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:06.279+0000 7fd457800640 1 -- 172.21.15.28:0/1225169458 wait complete. 2024-05-20T20:16:07.533 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:07 smithi028 ceph-mon[31178]: pgmap v325: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 283 MiB/s wr, 1.46k op/s 2024-05-20T20:16:07.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:07 smithi154 ceph-mon[36314]: pgmap v325: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 283 MiB/s wr, 1.46k op/s 2024-05-20T20:16:07.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:07 smithi033 ceph-mon[36179]: pgmap v325: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 283 MiB/s wr, 1.46k op/s 2024-05-20T20:16:09.402 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:09 smithi028 ceph-mon[31178]: pgmap v326: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 290 MiB/s wr, 1.57k op/s 2024-05-20T20:16:09.402 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:09 smithi028 ceph-mon[31178]: osdmap e90: 12 total, 12 up, 12 in 2024-05-20T20:16:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.622+0000 7f3391c00700 1 -- 172.21.15.28:0/1244951033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 msgr2=0x7f338c112520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.622+0000 7f3391c00700 1 --2- 172.21.15.28:0/1244951033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 0x7f338c112520 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f3384009f90 tx=0x7f338400b330 comp rx=0 tx=0).stop 2024-05-20T20:16:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.622+0000 7f3391c00700 1 -- 172.21.15.28:0/1244951033 shutdown_connections 2024-05-20T20:16:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.622+0000 7f3391c00700 1 --2- 172.21.15.28:0/1244951033 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 0x7f338c112520 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.622+0000 7f3391c00700 1 --2- 172.21.15.28:0/1244951033 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c10b650 0x7f338c10ba30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.622+0000 7f3391c00700 1 --2- 172.21.15.28:0/1244951033 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f338c105650 0x7f338c105ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.622+0000 7f3391c00700 1 -- 172.21.15.28:0/1244951033 >> 172.21.15.28:0/1244951033 conn(0x7f338c101120 msgr2=0x7f338c103540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:09.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.623+0000 7f3391c00700 1 -- 172.21.15.28:0/1244951033 shutdown_connections 2024-05-20T20:16:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:09 smithi154 ceph-mon[36314]: pgmap v326: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 290 MiB/s wr, 1.57k op/s 2024-05-20T20:16:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:09 smithi154 ceph-mon[36314]: osdmap e90: 12 total, 12 up, 12 in 2024-05-20T20:16:09.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.623+0000 7f3391c00700 1 -- 172.21.15.28:0/1244951033 wait complete. 2024-05-20T20:16:09.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.626+0000 7f3391c00700 1 Processor -- start 2024-05-20T20:16:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:09 smithi033 ceph-mon[36179]: pgmap v326: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 290 MiB/s wr, 1.57k op/s 2024-05-20T20:16:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:09 smithi033 ceph-mon[36179]: osdmap e90: 12 total, 12 up, 12 in 2024-05-20T20:16:09.627 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.627+0000 7f3391c00700 1 -- start start 2024-05-20T20:16:09.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.628+0000 7f3391c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c105650 0x7f338c112330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.629+0000 7f3391c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 0x7f338c10f430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.629+0000 7f3391c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f338c10b650 0x7f338c10f970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.629 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.630+0000 7f3391c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f338c1158e0 con 0x7f338c105650 2024-05-20T20:16:09.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.630+0000 7f3383400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 0x7f338c10f430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.632+0000 7f3391c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f338c115760 con 0x7f338c106010 2024-05-20T20:16:09.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.632+0000 7f3391c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f338c115a60 con 0x7f338c10b650 2024-05-20T20:16:09.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.632+0000 7f338be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c105650 0x7f338c112330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.632+0000 7f338be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c105650 0x7f338c112330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:38996/0 (socket says 172.21.15.28:38996) 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.632+0000 7f338be00700 1 -- 172.21.15.28:0/4071186190 learned_addr learned my addr 172.21.15.28:0/4071186190 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.632+0000 7f3391200700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f338c10b650 0x7f338c10f970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.633+0000 7f3391200700 1 -- 172.21.15.28:0/4071186190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 msgr2=0x7f338c10f430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.634+0000 7f3391200700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 0x7f338c10f430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.634+0000 7f3391200700 1 -- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c105650 msgr2=0x7f338c112330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.634+0000 7f3391200700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c105650 0x7f338c112330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.634+0000 7f3391200700 1 -- 172.21.15.28:0/4071186190 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f338c110270 con 0x7f338c10b650 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.635+0000 7f338be00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c105650 0x7f338c112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:09.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.635+0000 7f3383400700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 0x7f338c10f430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:09.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.635+0000 7f3391200700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f338c10b650 0x7f338c10f970 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f3384009f90 tx=0x7f338400cdb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:09.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.636+0000 7f3389600700 1 -- 172.21.15.28:0/4071186190 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3384021070 con 0x7f338c10b650 2024-05-20T20:16:09.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.636+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f338c1a1e40 con 0x7f338c10b650 2024-05-20T20:16:09.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.636+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f338c1a23a0 con 0x7f338c10b650 2024-05-20T20:16:09.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.636+0000 7f3389600700 1 -- 172.21.15.28:0/4071186190 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3384007b40 con 0x7f338c10b650 2024-05-20T20:16:09.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.637+0000 7f3389600700 1 -- 172.21.15.28:0/4071186190 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3384021070 con 0x7f338c10b650 2024-05-20T20:16:09.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.642+0000 7f3389600700 1 -- 172.21.15.28:0/4071186190 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3384024070 con 0x7f338c10b650 2024-05-20T20:16:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.645+0000 7f3389600700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33700b2cf0 0x7f33700b51b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.645+0000 7f3389600700 1 -- 172.21.15.28:0/4071186190 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f338400f020 con 0x7f338c10b650 2024-05-20T20:16:09.646 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.647+0000 7f338be00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33700b2cf0 0x7f33700b51b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.657 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.657+0000 7f338be00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33700b2cf0 0x7f33700b51b0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f337c006fd0 tx=0x7f337c008040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:09.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.658+0000 7f339dc61b80 1 Processor -- start 2024-05-20T20:16:09.658 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.659+0000 7f339dc61b80 1 -- start start 2024-05-20T20:16:09.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.660+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0293ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.660+0000 7f338be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0293ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.660+0000 7f338be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0293ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49678/0 (socket says 172.21.15.28:49678) 2024-05-20T20:16:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.661+0000 7f338be00700 1 -- 172.21.15.28:0/3870322174 learned_addr learned my addr 172.21.15.28:0/3870322174 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.661+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd02940e0 0x561fd028dd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.661+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd028e390 0x561fd028e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.661+0000 7f3383400700 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd02940e0 0x561fd028dd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.662+0000 7f3391200700 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd028e390 0x561fd028e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.663+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561fd01ebe40 con 0x561fd028e390 2024-05-20T20:16:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.663+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561fd01db9e0 con 0x561fd02940e0 2024-05-20T20:16:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.663+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561fd01dfa20 con 0x561fd02937c0 2024-05-20T20:16:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.664+0000 7f338be00700 1 -- 172.21.15.28:0/3870322174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd02940e0 msgr2=0x561fd028dd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.664+0000 7f338be00700 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd02940e0 0x561fd028dd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.664+0000 7f338be00700 1 -- 172.21.15.28:0/3870322174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd028e390 msgr2=0x561fd028e7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.664+0000 7f338be00700 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd028e390 0x561fd028e7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.664+0000 7f338be00700 1 -- 172.21.15.28:0/3870322174 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561fd01a4420 con 0x561fd02937c0 2024-05-20T20:16:09.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.665+0000 7f338be00700 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0293ba0 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f337c00e8f0 tx=0x7f337c011000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=796f49168c9368cf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.665+0000 7f3381600700 1 -- 172.21.15.28:0/3870322174 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f337c025070 con 0x561fd02937c0 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.666+0000 7f3381600700 1 -- 172.21.15.28:0/3870322174 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f337c015b70 con 0x561fd02937c0 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.667+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 msgr2=0x561fd0293ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.667+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0293ba0 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7f337c00e8f0 tx=0x7f337c011000 comp rx=0 tx=0).stop 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.668+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 shutdown_connections 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.668+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd028e390 0x561fd028e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.668+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0293ba0 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.668+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3870322174 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd02940e0 0x561fd028dd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.668+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 >> 172.21.15.28:0/3870322174 conn(0x561fd01bbbe0 msgr2=0x561fd0288810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.669+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 shutdown_connections 2024-05-20T20:16:09.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.669+0000 7f339dc61b80 1 -- 172.21.15.28:0/3870322174 wait complete. 2024-05-20T20:16:09.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.670+0000 7f339dc61b80 1 -- start start 2024-05-20T20:16:09.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.670+0000 7f339dc61b80 1 Processor -- start 2024-05-20T20:16:09.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.672+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd0299d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.672+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02937c0 0x561fd02970b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.673+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02940e0 0x561fd02975f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.674+0000 7f3391200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02940e0 0x561fd02975f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.674+0000 7f3391200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02940e0 0x561fd02975f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49688/0 (socket says 172.21.15.28:49688) 2024-05-20T20:16:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.674+0000 7f3391200700 1 -- 172.21.15.28:0/2135917724 learned_addr learned my addr 172.21.15.28:0/2135917724 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.674+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561fd00e1fa0 con 0x561fd02937c0 2024-05-20T20:16:09.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.675+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561fd017c760 con 0x561fd028e390 2024-05-20T20:16:09.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.675+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561fd029aca0 con 0x561fd02940e0 2024-05-20T20:16:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.675+0000 7f339dc61b80 10 client.0 _ll_register_callbacks cb 0x561fd0185c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.675+0000 7f338be00700 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd0299d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.676+0000 7f3391200700 1 -- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 msgr2=0x561fd0299d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.676+0000 7f3391200700 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd0299d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.676+0000 7f3391200700 1 -- 172.21.15.28:0/2135917724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02937c0 msgr2=0x561fd02970b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.676+0000 7f3391200700 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02937c0 0x561fd02970b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.676+0000 7f3391200700 1 -- 172.21.15.28:0/2135917724 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561fd012b8d0 con 0x561fd02940e0 2024-05-20T20:16:09.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.677+0000 7f338be00700 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd0299d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:09.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.677+0000 7f3391200700 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02940e0 0x561fd02975f0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f338401dcf0 tx=0x7f33840aa950 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bc18a4147b18b98e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:09.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.677+0000 7f337a000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:16:09.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.677+0000 7f337a000700 1 -- 172.21.15.28:0/2135917724 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f33840af140 con 0x561fd02940e0 2024-05-20T20:16:09.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.678+0000 7f337a000700 1 -- 172.21.15.28:0/2135917724 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f33840aaff0 con 0x561fd02940e0 2024-05-20T20:16:09.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.678+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f337c015b70 con 0x561fd02940e0 2024-05-20T20:16:09.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.678+0000 7f337a000700 1 -- 172.21.15.28:0/2135917724 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3384053030 con 0x561fd02940e0 2024-05-20T20:16:09.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.681+0000 7f339dc61b80 10 client.27496 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:09.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.681+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561fd01ac060 con 0x561fd02940e0 2024-05-20T20:16:09.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f337a000700 1 -- 172.21.15.28:0/2135917724 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f337c015b70 con 0x561fd02940e0 2024-05-20T20:16:09.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f337a000700 1 -- 172.21.15.28:0/2135917724 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3384011070 con 0x561fd02940e0 2024-05-20T20:16:09.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f339dc61b80 10 client.27496 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:09.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f339dc61b80 10 client.27496 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:09.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f339dc61b80 20 client.27496 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:09.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f339dc61b80 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd02a76e0 0x561fd02a9b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f339dc61b80 4 client.27496 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:09.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.684+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561fd01edd90 con 0x561fd02a76e0 2024-05-20T20:16:09.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.685+0000 7f338be00700 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd02a76e0 0x561fd02a9b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.686+0000 7f338be00700 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd02a76e0 0x561fd02a9b60 crc :-1 s=READY pgs=225 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=31127a1923d2cbdc server_cookie=6750d963bc9e737b in_seq=0 out_seq=0 2024-05-20T20:16:09.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.686+0000 7f337a000700 10 client.27496 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:09.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.687+0000 7f337a000700 1 -- 172.21.15.28:0/2135917724 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561fd01edd90 con 0x561fd02a76e0 2024-05-20T20:16:09.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.687+0000 7f337a000700 10 client.27496 handle_command_reply: tid=0 2024-05-20T20:16:09.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.687+0000 7f339dc61b80 1 client.27496 shutdown 2024-05-20T20:16:09.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.687+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02940e0 msgr2=0x561fd02975f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.687+0000 7f339dc61b80 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02940e0 0x561fd02975f0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7f338401dcf0 tx=0x7f33840aa950 comp rx=0 tx=0).stop 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 shutdown_connections 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd02a76e0 0x561fd02a9b60 crc :-1 s=READY pgs=225 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f337a000700 0 client.27496 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02940e0 0x561fd02975f0 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd0299d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 --2- 172.21.15.28:0/2135917724 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02937c0 0x561fd02970b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 >> 172.21.15.28:0/2135917724 conn(0x561fd01bbbe0 msgr2=0x561fd0287fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:09.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 shutdown_connections 2024-05-20T20:16:09.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 1 -- 172.21.15.28:0/2135917724 wait complete. 2024-05-20T20:16:09.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.688+0000 7f339dc61b80 20 client.27496 trim_cache size 0 max 16384 2024-05-20T20:16:09.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.737+0000 7f339dc61b80 1 Processor -- start 2024-05-20T20:16:09.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.738+0000 7f339dc61b80 1 -- start start 2024-05-20T20:16:09.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.739+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd028dd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.739+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0296010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.739+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 0x561fd0288740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.740+0000 7f338be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd028dd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.740+0000 7f338be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd028dd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36324/0 (socket says 172.21.15.28:36324) 2024-05-20T20:16:09.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.741+0000 7f3391200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 0x561fd0288740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.741+0000 7f3383400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0296010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.741+0000 7f3383400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0296010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49692/0 (socket says 172.21.15.28:49692) 2024-05-20T20:16:09.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.741+0000 7f339dc61b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f336800d740 con 0x561fd02940e0 2024-05-20T20:16:09.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.741+0000 7f339dc61b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f336800f280 con 0x561fd028e390 2024-05-20T20:16:09.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.741+0000 7f339dc61b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f336800de10 con 0x561fd02937c0 2024-05-20T20:16:09.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.742+0000 7f338be00700 1 -- 172.21.15.28:0/3251057054 learned_addr learned my addr 172.21.15.28:0/3251057054 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:09.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.742+0000 7f338be00700 1 -- 172.21.15.28:0/3251057054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 msgr2=0x561fd0296010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.742+0000 7f338be00700 1 --2- 172.21.15.28:0/3251057054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0296010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.742+0000 7f338be00700 1 -- 172.21.15.28:0/3251057054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 msgr2=0x561fd0288740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.742+0000 7f338be00700 1 --2- 172.21.15.28:0/3251057054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 0x561fd0288740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.742+0000 7f338be00700 1 -- 172.21.15.28:0/3251057054 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561fd0297920 con 0x561fd028e390 2024-05-20T20:16:09.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.742+0000 7f338be00700 1 --2- 172.21.15.28:0/3251057054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd028dd70 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f337c00ef90 tx=0x7f337c020970 comp rx=0 tx=0).ready entity=mon.2 client_cookie=11c433c30934133e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:09.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.743+0000 7f3380c00700 1 -- 172.21.15.28:0/3251057054 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f337c00f040 con 0x561fd028e390 2024-05-20T20:16:09.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.743+0000 7f3380c00700 1 -- 172.21.15.28:0/3251057054 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f337c020be0 con 0x561fd028e390 2024-05-20T20:16:09.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.743+0000 7f3380c00700 1 -- 172.21.15.28:0/3251057054 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f337c02c450 con 0x561fd028e390 2024-05-20T20:16:09.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.744+0000 7f339dc61b80 1 -- 172.21.15.28:0/3251057054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 msgr2=0x561fd028dd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.744 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.744+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3251057054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd028dd70 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f337c00ef90 tx=0x7f337c020970 comp rx=0 tx=0).stop 2024-05-20T20:16:09.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.745+0000 7f339dc61b80 1 -- 172.21.15.28:0/3251057054 shutdown_connections 2024-05-20T20:16:09.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.745+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3251057054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 0x561fd0288740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.745+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3251057054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd028dd70 unknown :-1 s=CLOSED pgs=538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.746+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3251057054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd0296010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.746+0000 7f339dc61b80 1 -- 172.21.15.28:0/3251057054 >> 172.21.15.28:0/3251057054 conn(0x561fd01bbbe0 msgr2=0x561fd0292270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:09.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.746+0000 7f339dc61b80 1 -- 172.21.15.28:0/3251057054 shutdown_connections 2024-05-20T20:16:09.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.747+0000 7f339dc61b80 1 -- 172.21.15.28:0/3251057054 wait complete. 2024-05-20T20:16:09.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.747+0000 7f339dc61b80 1 -- start start 2024-05-20T20:16:09.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.747+0000 7f339dc61b80 1 Processor -- start 2024-05-20T20:16:09.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.749+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd029cfe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.749+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd029d520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.749+0000 7f338be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd029cfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.749+0000 7f339dc61b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 0x561fd029da60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.749+0000 7f338be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd029cfe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36338/0 (socket says 172.21.15.28:36338) 2024-05-20T20:16:09.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.749+0000 7f338be00700 1 -- 172.21.15.28:0/3162960564 learned_addr learned my addr 172.21.15.28:0/3162960564 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:09.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.749+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f336800ebb0 con 0x561fd02940e0 2024-05-20T20:16:09.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.750+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f336800e4e0 con 0x561fd028e390 2024-05-20T20:16:09.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.750+0000 7f3383400700 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd029d520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.750+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f336800b700 con 0x561fd02937c0 2024-05-20T20:16:09.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.750+0000 7f339dc61b80 10 client.0 _ll_register_callbacks cb 0x561fd0185c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:09.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.752+0000 7f338be00700 1 -- 172.21.15.28:0/3162960564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 msgr2=0x561fd029d520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.752+0000 7f338be00700 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd029d520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.752+0000 7f338be00700 1 -- 172.21.15.28:0/3162960564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 msgr2=0x561fd029da60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:09.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.752+0000 7f338be00700 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 0x561fd029da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.752+0000 7f338be00700 1 -- 172.21.15.28:0/3162960564 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561fd0312350 con 0x561fd028e390 2024-05-20T20:16:09.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.752+0000 7f338be00700 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd029cfe0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f337c023040 tx=0x7f337c006d70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d1d5ca988e7c014f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:09.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.752+0000 7f337a000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:09.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.753+0000 7f337a000700 1 -- 172.21.15.28:0/3162960564 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f337c03d020 con 0x561fd028e390 2024-05-20T20:16:09.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.753+0000 7f337a000700 1 -- 172.21.15.28:0/3162960564 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f337c0158c0 con 0x561fd028e390 2024-05-20T20:16:09.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.754+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f337c020be0 con 0x561fd028e390 2024-05-20T20:16:09.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.755+0000 7f337a000700 1 -- 172.21.15.28:0/3162960564 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f337c015ac0 con 0x561fd028e390 2024-05-20T20:16:09.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.755+0000 7f339dc61b80 10 client.27575 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:09.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.755+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561fd0313600 con 0x561fd028e390 2024-05-20T20:16:09.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.755+0000 7f337a000700 1 -- 172.21.15.28:0/3162960564 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f337c020be0 con 0x561fd028e390 2024-05-20T20:16:09.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.757+0000 7f337a000700 1 -- 172.21.15.28:0/3162960564 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f337c035030 con 0x561fd028e390 2024-05-20T20:16:09.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.757+0000 7f339dc61b80 10 client.27575 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:09.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.757+0000 7f339dc61b80 10 client.27575 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:09.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.757+0000 7f339dc61b80 20 client.27575 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:09.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.757+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd0313960 0x561fd0315d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:09.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.758+0000 7f339dc61b80 4 client.27575 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:16:09.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.758+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x561fd01e7570 con 0x561fd0313960 2024-05-20T20:16:09.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.758+0000 7f3383400700 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd0313960 0x561fd0315d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:09.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.759+0000 7f3383400700 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd0313960 0x561fd0315d50 crc :-1 s=READY pgs=226 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f0c108fa5f689100 server_cookie=eca848ae05d9d12f in_seq=0 out_seq=0 2024-05-20T20:16:09.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.759+0000 7f337a000700 10 client.27575 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:09.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.759+0000 7f337a000700 1 -- 172.21.15.28:0/3162960564 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x561fd01e7570 con 0x561fd0313960 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.759+0000 7f337a000700 10 client.27575 handle_command_reply: tid=0 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.759+0000 7f339dc61b80 1 client.27575 shutdown 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 msgr2=0x561fd029cfe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd029cfe0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f337c023040 tx=0x7f337c006d70 comp rx=0 tx=0).stop 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 shutdown_connections 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x561fd0313960 0x561fd0315d50 crc :-1 s=READY pgs=226 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561fd02940e0 0x561fd029da60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561fd028e390 0x561fd029cfe0 unknown :-1 s=CLOSED pgs=539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.760 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 --2- 172.21.15.28:0/3162960564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561fd02937c0 0x561fd029d520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.760+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 >> 172.21.15.28:0/3162960564 conn(0x561fd01bbbe0 msgr2=0x561fd0294bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:09.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.761+0000 7f337a000700 0 client.27575 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:09.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.761+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 shutdown_connections 2024-05-20T20:16:09.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.761+0000 7f339dc61b80 1 -- 172.21.15.28:0/3162960564 wait complete. 2024-05-20T20:16:09.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.761+0000 7f339dc61b80 20 client.27575 trim_cache size 0 max 16384 2024-05-20T20:16:09.762 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:16:09.762 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:16:09.762 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:16:09.762 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:16:09.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.762+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33700b2cf0 msgr2=0x7f33700b51b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.762+0000 7f3391c00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33700b2cf0 0x7f33700b51b0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f337c006fd0 tx=0x7f337c008040 comp rx=0 tx=0).stop 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f338c10b650 msgr2=0x7f338c10f970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f338c10b650 0x7f338c10f970 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f3384009f90 tx=0x7f338400cdb0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 shutdown_connections 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f338c10b650 0x7f338c10f970 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f338c105650 0x7f338c112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f33700b2cf0 0x7f33700b51b0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 --2- 172.21.15.28:0/4071186190 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f338c106010 0x7f338c10f430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:09.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 >> 172.21.15.28:0/4071186190 conn(0x7f338c101120 msgr2=0x7f338c101640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:09.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.763+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 shutdown_connections 2024-05-20T20:16:09.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:09.764+0000 7f3391c00700 1 -- 172.21.15.28:0/4071186190 wait complete. 2024-05-20T20:16:11.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:11 smithi028 ceph-mon[31178]: pgmap v328: 97 pgs: 16 active+clean+snaptrim, 24 active+clean+snaptrim_wait, 57 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 308 MiB/s wr, 2.11k op/s 2024-05-20T20:16:11.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:11 smithi154 ceph-mon[36314]: pgmap v328: 97 pgs: 16 active+clean+snaptrim, 24 active+clean+snaptrim_wait, 57 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 308 MiB/s wr, 2.11k op/s 2024-05-20T20:16:11.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:11 smithi033 ceph-mon[36179]: pgmap v328: 97 pgs: 16 active+clean+snaptrim, 24 active+clean+snaptrim_wait, 57 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 308 MiB/s wr, 2.11k op/s 2024-05-20T20:16:12.205 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:16:12.206 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:16:12.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.338+0000 7f074a800700 1 -- 172.21.15.28:0/3053820248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f073c09be40 msgr2=0x7f073c09c220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:12.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.338+0000 7f074a800700 1 --2- 172.21.15.28:0/3053820248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f073c09be40 0x7f073c09c220 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f0744069fd0 tx=0x7f0744052350 comp rx=0 tx=0).stop 2024-05-20T20:16:12.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.339+0000 7f074a800700 1 -- 172.21.15.28:0/3053820248 shutdown_connections 2024-05-20T20:16:12.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.339+0000 7f074a800700 1 --2- 172.21.15.28:0/3053820248 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f073c005190 0x7f073c0a2bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.339+0000 7f074a800700 1 --2- 172.21.15.28:0/3053820248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f073c09be40 0x7f073c09c220 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.339+0000 7f074a800700 1 --2- 172.21.15.28:0/3053820248 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 0x7f073c004c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.339+0000 7f074a800700 1 -- 172.21.15.28:0/3053820248 >> 172.21.15.28:0/3053820248 conn(0x7f073c00a7a0 msgr2=0x7f073c00abb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.340+0000 7f074a800700 1 -- 172.21.15.28:0/3053820248 shutdown_connections 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.340+0000 7f074a800700 1 -- 172.21.15.28:0/3053820248 wait complete. 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.342+0000 7f074a800700 1 Processor -- start 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.343+0000 7f074a800700 1 -- start start 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.348+0000 7f074a800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 0x7f073c0a53e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.348+0000 7f074a800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f073c005190 0x7f073c0a5920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.348+0000 7f074a800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f073c09be40 0x7f073c0a9cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.349+0000 7f074a800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f073c0a3c00 con 0x7f073c0047d0 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.349+0000 7f074a800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f073c0a3a80 con 0x7f073c005190 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.349+0000 7f074a800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f073c0a3d80 con 0x7f073c09be40 2024-05-20T20:16:12.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.349+0000 7f0749400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 0x7f073c0a53e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.349+0000 7f0749400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 0x7f073c0a53e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:39048/0 (socket says 172.21.15.28:39048) 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.350+0000 7f0749400700 1 -- 172.21.15.28:0/2847716047 learned_addr learned my addr 172.21.15.28:0/2847716047 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.350+0000 7f0749400700 1 -- 172.21.15.28:0/2847716047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f073c09be40 msgr2=0x7f073c0a9cb0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.350+0000 7f0749400700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f073c09be40 0x7f073c0a9cb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.350+0000 7f0749400700 1 -- 172.21.15.28:0/2847716047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f073c005190 msgr2=0x7f073c0a5920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.350+0000 7f0748a00700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f073c005190 0x7f073c0a5920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.351+0000 7f0749400700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f073c005190 0x7f073c0a5920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.351+0000 7f0749400700 1 -- 172.21.15.28:0/2847716047 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f073c0aa3e0 con 0x7f073c0047d0 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.351+0000 7f0748a00700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f073c005190 0x7f073c0a5920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.353+0000 7f0749400700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 0x7f073c0a53e0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f0744069030 tx=0x7f0744075fd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.354+0000 7f0742000700 1 -- 172.21.15.28:0/2847716047 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f074408f070 con 0x7f073c0047d0 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.354+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f073c0aa670 con 0x7f073c0047d0 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.354+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f073c147d20 con 0x7f073c0047d0 2024-05-20T20:16:12.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.354+0000 7f0742000700 1 -- 172.21.15.28:0/2847716047 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f074406bea0 con 0x7f073c0047d0 2024-05-20T20:16:12.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.354+0000 7f0742000700 1 -- 172.21.15.28:0/2847716047 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0744099060 con 0x7f073c0047d0 2024-05-20T20:16:12.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.359+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0724005330 con 0x7f073c0047d0 2024-05-20T20:16:12.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.373+0000 7f0742000700 1 -- 172.21.15.28:0/2847716047 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0744092070 con 0x7f073c0047d0 2024-05-20T20:16:12.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.378+0000 7f0742000700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f072c0b2b30 0x7f072c0b4ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:12.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.378+0000 7f0742000700 1 -- 172.21.15.28:0/2847716047 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f0744151680 con 0x7f073c0047d0 2024-05-20T20:16:12.268 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:16:12.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.381+0000 7f0748a00700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f072c0b2b30 0x7f072c0b4ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:12.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.383+0000 7f0748a00700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f072c0b2b30 0x7f072c0b4ff0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f073c0a69d0 tx=0x7f0738009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:12.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.385+0000 7f0742000700 1 -- 172.21.15.28:0/2847716047 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f0744119050 con 0x7f073c0047d0 2024-05-20T20:16:12.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.923+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f0724005620 con 0x7f073c0047d0 2024-05-20T20:16:12.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.927+0000 7f0742000700 1 -- 172.21.15.28:0/2847716047 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f074407fe20 con 0x7f073c0047d0 2024-05-20T20:16:12.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f072c0b2b30 msgr2=0x7f072c0b4ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:12.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f072c0b2b30 0x7f072c0b4ff0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f073c0a69d0 tx=0x7f0738009260 comp rx=0 tx=0).stop 2024-05-20T20:16:12.269 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 msgr2=0x7f073c0a53e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 0x7f073c0a53e0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f0744069030 tx=0x7f0744075fd0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 shutdown_connections 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f073c09be40 0x7f073c0a9cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f073c0047d0 0x7f073c0a53e0 secure :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f0744069030 tx=0x7f0744075fd0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f072c0b2b30 0x7f072c0b4ff0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 --2- 172.21.15.28:0/2847716047 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f073c005190 0x7f073c0a5920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.933+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 >> 172.21.15.28:0/2847716047 conn(0x7f073c00a7a0 msgr2=0x7f073c093c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.934+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 shutdown_connections 2024-05-20T20:16:12.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:11.935+0000 7f074a800700 1 -- 172.21.15.28:0/2847716047 wait complete. 2024-05-20T20:16:12.271 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:16:12.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:12 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2847716047' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:16:12.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:12 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2847716047' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:16:12.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:12 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2847716047' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:16:13.068 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id 7ee8534d --release --await' 2024-05-20T20:16:13.069 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id 7ee8534d --release --await 2024-05-20T20:16:13.180 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:13.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:13 smithi028 ceph-mon[31178]: pgmap v329: 97 pgs: 18 active+clean+snaptrim, 30 active+clean+snaptrim_wait, 49 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 310 MiB/s wr, 2.20k op/s 2024-05-20T20:16:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:13 smithi154 ceph-mon[36314]: pgmap v329: 97 pgs: 18 active+clean+snaptrim, 30 active+clean+snaptrim_wait, 49 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 310 MiB/s wr, 2.20k op/s 2024-05-20T20:16:13.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:13 smithi033 ceph-mon[36179]: pgmap v329: 97 pgs: 18 active+clean+snaptrim, 30 active+clean+snaptrim_wait, 49 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 310 MiB/s wr, 2.20k op/s 2024-05-20T20:16:14.738 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:15.455 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:15 smithi028 ceph-mon[31178]: pgmap v330: 97 pgs: 4 active+clean+snaptrim, 7 active+clean+snaptrim_wait, 86 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 234 MiB/s wr, 2.10k op/s 2024-05-20T20:16:15.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:15 smithi154 ceph-mon[36314]: pgmap v330: 97 pgs: 4 active+clean+snaptrim, 7 active+clean+snaptrim_wait, 86 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 234 MiB/s wr, 2.10k op/s 2024-05-20T20:16:15.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:15 smithi033 ceph-mon[36179]: pgmap v330: 97 pgs: 4 active+clean+snaptrim, 7 active+clean+snaptrim_wait, 86 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 234 MiB/s wr, 2.10k op/s 2024-05-20T20:16:15.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 -- 172.21.15.28:0/3798613740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 msgr2=0x7f15a8101fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:15.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 --2- 172.21.15.28:0/3798613740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8101fb0 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f15940099c0 tx=0x7f1594009cd0 comp rx=0 tx=0).stop 2024-05-20T20:16:15.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 -- 172.21.15.28:0/3798613740 shutdown_connections 2024-05-20T20:16:15.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 --2- 172.21.15.28:0/3798613740 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15a81024f0 0x7f15a81103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:15.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 --2- 172.21.15.28:0/3798613740 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15a8109520 0x7f15a8109900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:15.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 --2- 172.21.15.28:0/3798613740 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8101fb0 unknown :-1 s=CLOSED pgs=540 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:15.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 -- 172.21.15.28:0/3798613740 >> 172.21.15.28:0/3798613740 conn(0x7f15a8078ab0 msgr2=0x7f15a80ffe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:15.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 -- 172.21.15.28:0/3798613740 shutdown_connections 2024-05-20T20:16:15.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.713+0000 7f15acc00700 1 -- 172.21.15.28:0/3798613740 wait complete. 2024-05-20T20:16:15.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.714+0000 7f15acc00700 1 Processor -- start 2024-05-20T20:16:15.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.715+0000 7f15acc00700 1 -- start start 2024-05-20T20:16:15.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.715+0000 7f15acc00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8102d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:15.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.715+0000 7f15acc00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15a81024f0 0x7f15a8103250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:15.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.715+0000 7f15acc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15a8109520 0x7f15a8103790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:15.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.715+0000 7f15acc00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f15a8113770 con 0x7f15a81024f0 2024-05-20T20:16:15.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.715+0000 7f15acc00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f15a81135f0 con 0x7f15a8101b30 2024-05-20T20:16:15.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.715+0000 7f15acc00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f15a81138f0 con 0x7f15a8109520 2024-05-20T20:16:15.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8102d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:15.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15a81024f0 0x7f15a8103250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:15.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a6800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15a8109520 0x7f15a8103790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:15.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8102d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:36382/0 (socket says 172.21.15.28:36382) 2024-05-20T20:16:15.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 -- 172.21.15.28:0/1803706544 learned_addr learned my addr 172.21.15.28:0/1803706544 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:15.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 -- 172.21.15.28:0/1803706544 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15a8109520 msgr2=0x7f15a8103790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:15.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15a8109520 0x7f15a8103790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:15.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 -- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15a81024f0 msgr2=0x7f15a8103250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:15.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15a81024f0 0x7f15a8103250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:15.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 -- 172.21.15.28:0/1803706544 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f15a81a8150 con 0x7f15a8101b30 2024-05-20T20:16:15.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.716+0000 7f15a5e00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8102d10 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f159800e9a0 tx=0x7f159800ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:15.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.717+0000 7f159ea00700 1 -- 172.21.15.28:0/1803706544 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f159800ca10 con 0x7f15a8101b30 2024-05-20T20:16:15.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.717+0000 7f159ea00700 1 -- 172.21.15.28:0/1803706544 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f159800cbb0 con 0x7f15a8101b30 2024-05-20T20:16:15.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.717+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f15a81a8440 con 0x7f15a8101b30 2024-05-20T20:16:15.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.717+0000 7f159ea00700 1 -- 172.21.15.28:0/1803706544 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1598018680 con 0x7f15a8101b30 2024-05-20T20:16:15.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.717+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f15a81a8950 con 0x7f15a8101b30 2024-05-20T20:16:15.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.718+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f15a8104e50 con 0x7f15a8101b30 2024-05-20T20:16:15.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.720+0000 7f159ea00700 1 -- 172.21.15.28:0/1803706544 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1598010430 con 0x7f15a8101b30 2024-05-20T20:16:15.721 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.722+0000 7f159ea00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f158c0b29f0 0x7f158c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:15.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.722+0000 7f159ea00700 1 -- 172.21.15.28:0/1803706544 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f15980e2ad0 con 0x7f15a8101b30 2024-05-20T20:16:15.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.722+0000 7f15a5400700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f158c0b29f0 0x7f158c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:15.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.723+0000 7f15a5400700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f158c0b29f0 0x7f158c0b4eb0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f15a81047a0 tx=0x7f1594005fb0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:15.724 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:15.724+0000 7f159ea00700 1 -- 172.21.15.28:0/1803706544 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f15980aa410 con 0x7f15a8101b30 2024-05-20T20:16:16.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.021+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f15a802e560 con 0x7f15a8101b30 2024-05-20T20:16:16.022 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.023+0000 7f159ea00700 1 -- 172.21.15.28:0/1803706544 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f15980a9d10 con 0x7f15a8101b30 2024-05-20T20:16:16.023 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:16:16.023 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:16:16.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f158c0b29f0 msgr2=0x7f158c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:16.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f158c0b29f0 0x7f158c0b4eb0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f15a81047a0 tx=0x7f1594005fb0 comp rx=0 tx=0).stop 2024-05-20T20:16:16.026 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 msgr2=0x7f15a8102d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8102d10 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f159800e9a0 tx=0x7f159800ecb0 comp rx=0 tx=0).stop 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 shutdown_connections 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f15a8109520 0x7f15a8103790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f158c0b29f0 0x7f158c0b4eb0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f15a8101b30 0x7f15a8102d10 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 --2- 172.21.15.28:0/1803706544 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f15a81024f0 0x7f15a8103250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.026+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 >> 172.21.15.28:0/1803706544 conn(0x7f15a8078ab0 msgr2=0x7f15a81002f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.027+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 shutdown_connections 2024-05-20T20:16:16.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:16.027+0000 7f15acc00700 1 -- 172.21.15.28:0/1803706544 wait complete. 2024-05-20T20:16:16.028 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:16:16.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:16 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1803706544' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:16:16.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:16 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1803706544' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:16:16.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:16 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1803706544' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:16:17.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:17 smithi028 ceph-mon[31178]: pgmap v331: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 212 MiB/s wr, 1.96k op/s 2024-05-20T20:16:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:17 smithi154 ceph-mon[36314]: pgmap v331: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 212 MiB/s wr, 1.96k op/s 2024-05-20T20:16:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:17 smithi033 ceph-mon[36179]: pgmap v331: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 212 MiB/s wr, 1.96k op/s 2024-05-20T20:16:19.198 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 -- 172.21.15.28:0/1583116556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 msgr2=0x7f0488115cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 --2- 172.21.15.28:0/1583116556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 0x7f0488115cc0 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7f0480009cd0 tx=0x7f048002a8f0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 -- 172.21.15.28:0/1583116556 shutdown_connections 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 --2- 172.21.15.28:0/1583116556 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 0x7f0488115cc0 unknown :-1 s=CLOSED pgs=542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 --2- 172.21.15.28:0/1583116556 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f04880778a0 0x7f0488113390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 --2- 172.21.15.28:0/1583116556 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0488076f80 0x7f0488077360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 -- 172.21.15.28:0/1583116556 >> 172.21.15.28:0/1583116556 conn(0x7f0488100740 msgr2=0x7f0488102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:19.446 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 -- 172.21.15.28:0/1583116556 shutdown_connections 2024-05-20T20:16:19.447 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.446+0000 7f0490200640 1 -- 172.21.15.28:0/1583116556 wait complete. 2024-05-20T20:16:19.447 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.447+0000 7f0490200640 1 Processor -- start 2024-05-20T20:16:19.447 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.447+0000 7f0490200640 1 -- start start 2024-05-20T20:16:19.447 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f0490200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0488076f80 0x7f04881a55b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f0490200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f04880778a0 0x7f04881a5af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f0490200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 0x7f048819f670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f0490200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0488119310 con 0x7f04880778a0 2024-05-20T20:16:19.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f0490200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0488119190 con 0x7f04881138d0 2024-05-20T20:16:19.448 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f0490200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0488119490 con 0x7f0488076f80 2024-05-20T20:16:19.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f048d800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0488076f80 0x7f04881a55b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f048ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f04880778a0 0x7f04881a5af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f048ce00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f04880778a0 0x7f04881a5af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50908/0 (socket says 172.21.15.28:50908) 2024-05-20T20:16:19.449 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f048ce00640 1 -- 172.21.15.28:0/1268399961 learned_addr learned my addr 172.21.15.28:0/1268399961 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f048d800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0488076f80 0x7f04881a55b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52444/0 (socket says 172.21.15.28:52444) 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.448+0000 7f048e200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 0x7f048819f670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f048d800640 1 -- 172.21.15.28:0/1268399961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 msgr2=0x7f048819f670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f048d800640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 0x7f048819f670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f048d800640 1 -- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f04880778a0 msgr2=0x7f04881a5af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f048d800640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f04880778a0 0x7f04881a5af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f048d800640 1 -- 172.21.15.28:0/1268399961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f048819ff30 con 0x7f0488076f80 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f048d800640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0488076f80 0x7f04881a55b0 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f047c00e9f0 tx=0x7f047c00eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f0486000640 1 -- 172.21.15.28:0/1268399961 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c00ccc0 con 0x7f0488076f80 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04881a0220 con 0x7f0488076f80 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f0486000640 1 -- 172.21.15.28:0/1268399961 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f047c0108e0 con 0x7f0488076f80 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.449+0000 7f0486000640 1 -- 172.21.15.28:0/1268399961 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c018880 con 0x7f0488076f80 2024-05-20T20:16:19.450 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.450+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f04881ac530 con 0x7f0488076f80 2024-05-20T20:16:19.454 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.454+0000 7f0492fc7740 1 Processor -- start 2024-05-20T20:16:19.454 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.454+0000 7f0492fc7740 1 -- start start 2024-05-20T20:16:19.455 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb54070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb61260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f0486000640 1 -- 172.21.15.28:0/1268399961 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f047c010430 con 0x7f0488076f80 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f0492fc7740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55b5ceac4b90 con 0x55b5ceb53c90 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f0492fc7740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55b5ceab4730 con 0x55b5ceb5ee70 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f0492fc7740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55b5ceab8770 con 0x55b5ceb545b0 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.455+0000 7f048d800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52458/0 (socket says 172.21.15.28:52458) 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 -- 172.21.15.28:0/1913299428 learned_addr learned my addr 172.21.15.28:0/1913299428 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048ce00640 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb61260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 -- 172.21.15.28:0/1913299428 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb5ee70 msgr2=0x55b5ceb61260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb61260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.456 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 -- 172.21.15.28:0/1913299428 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 msgr2=0x55b5ceb54070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.457 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb54070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.457 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 -- 172.21.15.28:0/1913299428 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b5cea4d8b0 con 0x55b5ceb545b0 2024-05-20T20:16:19.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.456+0000 7f048d800640 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e930 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f047c0dff60 tx=0x7f047c0ea2a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=115a09c71707fbf9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:19.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.457+0000 7f047aa00640 1 -- 172.21.15.28:0/1913299428 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c0f1070 con 0x55b5ceb545b0 2024-05-20T20:16:19.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.457+0000 7f0486000640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04700b2a90 0x7f04700b4f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.457+0000 7f0486000640 1 -- 172.21.15.28:0/1268399961 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f047c0e2970 con 0x7f0488076f80 2024-05-20T20:16:19.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.458+0000 7f048e200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04700b2a90 0x7f04700b4f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.458+0000 7f047aa00640 1 -- 172.21.15.28:0/1913299428 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f047c010720 con 0x55b5ceb545b0 2024-05-20T20:16:19.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.458+0000 7f047aa00640 1 -- 172.21.15.28:0/1913299428 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c0a93d0 con 0x55b5ceb545b0 2024-05-20T20:16:19.459 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.458+0000 7f0492fc7740 1 -- 172.21.15.28:0/1913299428 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb545b0 msgr2=0x55b5ceb5e930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.459 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e930 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7f047c0dff60 tx=0x7f047c0ea2a0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.459 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f048e200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04700b2a90 0x7f04700b4f50 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f0480005e30 tx=0x7f0480009360 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 -- 172.21.15.28:0/1913299428 shutdown_connections 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb61260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e930 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 --2- 172.21.15.28:0/1913299428 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb54070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 -- 172.21.15.28:0/1913299428 >> 172.21.15.28:0/1913299428 conn(0x55b5ceb501a0 msgr2=0x55b5ceb50580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 -- 172.21.15.28:0/1913299428 shutdown_connections 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 -- 172.21.15.28:0/1913299428 wait complete. 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.459+0000 7f0492fc7740 1 -- start start 2024-05-20T20:16:19.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 1 Processor -- start 2024-05-20T20:16:19.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb5e880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5bb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb5c180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55b5ce79a790 con 0x55b5ceb545b0 2024-05-20T20:16:19.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55b5cea5cbc0 con 0x55b5ceb53c90 2024-05-20T20:16:19.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55b5ceb61aa0 con 0x55b5ceb5ee70 2024-05-20T20:16:19.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f0492fc7740 10 client.0 _ll_register_callbacks cb 0x55b5cea3a400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f048d800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb5e880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f048d800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb5e880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60142/0 (socket says 172.21.15.28:60142) 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f048d800640 1 -- 172.21.15.28:0/3087768465 learned_addr learned my addr 172.21.15.28:0/3087768465 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.460+0000 7f048ce00640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5bb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048e200640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb5c180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048d800640 1 -- 172.21.15.28:0/3087768465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 msgr2=0x55b5ceb5c180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048d800640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb5c180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048d800640 1 -- 172.21.15.28:0/3087768465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 msgr2=0x55b5ceb5bb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048d800640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5bb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048d800640 1 -- 172.21.15.28:0/3087768465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b5ce7c9210 con 0x55b5ceb53c90 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048ce00640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5bb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:19.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048e200640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb5c180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f048d800640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb5e880 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f047c0a6ed0 tx=0x7f047c00cef0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4b2526de3ba229d9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f046f400640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f046f400640 1 -- 172.21.15.28:0/3087768465 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c0f1030 con 0x55b5ceb53c90 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f046f400640 1 -- 172.21.15.28:0/3087768465 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f047c0a5ef0 con 0x55b5ceb53c90 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.461+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55b5ceac6ae0 con 0x55b5ceb53c90 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f046f400640 1 -- 172.21.15.28:0/3087768465 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f047c0cae60 con 0x55b5ceb53c90 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 10 client.27596 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55b5ce895620 con 0x55b5ceb53c90 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f046f400640 1 -- 172.21.15.28:0/3087768465 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55b5ceac6ae0 con 0x55b5ceb53c90 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f046f400640 1 -- 172.21.15.28:0/3087768465 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f047c0eacb0 con 0x55b5ceb53c90 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 10 client.27596 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 10 client.27596 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:19.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 20 client.27596 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:19.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5ceb6e700 0x55b5ceb70af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 4 client.27596 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:19.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.462+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55b5ceac02c0 con 0x55b5ceb6e700 2024-05-20T20:16:19.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.463+0000 7f048ce00640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5ceb6e700 0x55b5ceb70af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.463+0000 7f048ce00640 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5ceb6e700 0x55b5ceb70af0 crc :-1 s=READY pgs=227 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a8a4efdda66e68b4 server_cookie=26a088d9d06f864a in_seq=0 out_seq=0 2024-05-20T20:16:19.466 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.463+0000 7f046f400640 10 client.27596 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:19.467 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f046f400640 1 -- 172.21.15.28:0/3087768465 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55b5ceac02c0 con 0x55b5ceb6e700 2024-05-20T20:16:19.467 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f046f400640 10 client.27596 handle_command_reply: tid=0 2024-05-20T20:16:19.467 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 client.27596 shutdown 2024-05-20T20:16:19.467 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 msgr2=0x55b5ceb5e880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.467 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb5e880 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f047c0a6ed0 tx=0x7f047c00cef0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 shutdown_connections 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5ceb6e700 0x55b5ceb70af0 crc :-1 s=READY pgs=227 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb5c180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5bb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3087768465 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb5e880 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 >> 172.21.15.28:0/3087768465 conn(0x55b5ceb501a0 msgr2=0x55b5ceb55f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.464+0000 7f046f400640 0 client.27596 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.465+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 shutdown_connections 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.465+0000 7f0492fc7740 1 -- 172.21.15.28:0/3087768465 wait complete. 2024-05-20T20:16:19.468 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.465+0000 7f0492fc7740 20 client.27596 trim_cache size 0 max 16384 2024-05-20T20:16:19.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 Processor -- start 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 -- start start 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb4f190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb73730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f04881b4eb0 con 0x55b5ceb53c90 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f04881b69f0 con 0x55b5ceb545b0 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.484+0000 7f0492fc7740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f04881b5580 con 0x55b5ceb5ee70 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb4f190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048ce00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb4f190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50932/0 (socket says 172.21.15.28:50932) 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 -- 172.21.15.28:0/3325012577 learned_addr learned my addr 172.21.15.28:0/3325012577 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:19.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048e200640 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb73730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 -- 172.21.15.28:0/3325012577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 msgr2=0x55b5ceb73730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb73730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 -- 172.21.15.28:0/3325012577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb545b0 msgr2=0x55b5ceb5e3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048d800640 1 -- 172.21.15.28:0/3325012577 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55b5ceb61590 con 0x55b5ceb53c90 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.485+0000 7f048ce00640 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.486+0000 7f048d800640 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb4f190 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f047c0a5ce0 tx=0x7f047c0a98b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=51d545acca4f058c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.486+0000 7f047a000640 1 -- 172.21.15.28:0/3325012577 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c0f1070 con 0x55b5ceb53c90 2024-05-20T20:16:19.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.486+0000 7f047a000640 1 -- 172.21.15.28:0/3325012577 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f047c091030 con 0x55b5ceb53c90 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.486+0000 7f047a000640 1 -- 172.21.15.28:0/3325012577 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c0e8300 con 0x55b5ceb53c90 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 -- 172.21.15.28:0/3325012577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 msgr2=0x55b5ceb4f190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb4f190 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f047c0a5ce0 tx=0x7f047c0a98b0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 -- 172.21.15.28:0/3325012577 shutdown_connections 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb545b0 0x55b5ceb5e3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb73730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 --2- 172.21.15.28:0/3325012577 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb53c90 0x55b5ceb4f190 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 -- 172.21.15.28:0/3325012577 >> 172.21.15.28:0/3325012577 conn(0x55b5ceb501a0 msgr2=0x55b5ceb4ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 -- 172.21.15.28:0/3325012577 shutdown_connections 2024-05-20T20:16:19.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.487+0000 7f0492fc7740 1 -- 172.21.15.28:0/3325012577 wait complete. 2024-05-20T20:16:19.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.488+0000 7f0492fc7740 1 -- start start 2024-05-20T20:16:19.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.488+0000 7f0492fc7740 1 Processor -- start 2024-05-20T20:16:19.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb70490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb709d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f0492fc7740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb70f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f0492fc7740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f04881b6320 con 0x55b5ceb545b0 2024-05-20T20:16:19.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f0492fc7740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f04881b5c50 con 0x55b5ceb53c90 2024-05-20T20:16:19.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f0492fc7740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f04881b2eb0 con 0x55b5ceb5ee70 2024-05-20T20:16:19.490 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f0492fc7740 10 client.0 _ll_register_callbacks cb 0x7f047c0e89c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:19.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f048d800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb70490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f048d800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb70490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60154/0 (socket says 172.21.15.28:60154) 2024-05-20T20:16:19.491 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f048d800640 1 -- 172.21.15.28:0/2700214746 learned_addr learned my addr 172.21.15.28:0/2700214746 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f048e200640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb70f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.489+0000 7f048ce00640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb709d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f048d800640 1 -- 172.21.15.28:0/2700214746 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 msgr2=0x55b5ceb70f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f048d800640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb70f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f048d800640 1 -- 172.21.15.28:0/2700214746 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 msgr2=0x55b5ceb709d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f048d800640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb709d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f048d800640 1 -- 172.21.15.28:0/2700214746 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55b5cebd6f30 con 0x55b5ceb53c90 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f048e200640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb70f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f048d800640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb70490 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f047c010d40 tx=0x7f047c0e0100 comp rx=0 tx=0).ready entity=mon.2 client_cookie=23fa7fd395d2d5d7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.490+0000 7f046f400640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f046f400640 1 -- 172.21.15.28:0/2700214746 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f047c0f1030 con 0x55b5ceb53c90 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f046f400640 1 -- 172.21.15.28:0/2700214746 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f047c0e02a0 con 0x55b5ceb53c90 2024-05-20T20:16:19.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f047c091030 con 0x55b5ceb53c90 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f046f400640 1 -- 172.21.15.28:0/2700214746 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f047c0a5b20 con 0x55b5ceb53c90 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f0492fc7740 10 client.27602 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55b5cebd6f30 con 0x55b5ceb53c90 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f046f400640 1 -- 172.21.15.28:0/2700214746 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f047c091030 con 0x55b5ceb53c90 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.491+0000 7f046f400640 1 -- 172.21.15.28:0/2700214746 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f047c0b2580 con 0x55b5ceb53c90 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.492+0000 7f0492fc7740 10 client.27602 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.492+0000 7f0492fc7740 10 client.27602 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.492+0000 7f0492fc7740 20 client.27602 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.492+0000 7f0492fc7740 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5cebd85d0 0x55b5cebda9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.492+0000 7f0492fc7740 4 client.27602 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.492+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55b5cea3a4b0 con 0x55b5cebd85d0 2024-05-20T20:16:19.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.492+0000 7f048ce00640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5cebd85d0 0x55b5cebda9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:19.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.493+0000 7f048ce00640 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5cebd85d0 0x55b5cebda9f0 crc :-1 s=READY pgs=228 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d7ea3b1745f58942 server_cookie=7b682383f64896fc in_seq=0 out_seq=0 2024-05-20T20:16:19.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.493+0000 7f046f400640 10 client.27602 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:19.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f046f400640 1 -- 172.21.15.28:0/2700214746 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55b5cea3a4b0 con 0x55b5cebd85d0 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f046f400640 10 client.27602 handle_command_reply: tid=0 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 client.27602 shutdown 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 msgr2=0x55b5ceb70490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb70490 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f047c010d40 tx=0x7f047c0e0100 comp rx=0 tx=0).stop 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 shutdown_connections 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55b5cebd85d0 0x55b5cebda9f0 crc :-1 s=READY pgs=228 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55b5ceb5ee70 0x55b5ceb70f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55b5ceb545b0 0x55b5ceb709d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 --2- 172.21.15.28:0/2700214746 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55b5ceb53c90 0x55b5ceb70490 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.507 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 >> 172.21.15.28:0/2700214746 conn(0x55b5ceb501a0 msgr2=0x55b5ceb4f310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.506+0000 7f046f400640 0 client.27602 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.508+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 shutdown_connections 2024-05-20T20:16:19.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.508+0000 7f0492fc7740 1 -- 172.21.15.28:0/2700214746 wait complete. 2024-05-20T20:16:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.510+0000 7f0492fc7740 20 client.27602 trim_cache size 0 max 16384 2024-05-20T20:16:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:16:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:16:19.509 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:16:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.510+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04700b2a90 msgr2=0x7f04700b4f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.510 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.510+0000 7f0490200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04700b2a90 0x7f04700b4f50 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f0480005e30 tx=0x7f0480009360 comp rx=0 tx=0).stop 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.510+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0488076f80 msgr2=0x7f04881a55b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.510+0000 7f0490200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0488076f80 0x7f04881a55b0 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f047c00e9f0 tx=0x7f047c00eeb0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 shutdown_connections 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f04881138d0 0x7f048819f670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f04700b2a90 0x7f04700b4f50 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f04880778a0 0x7f04881a5af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 --2- 172.21.15.28:0/1268399961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0488076f80 0x7f04881a55b0 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 >> 172.21.15.28:0/1268399961 conn(0x7f0488100740 msgr2=0x7f0488102b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 shutdown_connections 2024-05-20T20:16:19.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:19.511+0000 7f0490200640 1 -- 172.21.15.28:0/1268399961 wait complete. 2024-05-20T20:16:19.548 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:19 smithi033 ceph-mon[36179]: pgmap v332: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 180 MiB/s wr, 1.72k op/s 2024-05-20T20:16:19.548 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:19 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:16:19.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:19 smithi028 ceph-mon[31178]: pgmap v332: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 180 MiB/s wr, 1.72k op/s 2024-05-20T20:16:19.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:19 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:16:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:19 smithi154 ceph-mon[36314]: pgmap v332: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 180 MiB/s wr, 1.72k op/s 2024-05-20T20:16:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:19 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:16:20.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.210+0000 7f14e0800700 1 -- 172.21.15.28:0/1038100961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8105750 msgr2=0x7f14d8105bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.210+0000 7f14e0800700 1 --2- 172.21.15.28:0/1038100961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8105750 0x7f14d8105bd0 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7f14c8009990 tx=0x7f14c8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.211+0000 7f14e0800700 1 -- 172.21.15.28:0/1038100961 shutdown_connections 2024-05-20T20:16:20.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.211+0000 7f14e0800700 1 --2- 172.21.15.28:0/1038100961 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f14d8106110 0x7f14d8112620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.211+0000 7f14e0800700 1 --2- 172.21.15.28:0/1038100961 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d810b750 0x7f14d810bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.211+0000 7f14e0800700 1 --2- 172.21.15.28:0/1038100961 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8105750 0x7f14d8105bd0 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.211+0000 7f14e0800700 1 -- 172.21.15.28:0/1038100961 >> 172.21.15.28:0/1038100961 conn(0x7f14d8101240 msgr2=0x7f14d8103660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:20.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.211+0000 7f14e0800700 1 -- 172.21.15.28:0/1038100961 shutdown_connections 2024-05-20T20:16:20.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.211+0000 7f14e0800700 1 -- 172.21.15.28:0/1038100961 wait complete. 2024-05-20T20:16:20.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.212+0000 7f14e0800700 1 Processor -- start 2024-05-20T20:16:20.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.212+0000 7f14e0800700 1 -- start start 2024-05-20T20:16:20.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14e0800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d8105750 0x7f14d819be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14e0800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8106110 0x7f14d819c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14e0800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f14d810b750 0x7f14d81a06f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14e0800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f14d8115930 con 0x7f14d8106110 2024-05-20T20:16:20.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14e0800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f14d81157b0 con 0x7f14d8105750 2024-05-20T20:16:20.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14e0800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f14d8115ab0 con 0x7f14d810b750 2024-05-20T20:16:20.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14de800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f14d810b750 0x7f14d81a06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14dde00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d8105750 0x7f14d819be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14dde00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d8105750 0x7f14d819be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60166/0 (socket says 172.21.15.28:60166) 2024-05-20T20:16:20.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14de800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f14d810b750 0x7f14d81a06f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52504/0 (socket says 172.21.15.28:52504) 2024-05-20T20:16:20.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.213+0000 7f14dde00700 1 -- 172.21.15.28:0/3283296581 learned_addr learned my addr 172.21.15.28:0/3283296581 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14dd400700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8106110 0x7f14d819c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14dde00700 1 -- 172.21.15.28:0/3283296581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f14d810b750 msgr2=0x7f14d81a06f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14dde00700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f14d810b750 0x7f14d81a06f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14dde00700 1 -- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8106110 msgr2=0x7f14d819c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14dde00700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8106110 0x7f14d819c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14dde00700 1 -- 172.21.15.28:0/3283296581 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f14d81a0e30 con 0x7f14d8105750 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14dde00700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d8105750 0x7f14d819be20 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f14cc00e9a0 tx=0x7f14cc00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14d2a00700 1 -- 172.21.15.28:0/3283296581 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc00cbd0 con 0x7f14d8105750 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14d2a00700 1 -- 172.21.15.28:0/3283296581 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f14cc00cd70 con 0x7f14d8105750 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.214+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f14d81a1120 con 0x7f14d8105750 2024-05-20T20:16:20.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.215+0000 7f14d2a00700 1 -- 172.21.15.28:0/3283296581 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc018680 con 0x7f14d8105750 2024-05-20T20:16:20.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.215+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f14d8077980 con 0x7f14d8105750 2024-05-20T20:16:20.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.218+0000 7f14d2a00700 1 -- 172.21.15.28:0/3283296581 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f14cc010430 con 0x7f14d8105750 2024-05-20T20:16:20.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14d2a00700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f14c00b29f0 0x7f14c00b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14eb211b80 1 Processor -- start 2024-05-20T20:16:20.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14eb211b80 1 -- start start 2024-05-20T20:16:20.220 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14d2a00700 1 -- 172.21.15.28:0/3283296581 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f14cc0e28a0 con 0x7f14d8105750 2024-05-20T20:16:20.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14dd400700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f14c00b29f0 0x7f14c00b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a3fd7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3fdd00 0x56472a3f7960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.220+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3f7f80 0x56472a3f83c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.221+0000 7f14eb211b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56472a355f40 con 0x56472a3fd3e0 2024-05-20T20:16:20.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.221+0000 7f14eb211b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56472a345ae0 con 0x56472a3f7f80 2024-05-20T20:16:20.221 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.221+0000 7f14eb211b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56472a349b20 con 0x56472a3fdd00 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.221+0000 7f14dde00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a3fd7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.221+0000 7f14dde00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a3fd7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50980/0 (socket says 172.21.15.28:50980) 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.221+0000 7f14dde00700 1 -- 172.21.15.28:0/84362105 learned_addr learned my addr 172.21.15.28:0/84362105 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.221+0000 7f14dd400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3fdd00 0x56472a3f7960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14dde00700 1 -- 172.21.15.28:0/84362105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3fdd00 msgr2=0x56472a3f7960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14dde00700 1 --2- 172.21.15.28:0/84362105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3fdd00 0x56472a3f7960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14dde00700 1 -- 172.21.15.28:0/84362105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3f7f80 msgr2=0x56472a3f83c0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:20.222 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14dde00700 1 --2- 172.21.15.28:0/84362105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3f7f80 0x56472a3f83c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14dde00700 1 -- 172.21.15.28:0/84362105 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56472a30e520 con 0x56472a3fd3e0 2024-05-20T20:16:20.223 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14dd400700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f14c00b29f0 0x7f14c00b4eb0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f14c8006010 tx=0x7f14c800b720 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:20.223 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14dde00700 1 --2- 172.21.15.28:0/84362105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a3fd7c0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f14cc0a5b60 tx=0x7f14cc0a52f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bde874c858f72b72 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14c7400700 1 -- 172.21.15.28:0/84362105 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc0aa1e0 con 0x56472a3fd3e0 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14c7400700 1 -- 172.21.15.28:0/84362105 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f14cc0a9ae0 con 0x56472a3fd3e0 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.222+0000 7f14c7400700 1 -- 172.21.15.28:0/84362105 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc010720 con 0x56472a3fd3e0 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.223+0000 7f14eb211b80 1 -- 172.21.15.28:0/84362105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 msgr2=0x56472a3fd7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.223+0000 7f14eb211b80 1 --2- 172.21.15.28:0/84362105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a3fd7c0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f14cc0a5b60 tx=0x7f14cc0a52f0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.224+0000 7f14eb211b80 1 -- 172.21.15.28:0/84362105 shutdown_connections 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.224+0000 7f14eb211b80 1 --2- 172.21.15.28:0/84362105 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3f7f80 0x56472a3f83c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.224+0000 7f14eb211b80 1 --2- 172.21.15.28:0/84362105 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a3fd7c0 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.224+0000 7f14eb211b80 1 --2- 172.21.15.28:0/84362105 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3fdd00 0x56472a3f7960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.224 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.224+0000 7f14eb211b80 1 -- 172.21.15.28:0/84362105 >> 172.21.15.28:0/84362105 conn(0x56472a325ce0 msgr2=0x56472a3f2420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:20.226 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.226+0000 7f14eb211b80 1 -- 172.21.15.28:0/84362105 shutdown_connections 2024-05-20T20:16:20.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.226+0000 7f14eb211b80 1 -- 172.21.15.28:0/84362105 wait complete. 2024-05-20T20:16:20.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.227+0000 7f14eb211b80 1 -- start start 2024-05-20T20:16:20.227 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.227+0000 7f14eb211b80 1 Processor -- start 2024-05-20T20:16:20.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.228+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a403aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.228 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.228+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fd3e0 0x56472a400e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.228+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fdd00 0x56472a401380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.228+0000 7f14eb211b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56472a24c0a0 con 0x56472a3fdd00 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.228+0000 7f14eb211b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56472a2e6860 con 0x56472a3fd3e0 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.228+0000 7f14eb211b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56472a4047e0 con 0x56472a3f7f80 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.228+0000 7f14eb211b80 10 client.0 _ll_register_callbacks cb 0x56472a2efd70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a403aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14de800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fdd00 0x56472a401380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a403aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52530/0 (socket says 172.21.15.28:52530) 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 -- 172.21.15.28:0/3809573505 learned_addr learned my addr 172.21.15.28:0/3809573505 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dd400700 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fd3e0 0x56472a400e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 -- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fd3e0 msgr2=0x56472a400e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fd3e0 0x56472a400e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.229 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 -- 172.21.15.28:0/3809573505 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fdd00 msgr2=0x56472a401380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:20.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fdd00 0x56472a401380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.230 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 -- 172.21.15.28:0/3809573505 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56472a2959d0 con 0x56472a3f7f80 2024-05-20T20:16:20.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.229+0000 7f14dde00700 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a403aa0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f14cc0a6d50 tx=0x7f14cc0b2840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=79992954866e784 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:20.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.230+0000 7f14bbe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:16:20.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.230+0000 7f14bbe00700 1 -- 172.21.15.28:0/3809573505 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc06e060 con 0x56472a3f7f80 2024-05-20T20:16:20.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.230+0000 7f14bbe00700 1 -- 172.21.15.28:0/3809573505 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f14cc010930 con 0x56472a3f7f80 2024-05-20T20:16:20.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.230+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f14cc0a9ae0 con 0x56472a3f7f80 2024-05-20T20:16:20.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14bbe00700 1 -- 172.21.15.28:0/3809573505 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f14cc0c4070 con 0x56472a3f7f80 2024-05-20T20:16:20.231 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 10 client.27538 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56472a316160 con 0x56472a3f7f80 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14bbe00700 1 -- 172.21.15.28:0/3809573505 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f14cc0a9ae0 con 0x56472a3f7f80 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14bbe00700 1 -- 172.21.15.28:0/3809573505 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f14cc0a5450 con 0x56472a3f7f80 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 10 client.27538 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 10 client.27538 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 20 client.27538 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a411240 0x56472a413630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 4 client.27538 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:20.232 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.231+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56472a357e90 con 0x56472a411240 2024-05-20T20:16:20.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.232+0000 7f14dd400700 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a411240 0x56472a413630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.232+0000 7f14dd400700 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a411240 0x56472a413630 crc :-1 s=READY pgs=229 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=12b1e2ecee8b3456 server_cookie=c5892b29cac0119b in_seq=0 out_seq=0 2024-05-20T20:16:20.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.232+0000 7f14bbe00700 10 client.27538 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:20.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.233+0000 7f14bbe00700 1 -- 172.21.15.28:0/3809573505 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56472a357e90 con 0x56472a411240 2024-05-20T20:16:20.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.233+0000 7f14bbe00700 10 client.27538 handle_command_reply: tid=0 2024-05-20T20:16:20.233 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.233+0000 7f14eb211b80 1 client.27538 shutdown 2024-05-20T20:16:20.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 msgr2=0x56472a403aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a403aa0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f14cc0a6d50 tx=0x7f14cc0b2840 comp rx=0 tx=0).stop 2024-05-20T20:16:20.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 shutdown_connections 2024-05-20T20:16:20.234 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a411240 0x56472a413630 crc :-1 s=READY pgs=229 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fdd00 0x56472a401380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a403aa0 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3809573505 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fd3e0 0x56472a400e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 >> 172.21.15.28:0/3809573505 conn(0x56472a325ce0 msgr2=0x56472a3f6950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14bbe00700 0 client.27538 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 shutdown_connections 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.234+0000 7f14eb211b80 1 -- 172.21.15.28:0/3809573505 wait complete. 2024-05-20T20:16:20.235 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.235+0000 7f14eb211b80 20 client.27538 trim_cache size 0 max 16384 2024-05-20T20:16:20.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 Processor -- start 2024-05-20T20:16:20.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 -- start start 2024-05-20T20:16:20.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a3fb5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a416660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 0x56472a416ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f14d81b45a0 con 0x56472a3fd3e0 2024-05-20T20:16:20.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f14d81b60e0 con 0x56472a3fdd00 2024-05-20T20:16:20.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.266+0000 7f14eb211b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f14d81b4c70 con 0x56472a3f7f80 2024-05-20T20:16:20.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.267+0000 7f14dde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a3fb5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.267+0000 7f14dde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a3fb5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52540/0 (socket says 172.21.15.28:52540) 2024-05-20T20:16:20.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.267+0000 7f14dde00700 1 -- 172.21.15.28:0/3540697463 learned_addr learned my addr 172.21.15.28:0/3540697463 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:20.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.267+0000 7f14dde00700 1 -- 172.21.15.28:0/3540697463 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 msgr2=0x56472a416ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:20.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.267+0000 7f14dde00700 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 0x56472a416ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.267+0000 7f14dde00700 1 -- 172.21.15.28:0/3540697463 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 msgr2=0x56472a416660 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:20.268 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.268+0000 7f14dd400700 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a416660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.268+0000 7f14dde00700 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a416660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.268+0000 7f14dde00700 1 -- 172.21.15.28:0/3540697463 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56472a3ff3b0 con 0x56472a3f7f80 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.268+0000 7f14dd400700 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a416660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.268+0000 7f14dde00700 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a3fb5a0 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7f14cc0a5b60 tx=0x7f14cc0a52f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c1f8d406e66ff4d5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.268+0000 7f14c6a00700 1 -- 172.21.15.28:0/3540697463 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc06e070 con 0x56472a3f7f80 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14c6a00700 1 -- 172.21.15.28:0/3540697463 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f14cc0aa1e0 con 0x56472a3f7f80 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14c6a00700 1 -- 172.21.15.28:0/3540697463 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc06f250 con 0x56472a3f7f80 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14eb211b80 1 -- 172.21.15.28:0/3540697463 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 msgr2=0x56472a3fb5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.269 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a3fb5a0 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7f14cc0a5b60 tx=0x7f14cc0a52f0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14eb211b80 1 -- 172.21.15.28:0/3540697463 shutdown_connections 2024-05-20T20:16:20.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 0x56472a416ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a3fb5a0 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14eb211b80 1 --2- 172.21.15.28:0/3540697463 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a416660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.269+0000 7f14eb211b80 1 -- 172.21.15.28:0/3540697463 >> 172.21.15.28:0/3540697463 conn(0x56472a325ce0 msgr2=0x56472a3ff730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.270+0000 7f14eb211b80 1 -- 172.21.15.28:0/3540697463 shutdown_connections 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.270+0000 7f14eb211b80 1 -- 172.21.15.28:0/3540697463 wait complete. 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.270+0000 7f14eb211b80 1 -- start start 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 1 Processor -- start 2024-05-20T20:16:20.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a416c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a415030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 0x56472a415570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f14d81b5a10 con 0x56472a3fd3e0 2024-05-20T20:16:20.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f14d81b5340 con 0x56472a3fdd00 2024-05-20T20:16:20.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f14d81b2580 con 0x56472a3f7f80 2024-05-20T20:16:20.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.271+0000 7f14eb211b80 10 client.0 _ll_register_callbacks cb 0x56472a3f2a80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:20.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a416c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a416c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:52546/0 (socket says 172.21.15.28:52546) 2024-05-20T20:16:20.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 -- 172.21.15.28:0/2224824163 learned_addr learned my addr 172.21.15.28:0/2224824163 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:20.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dd400700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a415030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14de800700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 0x56472a415570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 -- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 msgr2=0x56472a415570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:20.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 0x56472a415570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 -- 172.21.15.28:0/2224824163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 msgr2=0x56472a415030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a415030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dde00700 1 -- 172.21.15.28:0/2224824163 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56472a415c70 con 0x56472a3f7f80 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.272+0000 7f14dd400700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a415030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14dde00700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a416c30 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f14cc00f010 tx=0x7f14cc0a98b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d3b5458cf500ab68 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14bbe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14bbe00700 1 -- 172.21.15.28:0/2224824163 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f14cc06e070 con 0x56472a3f7f80 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f14cc0aa1e0 con 0x56472a3f7f80 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14bbe00700 1 -- 172.21.15.28:0/2224824163 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(90..90 src has 1..90) ==== 8586+0+0 (secure 0 0 0) 0x7f14cc0a5450 con 0x56472a3f7f80 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14bbe00700 1 -- 172.21.15.28:0/2224824163 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f14cc0ade90 con 0x56472a3f7f80 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14eb211b80 10 client.27544 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:20.274 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.273+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56472a415c70 con 0x56472a3f7f80 2024-05-20T20:16:20.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.274+0000 7f14bbe00700 1 -- 172.21.15.28:0/2224824163 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f14cc0aa1e0 con 0x56472a3f7f80 2024-05-20T20:16:20.275 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.274+0000 7f14bbe00700 1 -- 172.21.15.28:0/2224824163 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f14cc0af0b0 con 0x56472a3f7f80 2024-05-20T20:16:20.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.276+0000 7f14eb211b80 10 client.27544 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:20.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.277+0000 7f14eb211b80 10 client.27544 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:16:20.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.277+0000 7f14eb211b80 20 client.27544 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:20.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.277+0000 7f14eb211b80 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a47cb80 0x56472a47ef70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:20.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.277+0000 7f14eb211b80 4 client.27544 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "7ee8534d", "release": true, "await": true}] 2024-05-20T20:16:20.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.277+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "7ee8534d", "release": true, "await": true}) -- 0x56472a351670 con 0x56472a47cb80 2024-05-20T20:16:20.277 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.277+0000 7f14dd400700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a47cb80 0x56472a47ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:20.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.278+0000 7f14dd400700 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a47cb80 0x56472a47ef70 crc :-1 s=READY pgs=230 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f6f84687efdfcac7 server_cookie=bdc8ada38cdfed45 in_seq=0 out_seq=0 2024-05-20T20:16:20.278 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.279+0000 7f14bbe00700 10 client.27544 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:20.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.281+0000 7f14bbe00700 1 -- 172.21.15.28:0/2224824163 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x56472a351670 con 0x56472a47cb80 2024-05-20T20:16:20.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.281+0000 7f14bbe00700 10 client.27544 handle_command_reply: tid=0 2024-05-20T20:16:20.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.281+0000 7f14eb211b80 1 client.27544 shutdown 2024-05-20T20:16:20.281 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 msgr2=0x56472a416c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a416c30 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f14cc00f010 tx=0x7f14cc0a98b0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 shutdown_connections 2024-05-20T20:16:20.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56472a47cb80 0x56472a47ef70 crc :-1 s=READY pgs=230 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56472a3fdd00 0x56472a415570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56472a3f7f80 0x56472a416c30 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.282 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 --2- 172.21.15.28:0/2224824163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56472a3fd3e0 0x56472a415030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14bbe00700 0 client.27544 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:20.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 >> 172.21.15.28:0/2224824163 conn(0x56472a325ce0 msgr2=0x56472a3ebd80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:20.283 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.282+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 shutdown_connections 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 28, 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "7ee8534d": { 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 28, 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:16:20.284 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 152.6, 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:16:20.285 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:16:20.286 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:16:20.286 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:16:20.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.283+0000 7f14eb211b80 1 -- 172.21.15.28:0/2224824163 wait complete. 2024-05-20T20:16:20.286 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.283+0000 7f14eb211b80 20 client.27544 trim_cache size 0 max 16384 2024-05-20T20:16:20.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.286+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f14c00b29f0 msgr2=0x7f14c00b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.286+0000 7f14e0800700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f14c00b29f0 0x7f14c00b4eb0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f14c8006010 tx=0x7f14c800b720 comp rx=0 tx=0).stop 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.286+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d8105750 msgr2=0x7f14d819be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.286+0000 7f14e0800700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d8105750 0x7f14d819be20 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f14cc00e9a0 tx=0x7f14cc00ecb0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.287+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 shutdown_connections 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.287+0000 7f14e0800700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f14d810b750 0x7f14d81a06f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.287+0000 7f14e0800700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f14c00b29f0 0x7f14c00b4eb0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.287+0000 7f14e0800700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f14d8105750 0x7f14d819be20 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.287+0000 7f14e0800700 1 --2- 172.21.15.28:0/3283296581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f14d8106110 0x7f14d819c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.287+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 >> 172.21.15.28:0/3283296581 conn(0x7f14d8101240 msgr2=0x7f14d81019e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.288+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 shutdown_connections 2024-05-20T20:16:20.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:20.289+0000 7f14e0800700 1 -- 172.21.15.28:0/3283296581 wait complete. 2024-05-20T20:16:21.469 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: pgmap v333: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 151 MiB/s wr, 1.45k op/s 2024-05-20T20:16:21.469 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.469 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.469 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: osdmap e91: 12 total, 12 up, 12 in 2024-05-20T20:16:21.470 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.470 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.470 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.470 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.470 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.470 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:21 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: pgmap v333: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 151 MiB/s wr, 1.45k op/s 2024-05-20T20:16:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: osdmap e91: 12 total, 12 up, 12 in 2024-05-20T20:16:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:21 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: pgmap v333: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 151 MiB/s wr, 1.45k op/s 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: osdmap e91: 12 total, 12 up, 12 in 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:21 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:22 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi033 to 177.2M 2024-05-20T20:16:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:22 smithi154 ceph-mon[36314]: Unable to set osd_memory_target on smithi033 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-05-20T20:16:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:22 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:22 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:22 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:22 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:22 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi033 to 177.2M 2024-05-20T20:16:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:22 smithi033 ceph-mon[36179]: Unable to set osd_memory_target on smithi033 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-05-20T20:16:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:22 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:22 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:22 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:22 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.640 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:22 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi033 to 177.2M 2024-05-20T20:16:22.640 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:22 smithi028 ceph-mon[31178]: Unable to set osd_memory_target on smithi033 to 185897062: error parsing value: Value '185897062' is below minimum 939524096 2024-05-20T20:16:22.640 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:22 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.640 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:22 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.640 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:22 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:22.640 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:22 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:23.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:23 smithi154 ceph-mon[36314]: pgmap v335: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.8 KiB/s rd, 64 MiB/s wr, 738 op/s 2024-05-20T20:16:23.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:23 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi154 to 176.2M 2024-05-20T20:16:23.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:23 smithi154 ceph-mon[36314]: Unable to set osd_memory_target on smithi154 to 184788172: error parsing value: Value '184788172' is below minimum 939524096 2024-05-20T20:16:23.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:23 smithi033 ceph-mon[36179]: pgmap v335: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.8 KiB/s rd, 64 MiB/s wr, 738 op/s 2024-05-20T20:16:23.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:23 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi154 to 176.2M 2024-05-20T20:16:23.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:23 smithi033 ceph-mon[36179]: Unable to set osd_memory_target on smithi154 to 184788172: error parsing value: Value '184788172' is below minimum 939524096 2024-05-20T20:16:23.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:23 smithi028 ceph-mon[31178]: pgmap v335: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.8 KiB/s rd, 64 MiB/s wr, 738 op/s 2024-05-20T20:16:23.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:23 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi154 to 176.2M 2024-05-20T20:16:23.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:23 smithi028 ceph-mon[31178]: Unable to set osd_memory_target on smithi154 to 184788172: error parsing value: Value '184788172' is below minimum 939524096 2024-05-20T20:16:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:25 smithi154 ceph-mon[36314]: pgmap v336: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 44 MiB/s wr, 582 op/s 2024-05-20T20:16:25.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:25 smithi033 ceph-mon[36179]: pgmap v336: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 44 MiB/s wr, 582 op/s 2024-05-20T20:16:25.776 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:25 smithi028 ceph-mon[31178]: pgmap v336: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 44 MiB/s wr, 582 op/s 2024-05-20T20:16:27.560 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:27 smithi028 ceph-mon[31178]: pgmap v337: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 45 MiB/s wr, 593 op/s 2024-05-20T20:16:27.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:27 smithi154 ceph-mon[36314]: pgmap v337: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 45 MiB/s wr, 593 op/s 2024-05-20T20:16:27.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:27 smithi033 ceph-mon[36179]: pgmap v337: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 45 MiB/s wr, 593 op/s 2024-05-20T20:16:28.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:28 smithi154 ceph-mon[36314]: pgmap v338: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 87 MiB/s wr, 983 op/s 2024-05-20T20:16:28.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:28 smithi033 ceph-mon[36179]: pgmap v338: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 87 MiB/s wr, 983 op/s 2024-05-20T20:16:28.706 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:28 smithi028 ceph-mon[31178]: pgmap v338: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 87 MiB/s wr, 983 op/s 2024-05-20T20:16:28.858 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set '7ee8534d', total seconds elapsed: 56.7 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.916+0000 7ff881c00700 1 -- 172.21.15.28:0/63387983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c1095b0 msgr2=0x7ff87c109990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.916+0000 7ff881c00700 1 --2- 172.21.15.28:0/63387983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c1095b0 0x7ff87c109990 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7ff86c009990 tx=0x7ff86c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.916+0000 7ff881c00700 1 -- 172.21.15.28:0/63387983 shutdown_connections 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.916+0000 7ff881c00700 1 --2- 172.21.15.28:0/63387983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff87c06bbe0 0x7ff87c1103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.916+0000 7ff881c00700 1 --2- 172.21.15.28:0/63387983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c1095b0 0x7ff87c109990 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.916+0000 7ff881c00700 1 --2- 172.21.15.28:0/63387983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff87c06b220 0x7ff87c06b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.916+0000 7ff881c00700 1 -- 172.21.15.28:0/63387983 >> 172.21.15.28:0/63387983 conn(0x7ff87c0793f0 msgr2=0x7ff87c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.917+0000 7ff881c00700 1 -- 172.21.15.28:0/63387983 shutdown_connections 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.917+0000 7ff881c00700 1 -- 172.21.15.28:0/63387983 wait complete. 2024-05-20T20:16:28.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 Processor -- start 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 -- start start 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c06b220 0x7ff87c1120c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff87c06bbe0 0x7ff87c112600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff87c1095b0 0x7ff87c112b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff87c103b30 con 0x7ff87c1095b0 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff87c1039b0 con 0x7ff87c06bbe0 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff881c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff87c103cb0 con 0x7ff87c06b220 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c06b220 0x7ff87c1120c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c06b220 0x7ff87c1120c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33816/0 (socket says 172.21.15.28:33816) 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff87aa00700 1 -- 172.21.15.28:0/1588136704 learned_addr learned my addr 172.21.15.28:0/1588136704 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:28.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.918+0000 7ff87a000700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff87c06bbe0 0x7ff87c112600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff87aa00700 1 -- 172.21.15.28:0/1588136704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff87c06bbe0 msgr2=0x7ff87c112600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff87c06bbe0 0x7ff87c112600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff87aa00700 1 -- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff87c1095b0 msgr2=0x7ff87c112b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff87c1095b0 0x7ff87c112b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff87aa00700 1 -- 172.21.15.28:0/1588136704 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff87c073b40 con 0x7ff87c06b220 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c06b220 0x7ff87c1120c0 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7ff86c000c00 tx=0x7ff86c0057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff873400700 1 -- 172.21.15.28:0/1588136704 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff86c027070 con 0x7ff87c06b220 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff873400700 1 -- 172.21.15.28:0/1588136704 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff86c022470 con 0x7ff87c06b220 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.919+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff87c073dd0 con 0x7ff87c06b220 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.920+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff87c074300 con 0x7ff87c06b220 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.920+0000 7ff873400700 1 -- 172.21.15.28:0/1588136704 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff86c017650 con 0x7ff87c06b220 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.925+0000 7ff873400700 1 -- 172.21.15.28:0/1588136704 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ff86c02a070 con 0x7ff87c06b220 2024-05-20T20:16:28.938 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.926+0000 7ff873400700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8600d4b00 0x7ff8600d6fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.926+0000 7ff873400700 1 -- 172.21.15.28:0/1588136704 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7ff86c0e31f0 con 0x7ff87c06b220 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.927+0000 7ff87a000700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8600d4b00 0x7ff8600d6fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.927+0000 7ff88c57ab80 1 Processor -- start 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.927+0000 7ff88c57ab80 1 -- start start 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.927+0000 7ff87a000700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8600d4b00 0x7ff8600d6fc0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7ff87c113870 tx=0x7ff868009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a30eb0 0x563ab2a31290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a317d0 0x563ab2a2b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2bce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff88c57ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563ab2989e40 con 0x563ab2a317d0 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff88c57ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563ab29799e0 con 0x563ab2a2b880 2024-05-20T20:16:28.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff88c57ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563ab297da20 con 0x563ab2a30eb0 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a30eb0 0x563ab2a31290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a30eb0 0x563ab2a31290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33824/0 (socket says 172.21.15.28:33824) 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff87a000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a317d0 0x563ab2a2b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff87b400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2bce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.928+0000 7ff87aa00700 1 -- 172.21.15.28:0/1159810137 learned_addr learned my addr 172.21.15.28:0/1159810137 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff87aa00700 1 -- 172.21.15.28:0/1159810137 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 msgr2=0x563ab2a2bce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1159810137 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2bce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff87aa00700 1 -- 172.21.15.28:0/1159810137 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a317d0 msgr2=0x563ab2a2b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1159810137 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a317d0 0x563ab2a2b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff87aa00700 1 -- 172.21.15.28:0/1159810137 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563ab2942420 con 0x563ab2a30eb0 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1159810137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a30eb0 0x563ab2a31290 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0058b0 tx=0x7ff86c0a9ce0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b96ad837458c00d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff867e00700 1 -- 172.21.15.28:0/1159810137 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff86c0aa000 con 0x563ab2a30eb0 2024-05-20T20:16:28.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff867e00700 1 -- 172.21.15.28:0/1159810137 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff86c0aabf0 con 0x563ab2a30eb0 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.929+0000 7ff867e00700 1 -- 172.21.15.28:0/1159810137 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff86c0b3090 con 0x563ab2a30eb0 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.930+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1159810137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a30eb0 msgr2=0x563ab2a31290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.930+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1159810137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a30eb0 0x563ab2a31290 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0058b0 tx=0x7ff86c0a9ce0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.931+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1159810137 shutdown_connections 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.931+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1159810137 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2bce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.931+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1159810137 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a30eb0 0x563ab2a31290 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.931+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1159810137 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a317d0 0x563ab2a2b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.931+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1159810137 >> 172.21.15.28:0/1159810137 conn(0x563ab2959be0 msgr2=0x563ab2a26010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.931+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1159810137 shutdown_connections 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.931+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1159810137 wait complete. 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.932+0000 7ff88c57ab80 1 -- start start 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.932+0000 7ff88c57ab80 1 Processor -- start 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.933+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a2b880 0x563ab2a37550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.933+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a348f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a2b880 0x563ab2a37550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a2b880 0x563ab2a37550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33840/0 (socket says 172.21.15.28:33840) 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff87aa00700 1 -- 172.21.15.28:0/1973260021 learned_addr learned my addr 172.21.15.28:0/1973260021 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a317d0 0x563ab2a34e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563ab287ffa0 con 0x563ab2a30eb0 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563ab291a760 con 0x563ab2a317d0 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563ab2a38000 con 0x563ab2a2b880 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff88c57ab80 10 client.0 _ll_register_callbacks cb 0x563ab2923c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff87aa00700 1 -- 172.21.15.28:0/1973260021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a317d0 msgr2=0x563ab2a34e30 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a317d0 0x563ab2a34e30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.942 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.934+0000 7ff87aa00700 1 -- 172.21.15.28:0/1973260021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 msgr2=0x563ab2a348f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff87a000700 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a348f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a348f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff87aa00700 1 -- 172.21.15.28:0/1973260021 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563ab28c98d0 con 0x563ab2a2b880 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff87a000700 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a348f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff87aa00700 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a2b880 0x563ab2a37550 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0aa480 tx=0x7ff86c005ef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a11b8beeaa9430e3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff864c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff864c00700 1 -- 172.21.15.28:0/1973260021 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff86c0c3070 con 0x563ab2a2b880 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff864c00700 1 -- 172.21.15.28:0/1973260021 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7ff86c0b1da0 con 0x563ab2a2b880 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.935+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff86c0aabf0 con 0x563ab2a2b880 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff864c00700 1 -- 172.21.15.28:0/1973260021 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff86c0aa4f0 con 0x563ab2a2b880 2024-05-20T20:16:28.943 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 10 client.27556 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563ab294a060 con 0x563ab2a2b880 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff864c00700 1 -- 172.21.15.28:0/1973260021 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff86c0aabf0 con 0x563ab2a2b880 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff864c00700 1 -- 172.21.15.28:0/1973260021 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff86c0a9870 con 0x563ab2a2b880 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 10 client.27556 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 10 client.27556 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 20 client.27556 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2a449f0 0x563ab2a46de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 4 client.27556 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.936+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563ab298bd90 con 0x563ab2a449f0 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.937+0000 7ff87a000700 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2a449f0 0x563ab2a46de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.937+0000 7ff864c00700 10 client.27556 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.937+0000 7ff87a000700 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2a449f0 0x563ab2a46de0 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=dda6036885bee98d server_cookie=896b8d81c2840e6 in_seq=0 out_seq=0 2024-05-20T20:16:28.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.938+0000 7ff864c00700 1 -- 172.21.15.28:0/1973260021 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x563ab298bd90 con 0x563ab2a449f0 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.938+0000 7ff864c00700 10 client.27556 handle_command_reply: tid=0 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.938+0000 7ff88c57ab80 1 client.27556 shutdown 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a2b880 msgr2=0x563ab2a37550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a2b880 0x563ab2a37550 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0aa480 tx=0x7ff86c005ef0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 shutdown_connections 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a317d0 0x563ab2a34e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2a449f0 0x563ab2a46de0 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a2b880 0x563ab2a37550 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/1973260021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a348f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 >> 172.21.15.28:0/1973260021 conn(0x563ab2959be0 msgr2=0x563ab2a2a250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff864c00700 0 client.27556 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 shutdown_connections 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 1 -- 172.21.15.28:0/1973260021 wait complete. 2024-05-20T20:16:28.945 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.939+0000 7ff88c57ab80 20 client.27556 trim_cache size 0 max 16384 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.974+0000 7ff88c57ab80 1 Processor -- start 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.974+0000 7ff88c57ab80 1 -- start start 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.976+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.976+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a2e9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.976+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a2ef00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.976+0000 7ff88c57ab80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff85c00d5e0 con 0x563ab2a30eb0 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.976+0000 7ff88c57ab80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff85c00f120 con 0x563ab2a2b880 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.976+0000 7ff88c57ab80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff85c00dcb0 con 0x563ab2a317d0 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.977+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.977+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53966/0 (socket says 172.21.15.28:53966) 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.977+0000 7ff87aa00700 1 -- 172.21.15.28:0/198729135 learned_addr learned my addr 172.21.15.28:0/198729135 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:28.946 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.978+0000 7ff87b400700 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a2ef00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.978+0000 7ff87a000700 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a2e9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.978+0000 7ff87b400700 1 -- 172.21.15.28:0/198729135 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 msgr2=0x563ab2a2b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.978+0000 7ff87b400700 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.978+0000 7ff87b400700 1 -- 172.21.15.28:0/198729135 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 msgr2=0x563ab2a2e9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.979+0000 7ff87b400700 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a2e9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.979+0000 7ff87b400700 1 -- 172.21.15.28:0/198729135 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563ab2a2f440 con 0x563ab2a317d0 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.979+0000 7ff87b400700 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a2ef00 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7ff874011350 tx=0x7ff87400b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7f7892018b930594 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.985+0000 7ff867400700 1 -- 172.21.15.28:0/198729135 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff8740092d0 con 0x563ab2a317d0 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.985+0000 7ff867400700 1 -- 172.21.15.28:0/198729135 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ff87401b070 con 0x563ab2a317d0 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.985+0000 7ff867400700 1 -- 172.21.15.28:0/198729135 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff874007500 con 0x563ab2a317d0 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.986+0000 7ff88c57ab80 1 -- 172.21.15.28:0/198729135 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 msgr2=0x563ab2a2ef00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.988+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a2ef00 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7ff874011350 tx=0x7ff87400b040 comp rx=0 tx=0).stop 2024-05-20T20:16:28.947 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.988+0000 7ff88c57ab80 1 -- 172.21.15.28:0/198729135 shutdown_connections 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.988+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a2ef00 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.988+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a2b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.988+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/198729135 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a2e9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.988+0000 7ff88c57ab80 1 -- 172.21.15.28:0/198729135 >> 172.21.15.28:0/198729135 conn(0x563ab2959be0 msgr2=0x563ab2a331e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.989+0000 7ff88c57ab80 1 -- 172.21.15.28:0/198729135 shutdown_connections 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.990+0000 7ff88c57ab80 1 -- 172.21.15.28:0/198729135 wait complete. 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.990+0000 7ff88c57ab80 1 -- start start 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.991+0000 7ff88c57ab80 1 Processor -- start 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.992+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a3d720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.993+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a4bc50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff88c57ab80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a4c190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.948 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a3d720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff87b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a4c190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff87aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a3d720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53982/0 (socket says 172.21.15.28:53982) 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff87b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a4c190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33862/0 (socket says 172.21.15.28:33862) 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff87b400700 1 -- 172.21.15.28:0/2750426956 learned_addr learned my addr 172.21.15.28:0/2750426956 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ff85c00ea50 con 0x563ab2a30eb0 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ff85c00e380 con 0x563ab2a2b880 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.994+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ff85c00b5a0 con 0x563ab2a317d0 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.996+0000 7ff87aa00700 1 -- 172.21.15.28:0/2750426956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 msgr2=0x563ab2a4c190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.996+0000 7ff87aa00700 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a4c190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.996+0000 7ff87aa00700 1 -- 172.21.15.28:0/2750426956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 msgr2=0x563ab2a4bc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.996+0000 7ff87aa00700 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a4bc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.997+0000 7ff88c57ab80 10 client.0 _ll_register_callbacks cb 0x563ab2923c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.997+0000 7ff87aa00700 1 -- 172.21.15.28:0/2750426956 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563ab2a4c830 con 0x563ab2a2b880 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.997+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ff87401b070 con 0x563ab2a2b880 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.998+0000 7ff87aa00700 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a3d720 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0920b0 tx=0x7ff86c0e3120 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3a77dba43c0cb2a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.998+0000 7ff864c00700 10 client.27629 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.998+0000 7ff864c00700 1 -- 172.21.15.28:0/2750426956 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff86c0c3070 con 0x563ab2a2b880 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.998+0000 7ff864c00700 1 -- 172.21.15.28:0/2750426956 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7ff86c0a6ae0 con 0x563ab2a2b880 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.998+0000 7ff864c00700 1 -- 172.21.15.28:0/2750426956 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ff86c0a9c60 con 0x563ab2a2b880 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.999+0000 7ff88c57ab80 10 client.27629 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:27.999+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x563ab2a4c830 con 0x563ab2a2b880 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.001+0000 7ff864c00700 1 -- 172.21.15.28:0/2750426956 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ff86c0a9750 con 0x563ab2a2b880 2024-05-20T20:16:28.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.001+0000 7ff88c57ab80 10 client.27629 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.001+0000 7ff88c57ab80 10 client.27629 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.001+0000 7ff88c57ab80 20 client.27629 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.001+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2aaf280 0x563ab2ab1740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.001+0000 7ff88c57ab80 4 client.27629 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.001+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x563ab2985570 con 0x563ab2aaf280 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.004+0000 7ff87a000700 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2aaf280 0x563ab2ab1740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.005+0000 7ff87a000700 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2aaf280 0x563ab2ab1740 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=56c78b09b1d5ae81 server_cookie=2535033b61fac984 in_seq=0 out_seq=0 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.005+0000 7ff864c00700 10 client.27629 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.006+0000 7ff864c00700 1 -- 172.21.15.28:0/2750426956 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x563ab2985570 con 0x563ab2aaf280 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.006+0000 7ff864c00700 10 client.27629 handle_command_reply: tid=0 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.006+0000 7ff88c57ab80 1 client.27629 shutdown 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.006+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 msgr2=0x563ab2a3d720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.006+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a3d720 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0920b0 tx=0x7ff86c0e3120 comp rx=0 tx=0).stop 2024-05-20T20:16:28.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.007+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 shutdown_connections 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.007+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x563ab2aaf280 0x563ab2ab1740 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.007+0000 7ff864c00700 0 client.27629 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.008+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563ab2a317d0 0x563ab2a4c190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.008+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563ab2a2b880 0x563ab2a3d720 secure :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0920b0 tx=0x7ff86c0e3120 comp rx=0 tx=0).stop 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.008+0000 7ff88c57ab80 1 --2- 172.21.15.28:0/2750426956 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563ab2a30eb0 0x563ab2a4bc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.008+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 >> 172.21.15.28:0/2750426956 conn(0x563ab2959be0 msgr2=0x563ab2a2e9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.010+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 shutdown_connections 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.010+0000 7ff88c57ab80 1 -- 172.21.15.28:0/2750426956 wait complete. 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.010+0000 7ff88c57ab80 20 client.27629 trim_cache size 0 max 16384 2024-05-20T20:16:28.952 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:16:28.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8600d4b00 msgr2=0x7ff8600d6fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8600d4b00 0x7ff8600d6fc0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7ff87c113870 tx=0x7ff868009260 comp rx=0 tx=0).stop 2024-05-20T20:16:28.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c06b220 msgr2=0x7ff87c1120c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:28.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c06b220 0x7ff87c1120c0 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7ff86c000c00 tx=0x7ff86c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 shutdown_connections 2024-05-20T20:16:28.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ff87c1095b0 0x7ff87c112b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff87c06b220 0x7ff87c1120c0 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ff8600d4b00 0x7ff8600d6fc0 secure :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7ff87c113870 tx=0x7ff868009260 comp rx=0 tx=0).stop 2024-05-20T20:16:28.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 --2- 172.21.15.28:0/1588136704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ff87c06bbe0 0x7ff87c112600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:28.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.012+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 >> 172.21.15.28:0/1588136704 conn(0x7ff87c0793f0 msgr2=0x7ff87c10a3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:28.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.016+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 shutdown_connections 2024-05-20T20:16:28.954 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:28.017+0000 7ff881c00700 1 -- 172.21.15.28:0/1588136704 wait complete. 2024-05-20T20:16:30.325 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:16:31.465 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:31.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:31 smithi028 ceph-mon[31178]: pgmap v339: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 170 MiB/s wr, 1.66k op/s 2024-05-20T20:16:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:31 smithi154 ceph-mon[36314]: pgmap v339: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 170 MiB/s wr, 1.66k op/s 2024-05-20T20:16:31.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:31 smithi033 ceph-mon[36179]: pgmap v339: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 170 MiB/s wr, 1.66k op/s 2024-05-20T20:16:33.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.147+0000 7fa9db000640 1 -- 172.21.15.28:0/2719275876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d4109f80 msgr2=0x7fa9d4111b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.147+0000 7fa9db000640 1 --2- 172.21.15.28:0/2719275876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d4109f80 0x7fa9d4111b50 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7fa9c4009900 tx=0x7fa9c402a420 comp rx=0 tx=0).stop 2024-05-20T20:16:33.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.147+0000 7fa9db000640 1 -- 172.21.15.28:0/2719275876 shutdown_connections 2024-05-20T20:16:33.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.147+0000 7fa9db000640 1 --2- 172.21.15.28:0/2719275876 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9d4105860 0x7fa9d41098a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.147+0000 7fa9db000640 1 --2- 172.21.15.28:0/2719275876 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d4109f80 0x7fa9d4111b50 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.148 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.147+0000 7fa9db000640 1 --2- 172.21.15.28:0/2719275876 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9d4104eb0 0x7fa9d4105290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.147+0000 7fa9db000640 1 -- 172.21.15.28:0/2719275876 >> 172.21.15.28:0/2719275876 conn(0x7fa9d4100750 msgr2=0x7fa9d4102b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:33.149 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.148+0000 7fa9db000640 1 -- 172.21.15.28:0/2719275876 shutdown_connections 2024-05-20T20:16:33.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.149+0000 7fa9db000640 1 -- 172.21.15.28:0/2719275876 wait complete. 2024-05-20T20:16:33.151 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.150+0000 7fa9db000640 1 Processor -- start 2024-05-20T20:16:33.152 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.151+0000 7fa9db000640 1 -- start start 2024-05-20T20:16:33.153 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.153+0000 7fa9db000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9d4104eb0 0x7fa9d4111990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.154 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.153+0000 7fa9db000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9d4109f80 0x7fa9d410ea30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.153+0000 7fa9db000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d410ef70 0x7fa9d410f420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.155 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.154+0000 7fa9d9000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d410ef70 0x7fa9d410f420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.154+0000 7fa9d9000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d410ef70 0x7fa9d410f420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33870/0 (socket says 172.21.15.28:33870) 2024-05-20T20:16:33.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.154+0000 7fa9d9000640 1 -- 172.21.15.28:0/1330956078 learned_addr learned my addr 172.21.15.28:0/1330956078 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:33.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.154+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d4114f30 con 0x7fa9d4109f80 2024-05-20T20:16:33.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.154+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d4114db0 con 0x7fa9d4104eb0 2024-05-20T20:16:33.156 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.154+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d41150b0 con 0x7fa9d410ef70 2024-05-20T20:16:33.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.155+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9d4104eb0 0x7fa9d4111990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.155+0000 7fa9d9000640 1 -- 172.21.15.28:0/1330956078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9d4104eb0 msgr2=0x7fa9d4111990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.155+0000 7fa9d9000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9d4104eb0 0x7fa9d4111990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.156+0000 7fa9d9000640 1 -- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9d4109f80 msgr2=0x7fa9d410ea30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.156+0000 7fa9d9000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9d4109f80 0x7fa9d410ea30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.156+0000 7fa9d9000640 1 -- 172.21.15.28:0/1330956078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa9d41a15f0 con 0x7fa9d410ef70 2024-05-20T20:16:33.157 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.156+0000 7fa9d9000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d410ef70 0x7fa9d410f420 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fa9c4005ab0 tx=0x7fa9c4003e90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:33.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.157+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9d4104eb0 0x7fa9d4111990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:33.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.157+0000 7fa9d0c00640 1 -- 172.21.15.28:0/1330956078 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c4040030 con 0x7fa9d410ef70 2024-05-20T20:16:33.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.157+0000 7fa9d0c00640 1 -- 172.21.15.28:0/1330956078 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9c403c8f0 con 0x7fa9d410ef70 2024-05-20T20:16:33.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.157+0000 7fa9d0c00640 1 -- 172.21.15.28:0/1330956078 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c4031810 con 0x7fa9d410ef70 2024-05-20T20:16:33.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.157+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa9d41a1880 con 0x7fa9d410ef70 2024-05-20T20:16:33.158 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.158+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa9d41a1d20 con 0x7fa9d410ef70 2024-05-20T20:16:33.164 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.163+0000 7fa9ddd7f740 1 Processor -- start 2024-05-20T20:16:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.170+0000 7fa9d0c00640 1 -- 172.21.15.28:0/1330956078 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa9c4043030 con 0x7fa9d410ef70 2024-05-20T20:16:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.173+0000 7fa9d0c00640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9b80b2b10 0x7fa9b80b4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.173+0000 7fa9d0c00640 1 -- 172.21.15.28:0/1330956078 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fa9c41024e0 con 0x7fa9d410ef70 2024-05-20T20:16:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.174+0000 7fa9ddd7f740 1 -- start start 2024-05-20T20:16:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.174+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881b8070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.175+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881c2930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.176 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.175+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c5260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.175+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560288128b90 con 0x5602881b7c90 2024-05-20T20:16:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.175+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x560288118730 con 0x5602881c2e70 2024-05-20T20:16:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.175+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56028811c770 con 0x5602881b85b0 2024-05-20T20:16:33.177 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.176+0000 7fa9d9000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c5260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.177+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881c2930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.178 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.177+0000 7fa9d3400640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9b80b2b10 0x7fa9b80b4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.178+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881c2930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33872/0 (socket says 172.21.15.28:33872) 2024-05-20T20:16:33.179 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.178+0000 7fa9d3400640 1 -- 172.21.15.28:0/3944266527 learned_addr learned my addr 172.21.15.28:0/3944266527 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:33.182 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.181+0000 7fa9d3400640 1 -- 172.21.15.28:0/3944266527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 msgr2=0x5602881c5260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.183+0000 7fa9d3400640 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c5260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.183+0000 7fa9d3400640 1 -- 172.21.15.28:0/3944266527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 msgr2=0x5602881b8070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.183+0000 7fa9d3400640 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881b8070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.184+0000 7fa9d3400640 1 -- 172.21.15.28:0/3944266527 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5602880b18b0 con 0x5602881b85b0 2024-05-20T20:16:33.185 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.184+0000 7fa9d9000640 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c5260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:33.188 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.185+0000 7fa9d3400640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9b80b2b10 0x7fa9b80b4fd0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0009f40 tx=0x7fa9c0009530 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:33.191 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.190+0000 7fa9d3400640 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881c2930 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0009fd0 tx=0x7fa9c001d000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=88649f79eab3deb7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:33.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.193+0000 7fa9c9600640 1 -- 172.21.15.28:0/3944266527 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c0005020 con 0x5602881b85b0 2024-05-20T20:16:33.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.193+0000 7fa9c9600640 1 -- 172.21.15.28:0/3944266527 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9c0022070 con 0x5602881b85b0 2024-05-20T20:16:33.194 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.193+0000 7fa9c9600640 1 -- 172.21.15.28:0/3944266527 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c001d780 con 0x5602881b85b0 2024-05-20T20:16:33.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.194+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3944266527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 msgr2=0x5602881c2930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.195 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.195+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881c2930 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0009fd0 tx=0x7fa9c001d000 comp rx=0 tx=0).stop 2024-05-20T20:16:33.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.195+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3944266527 shutdown_connections 2024-05-20T20:16:33.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.195+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c5260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.195+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881c2930 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.195+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3944266527 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881b8070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.197 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.195+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3944266527 >> 172.21.15.28:0/3944266527 conn(0x5602881b41a0 msgr2=0x5602881b4580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:33.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.196+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3944266527 shutdown_connections 2024-05-20T20:16:33.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.196+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3944266527 wait complete. 2024-05-20T20:16:33.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.198+0000 7fa9ddd7f740 1 -- start start 2024-05-20T20:16:33.202 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.199+0000 7fa9ddd7f740 1 Processor -- start 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.203+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c28b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.204+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881bfba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.204+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c02e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881bfba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881bfba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33884/0 (socket says 172.21.15.28:33884) 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9d3400640 1 -- 172.21.15.28:0/3581320844 learned_addr learned my addr 172.21.15.28:0/3581320844 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x560287dfe790 con 0x5602881b7c90 2024-05-20T20:16:33.206 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9d9000640 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c02e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5602880c0bc0 con 0x5602881c2e70 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5602881c5a40 con 0x5602881b85b0 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.205+0000 7fa9ddd7f740 10 client.0 _ll_register_callbacks cb 0x56028809e400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.206+0000 7fa9d9000640 1 -- 172.21.15.28:0/3581320844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 msgr2=0x5602881bfba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.206+0000 7fa9d9000640 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881bfba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.206+0000 7fa9d9000640 1 -- 172.21.15.28:0/3581320844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 msgr2=0x5602881c28b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.206+0000 7fa9d9000640 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c28b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.206+0000 7fa9d9000640 1 -- 172.21.15.28:0/3581320844 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x560287e2d210 con 0x5602881c2e70 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.206+0000 7fa9d9000640 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c02e0 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7fa9c40ffbc0 tx=0x7fa9c40c8ff0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f94996bf376d63b3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.207+0000 7fa9be000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.207+0000 7fa9be000640 1 -- 172.21.15.28:0/3581320844 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c4100050 con 0x5602881c2e70 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.207+0000 7fa9be000640 1 -- 172.21.15.28:0/3581320844 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fa9c40b10b0 con 0x5602881c2e70 2024-05-20T20:16:33.208 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.207+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa9c0022070 con 0x5602881c2e70 2024-05-20T20:16:33.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.216+0000 7fa9be000640 1 -- 172.21.15.28:0/3581320844 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa9c40c9750 con 0x5602881c2e70 2024-05-20T20:16:33.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.216+0000 7fa9ddd7f740 10 client.27638 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:33.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.216+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x560287ef9620 con 0x5602881c2e70 2024-05-20T20:16:33.217 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.217+0000 7fa9be000640 1 -- 172.21.15.28:0/3581320844 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9c40cdb00 con 0x5602881c2e70 2024-05-20T20:16:33.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.217+0000 7fa9be000640 1 -- 172.21.15.28:0/3581320844 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa9c40d2120 con 0x5602881c2e70 2024-05-20T20:16:33.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.218+0000 7fa9ddd7f740 10 client.27638 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:33.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.218+0000 7fa9ddd7f740 10 client.27638 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:33.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.218+0000 7fa9ddd7f740 20 client.27638 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:33.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.218+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5602881d2800 0x5602881d4c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.218+0000 7fa9ddd7f740 4 client.27638 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:33.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.218+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x7fa9c0022070 con 0x5602881d2800 2024-05-20T20:16:33.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.218+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5602881d2800 0x5602881d4c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.219+0000 7fa9be000640 10 client.27638 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:33.220 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.219+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5602881d2800 0x5602881d4c80 crc :-1 s=READY pgs=231 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d49b6dabfe10224a server_cookie=a646ead1daabccce in_seq=0 out_seq=0 2024-05-20T20:16:33.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.220+0000 7fa9be000640 1 -- 172.21.15.28:0/3581320844 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x7fa9c0022070 con 0x5602881d2800 2024-05-20T20:16:33.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.220+0000 7fa9be000640 10 client.27638 handle_command_reply: tid=0 2024-05-20T20:16:33.221 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.220+0000 7fa9ddd7f740 1 client.27638 shutdown 2024-05-20T20:16:33.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.221+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 msgr2=0x5602881c02e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.221+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c02e0 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x7fa9c40ffbc0 tx=0x7fa9c40c8ff0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.226+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 shutdown_connections 2024-05-20T20:16:33.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.226+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5602881d2800 0x5602881d4c80 crc :-1 s=READY pgs=231 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.226+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881c2e70 0x5602881c02e0 unknown :-1 s=CLOSED pgs=547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.226+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881bfba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.226+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3581320844 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c28b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.226+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 >> 172.21.15.28:0/3581320844 conn(0x5602881b41a0 msgr2=0x5602881b9f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:33.227 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.226+0000 7fa9be000640 0 client.27638 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:33.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.227+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 shutdown_connections 2024-05-20T20:16:33.228 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.227+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3581320844 wait complete. 2024-05-20T20:16:33.229 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.228+0000 7fa9ddd7f740 20 client.27638 trim_cache size 0 max 16384 2024-05-20T20:16:33.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.281+0000 7fa9ddd7f740 1 Processor -- start 2024-05-20T20:16:33.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.282+0000 7fa9ddd7f740 1 -- start start 2024-05-20T20:16:33.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.282+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b7c90 0x5602881bfb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.282+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881ceaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.282+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881c2e70 0x5602881cefe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.282+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d41bd530 con 0x5602881c2e70 2024-05-20T20:16:33.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.282+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d41bf070 con 0x5602881b7c90 2024-05-20T20:16:33.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.282+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d41bdc00 con 0x5602881b85b0 2024-05-20T20:16:33.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.283+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881ceaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.283+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881ceaa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:33896/0 (socket says 172.21.15.28:33896) 2024-05-20T20:16:33.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.284+0000 7fa9d3400640 1 -- 172.21.15.28:0/1890491054 learned_addr learned my addr 172.21.15.28:0/1890491054 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:33.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.284+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b7c90 0x5602881bfb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.284+0000 7fa9d3400640 1 -- 172.21.15.28:0/1890491054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b7c90 msgr2=0x5602881bfb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.284+0000 7fa9d3400640 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b7c90 0x5602881bfb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.284+0000 7fa9d3400640 1 -- 172.21.15.28:0/1890491054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881c2e70 msgr2=0x5602881cefe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.284+0000 7fa9d3400640 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881c2e70 0x5602881cefe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.284+0000 7fa9d3400640 1 -- 172.21.15.28:0/1890491054 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5602881b26b0 con 0x5602881b85b0 2024-05-20T20:16:33.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.285+0000 7fa9d3400640 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881ceaa0 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0012cb0 tx=0x7fa9c0005d80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=556504e1e0b46d85 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:33.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.286+0000 7fa9c8c00640 1 -- 172.21.15.28:0/1890491054 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c0010e80 con 0x5602881b85b0 2024-05-20T20:16:33.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.286+0000 7fa9c8c00640 1 -- 172.21.15.28:0/1890491054 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9c0022050 con 0x5602881b85b0 2024-05-20T20:16:33.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.286+0000 7fa9c8c00640 1 -- 172.21.15.28:0/1890491054 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c001dd70 con 0x5602881b85b0 2024-05-20T20:16:33.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.287+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/1890491054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 msgr2=0x5602881ceaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.287+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881ceaa0 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0012cb0 tx=0x7fa9c0005d80 comp rx=0 tx=0).stop 2024-05-20T20:16:33.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.288+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/1890491054 shutdown_connections 2024-05-20T20:16:33.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.288+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881c2e70 0x5602881cefe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.288+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881b85b0 0x5602881ceaa0 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.288+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/1890491054 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b7c90 0x5602881bfb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.288+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/1890491054 >> 172.21.15.28:0/1890491054 conn(0x5602881b41a0 msgr2=0x5602881bf430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:33.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.288+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/1890491054 shutdown_connections 2024-05-20T20:16:33.289 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.288+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/1890491054 wait complete. 2024-05-20T20:16:33.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.289+0000 7fa9ddd7f740 1 -- start start 2024-05-20T20:16:33.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.290+0000 7fa9ddd7f740 1 Processor -- start 2024-05-20T20:16:33.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c8260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b85b0 0x5602881c87a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9ddd7f740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881c2e70 0x56028823b460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d41be9a0 con 0x5602881b7c90 2024-05-20T20:16:33.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d41be2d0 con 0x5602881b85b0 2024-05-20T20:16:33.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9ddd7f740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa9d41bb530 con 0x5602881c2e70 2024-05-20T20:16:33.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9ddd7f740 10 client.0 _ll_register_callbacks cb 0x7fa9c40095e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.291+0000 7fa9d3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c8260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.292+0000 7fa9d3e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c8260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53678/0 (socket says 172.21.15.28:53678) 2024-05-20T20:16:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.292+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b85b0 0x5602881c87a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.292+0000 7fa9d3400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b85b0 0x5602881c87a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54034/0 (socket says 172.21.15.28:54034) 2024-05-20T20:16:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.292+0000 7fa9d3400640 1 -- 172.21.15.28:0/3645744321 learned_addr learned my addr 172.21.15.28:0/3645744321 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:33.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.292+0000 7fa9d9000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881c2e70 0x56028823b460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.294+0000 7fa9d3400640 1 -- 172.21.15.28:0/3645744321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881c2e70 msgr2=0x56028823b460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.294+0000 7fa9d3400640 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881c2e70 0x56028823b460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.294+0000 7fa9d3400640 1 -- 172.21.15.28:0/3645744321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 msgr2=0x5602881c8260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.294+0000 7fa9d3400640 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c8260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.294+0000 7fa9d3400640 1 -- 172.21.15.28:0/3645744321 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56028823b9f0 con 0x5602881b85b0 2024-05-20T20:16:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.294+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c8260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:33.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.295+0000 7fa9be000640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.295+0000 7fa9d3400640 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b85b0 0x5602881c87a0 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0009dd0 tx=0x7fa9c0004920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1d20a5e2a0537c26 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.295+0000 7fa9be000640 1 -- 172.21.15.28:0/3645744321 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9c003d060 con 0x5602881b85b0 2024-05-20T20:16:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.295+0000 7fa9be000640 1 -- 172.21.15.28:0/3645744321 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fa9c001dc50 con 0x5602881b85b0 2024-05-20T20:16:33.296 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.296+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa9c0022050 con 0x5602881b85b0 2024-05-20T20:16:33.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.296+0000 7fa9be000640 1 -- 172.21.15.28:0/3645744321 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa9c0010e90 con 0x5602881b85b0 2024-05-20T20:16:33.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.296+0000 7fa9ddd7f740 10 client.27644 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:33.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.296+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56028823cd00 con 0x5602881b85b0 2024-05-20T20:16:33.297 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.297+0000 7fa9be000640 1 -- 172.21.15.28:0/3645744321 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9c0010e90 con 0x5602881b85b0 2024-05-20T20:16:33.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.298+0000 7fa9be000640 1 -- 172.21.15.28:0/3645744321 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa9c0004040 con 0x5602881b85b0 2024-05-20T20:16:33.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.298+0000 7fa9ddd7f740 10 client.27644 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:33.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.298+0000 7fa9ddd7f740 10 client.27644 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:33.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.298+0000 7fa9ddd7f740 20 client.27644 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:33.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.298+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56028823d370 0x56028823f790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:33.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.298+0000 7fa9ddd7f740 4 client.27644 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:16:33.299 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.298+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56028812aae0 con 0x56028823d370 2024-05-20T20:16:33.301 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.301+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56028823d370 0x56028823f790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:33.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.302+0000 7fa9d3e00640 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56028823d370 0x56028823f790 crc :-1 s=READY pgs=232 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7393b07efc03c082 server_cookie=4426cb050d1b8af3 in_seq=0 out_seq=0 2024-05-20T20:16:33.303 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.302+0000 7fa9be000640 10 client.27644 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:33.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:33 smithi154 ceph-mon[36314]: pgmap v340: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 149 MiB/s wr, 1.45k op/s 2024-05-20T20:16:33.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.385+0000 7fa9be000640 1 -- 172.21.15.28:0/3645744321 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x56028812aae0 con 0x56028823d370 2024-05-20T20:16:33.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.385+0000 7fa9be000640 10 client.27644 handle_command_reply: tid=0 2024-05-20T20:16:33.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.386+0000 7fa9ddd7f740 1 client.27644 shutdown 2024-05-20T20:16:33.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.389+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b85b0 msgr2=0x5602881c87a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.389+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b85b0 0x5602881c87a0 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0009dd0 tx=0x7fa9c0004920 comp rx=0 tx=0).stop 2024-05-20T20:16:33.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.389+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 shutdown_connections 2024-05-20T20:16:33.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.389+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56028823d370 0x56028823f790 crc :-1 s=READY pgs=232 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.389+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5602881b85b0 0x5602881c87a0 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.389+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5602881c2e70 0x56028823b460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.389+0000 7fa9ddd7f740 1 --2- 172.21.15.28:0/3645744321 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5602881b7c90 0x5602881c8260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.390+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 >> 172.21.15.28:0/3645744321 conn(0x5602881b41a0 msgr2=0x5602881b3310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.390+0000 7fa9be000640 0 client.27644 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:33.391 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.390+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 shutdown_connections 2024-05-20T20:16:33.392 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.390+0000 7fa9ddd7f740 1 -- 172.21.15.28:0/3645744321 wait complete. 2024-05-20T20:16:33.393 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.392+0000 7fa9ddd7f740 20 client.27644 trim_cache size 0 max 16384 2024-05-20T20:16:33.398 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:16:33.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:16:33.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:16:33.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.399+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9b80b2b10 msgr2=0x7fa9b80b4fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.405 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.400+0000 7fa9db000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9b80b2b10 0x7fa9b80b4fd0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fa9c0009f40 tx=0x7fa9c0009530 comp rx=0 tx=0).stop 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.400+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d410ef70 msgr2=0x7fa9d410f420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.400+0000 7fa9db000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d410ef70 0x7fa9d410f420 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fa9c4005ab0 tx=0x7fa9c4003e90 comp rx=0 tx=0).stop 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.402+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 shutdown_connections 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.402+0000 7fa9db000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9b80b2b10 0x7fa9b80b4fd0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.402+0000 7fa9db000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9d410ef70 0x7fa9d410f420 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.402+0000 7fa9db000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9d4109f80 0x7fa9d410ea30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.402+0000 7fa9db000640 1 --2- 172.21.15.28:0/1330956078 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9d4104eb0 0x7fa9d4111990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:33.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.402+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 >> 172.21.15.28:0/1330956078 conn(0x7fa9d4100750 msgr2=0x7fa9d41011b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:33.409 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.405+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 shutdown_connections 2024-05-20T20:16:33.409 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:33.406+0000 7fa9db000640 1 -- 172.21.15.28:0/1330956078 wait complete. 2024-05-20T20:16:33.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:33 smithi028 ceph-mon[31178]: pgmap v340: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 149 MiB/s wr, 1.45k op/s 2024-05-20T20:16:33.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:33 smithi033 ceph-mon[36179]: pgmap v340: 97 pgs: 97 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 149 MiB/s wr, 1.45k op/s 2024-05-20T20:16:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:35 smithi028 ceph-mon[31178]: pgmap v341: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 245 MiB/s wr, 2.06k op/s 2024-05-20T20:16:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:35 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:35 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:35 smithi154 ceph-mon[36314]: pgmap v341: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 245 MiB/s wr, 2.06k op/s 2024-05-20T20:16:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:35 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:35 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:35.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:35 smithi033 ceph-mon[36179]: pgmap v341: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 245 MiB/s wr, 2.06k op/s 2024-05-20T20:16:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:35 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:35 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:37.207 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:37 smithi028 ceph-mon[31178]: pgmap v342: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 216 MiB/s wr, 1.64k op/s 2024-05-20T20:16:37.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:37 smithi154 ceph-mon[36314]: pgmap v342: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 216 MiB/s wr, 1.64k op/s 2024-05-20T20:16:37.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:37 smithi033 ceph-mon[36179]: pgmap v342: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 216 MiB/s wr, 1.64k op/s 2024-05-20T20:16:38.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.447+0000 7f5606000700 1 -- 172.21.15.28:0/2969510776 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f55f80abd40 msgr2=0x7f55f80ac120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.447+0000 7f5606000700 1 --2- 172.21.15.28:0/2969510776 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f55f80abd40 0x7f55f80ac120 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7f55ec00fb60 tx=0x7f55ec00fe70 comp rx=0 tx=0).stop 2024-05-20T20:16:38.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.448+0000 7f5606000700 1 -- 172.21.15.28:0/2969510776 shutdown_connections 2024-05-20T20:16:38.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.448+0000 7f5606000700 1 --2- 172.21.15.28:0/2969510776 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a5780 0x7f55f80b3be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.449 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.448+0000 7f5606000700 1 --2- 172.21.15.28:0/2969510776 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f55f80abd40 0x7f55f80ac120 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.448+0000 7f5606000700 1 --2- 172.21.15.28:0/2969510776 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f55f80a4dc0 0x7f55f80a5240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.448+0000 7f5606000700 1 -- 172.21.15.28:0/2969510776 >> 172.21.15.28:0/2969510776 conn(0x7f55f801a680 msgr2=0x7f55f801aa90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:38.450 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.449+0000 7f5606000700 1 -- 172.21.15.28:0/2969510776 shutdown_connections 2024-05-20T20:16:38.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.450+0000 7f5606000700 1 -- 172.21.15.28:0/2969510776 wait complete. 2024-05-20T20:16:38.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.453+0000 7f5606000700 1 Processor -- start 2024-05-20T20:16:38.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.453+0000 7f5606000700 1 -- start start 2024-05-20T20:16:38.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.454+0000 7f5606000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a4dc0 0x7f55f81496c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.455+0000 7f5606000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f55f80a5780 0x7f55f8149c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.455+0000 7f5606000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f55f80abd40 0x7f55f8143880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.455+0000 7f5604c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a4dc0 0x7f55f81496c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.455+0000 7f5604c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a4dc0 0x7f55f81496c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58330/0 (socket says 172.21.15.28:58330) 2024-05-20T20:16:38.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.456+0000 7f5604c00700 1 -- 172.21.15.28:0/2862700834 learned_addr learned my addr 172.21.15.28:0/2862700834 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:38.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.456+0000 7f5605600700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f55f80abd40 0x7f55f8143880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.456+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f55f80bd5a0 con 0x7f55f80a5780 2024-05-20T20:16:38.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.456+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f55f80bd420 con 0x7f55f80a4dc0 2024-05-20T20:16:38.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.456+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f55f80bd720 con 0x7f55f80abd40 2024-05-20T20:16:38.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.457+0000 7f5604c00700 1 -- 172.21.15.28:0/2862700834 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f55f80abd40 msgr2=0x7f55f8143880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.457+0000 7f5604c00700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f55f80abd40 0x7f55f8143880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.457+0000 7f5604c00700 1 -- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f55f80a5780 msgr2=0x7f55f8149c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.457+0000 7f5604c00700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f55f80a5780 0x7f55f8149c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.457+0000 7f5604c00700 1 -- 172.21.15.28:0/2862700834 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f55f8143fe0 con 0x7f55f80a4dc0 2024-05-20T20:16:38.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.457+0000 7f5604c00700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a4dc0 0x7f55f81496c0 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f55ec005e80 tx=0x7f55ec00e980 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:38.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.458+0000 7f55fd600700 1 -- 172.21.15.28:0/2862700834 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55ec01c030 con 0x7f55f80a4dc0 2024-05-20T20:16:38.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.458+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f55f8144270 con 0x7f55f80a4dc0 2024-05-20T20:16:38.459 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.458+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f55f8150590 con 0x7f55f80a4dc0 2024-05-20T20:16:38.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.458+0000 7f55fd600700 1 -- 172.21.15.28:0/2862700834 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f55ec005590 con 0x7f55f80a4dc0 2024-05-20T20:16:38.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.458+0000 7f55fd600700 1 -- 172.21.15.28:0/2862700834 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55ec0038e0 con 0x7f55f80a4dc0 2024-05-20T20:16:38.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.465+0000 7f55fd600700 1 -- 172.21.15.28:0/2862700834 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f55ec01f030 con 0x7f55f80a4dc0 2024-05-20T20:16:38.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.468+0000 7f55fd600700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f55e80b2af0 0x7f55e80b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.469+0000 7f55fd600700 1 -- 172.21.15.28:0/2862700834 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f55ec0e3740 con 0x7f55f80a4dc0 2024-05-20T20:16:38.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.475+0000 7f5611fdbb80 1 Processor -- start 2024-05-20T20:16:38.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.476+0000 7f5611fdbb80 1 -- start start 2024-05-20T20:16:38.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.477+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e28220 0x55e547e28600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.477+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e22910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.477+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e22e50 0x55e547e2ef70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.478+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e547d80e40 con 0x55e547e224b0 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.478+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e547d709e0 con 0x55e547e22e50 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.478+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e547d74a20 con 0x55e547e28220 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.478+0000 7f5605600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e22e50 0x55e547e2ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.478+0000 7f5605600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e22e50 0x55e547e2ef70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58342/0 (socket says 172.21.15.28:58342) 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.478+0000 7f5605600700 1 -- 172.21.15.28:0/2803476584 learned_addr learned my addr 172.21.15.28:0/2803476584 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.478+0000 7f5604c00700 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e28220 0x55e547e28600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.479+0000 7f5604c00700 1 -- 172.21.15.28:0/2803476584 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e22e50 msgr2=0x55e547e2ef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.479+0000 7f5604c00700 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e22e50 0x55e547e2ef70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.479+0000 7f5604c00700 1 -- 172.21.15.28:0/2803476584 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 msgr2=0x55e547e22910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:38.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.479+0000 7f5604c00700 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e22910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.479+0000 7f5604c00700 1 -- 172.21.15.28:0/2803476584 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e547d39420 con 0x55e547e28220 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.479+0000 7f5605600700 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e22e50 0x55e547e2ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.479+0000 7f5604c00700 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e28220 0x55e547e28600 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f55ec0e0e20 tx=0x7f55ec0aa810 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dfed8dc9ca3cd093 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.480+0000 7f55f2000700 1 -- 172.21.15.28:0/2803476584 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55ec026070 con 0x55e547e28220 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.480+0000 7f55f2000700 1 -- 172.21.15.28:0/2803476584 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f55ec0b3160 con 0x55e547e28220 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.480+0000 7f55f2000700 1 -- 172.21.15.28:0/2803476584 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55ec0cd310 con 0x55e547e28220 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.481+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2803476584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e28220 msgr2=0x55e547e28600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.481+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e28220 0x55e547e28600 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f55ec0e0e20 tx=0x7f55ec0aa810 comp rx=0 tx=0).stop 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.481+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2803476584 shutdown_connections 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.481+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e22e50 0x55e547e2ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.481+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e28220 0x55e547e28600 secure :-1 s=CLOSED pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f55ec0e0e20 tx=0x7f55ec0aa810 comp rx=0 tx=0).stop 2024-05-20T20:16:38.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.481+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2803476584 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e22910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.481+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2803476584 >> 172.21.15.28:0/2803476584 conn(0x55e547d50be0 msgr2=0x55e547e1d1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.482+0000 7f55ffe00700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f55e80b2af0 0x7f55e80b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.485+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2803476584 shutdown_connections 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.485+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2803476584 wait complete. 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.486+0000 7f5611fdbb80 1 -- start start 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.486+0000 7f5611fdbb80 1 Processor -- start 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.488+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e31b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.488+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e320d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.488+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e2c390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.488+0000 7f55ffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e320d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.489+0000 7f55ffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e320d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49988/0 (socket says 172.21.15.28:49988) 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.489+0000 7f55ffe00700 1 -- 172.21.15.28:0/2165394725 learned_addr learned my addr 172.21.15.28:0/2165394725 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.489+0000 7f5605600700 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e2c390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.489+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e547c76fa0 con 0x55e547e224b0 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.489+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e547d11760 con 0x55e547e2bf10 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.489+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e547d74a20 con 0x55e547e22e50 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.490+0000 7f55ffe00700 1 -- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 msgr2=0x55e547e2c390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.490+0000 7f55ffe00700 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e2c390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.490+0000 7f55ffe00700 1 -- 172.21.15.28:0/2165394725 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 msgr2=0x55e547e31b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.490+0000 7f55ffe00700 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e31b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.490+0000 7f55ffe00700 1 -- 172.21.15.28:0/2165394725 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e547cc08d0 con 0x55e547e22e50 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.491+0000 7f5605600700 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e2c390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:38.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.491+0000 7f5611fdbb80 10 client.0 _ll_register_callbacks cb 0x55e547d1ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:38.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.491+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f55ec0b3160 con 0x55e547e22e50 2024-05-20T20:16:38.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.496+0000 7f55ffe00700 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e320d0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f55f4014c30 tx=0x7f55f4014f40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a1967216b1e4032 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:38.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.496+0000 7f55e6a00700 10 client.27592 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:16:38.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.497+0000 7f55e6a00700 1 -- 172.21.15.28:0/2165394725 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55f4011080 con 0x55e547e22e50 2024-05-20T20:16:38.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.497+0000 7f55e6a00700 1 -- 172.21.15.28:0/2165394725 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f55f4011ac0 con 0x55e547e22e50 2024-05-20T20:16:38.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.497+0000 7f55e6a00700 1 -- 172.21.15.28:0/2165394725 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f55f4011ce0 con 0x55e547e22e50 2024-05-20T20:16:38.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.501+0000 7f5611fdbb80 10 client.27592 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:38.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.501+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({fsmap=29}) -- 0x55e547d41060 con 0x55e547e22e50 2024-05-20T20:16:38.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.501+0000 7f55ffe00700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f55e80b2af0 0x7f55e80b4fb0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f55f4011000 tx=0x7f55f401a000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:38.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.504+0000 7f55e6a00700 1 -- 172.21.15.28:0/2165394725 <== mon.1 v2:172.21.15.154:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f55f400fa60 con 0x55e547e22e50 2024-05-20T20:16:38.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.504+0000 7f5611fdbb80 10 client.27592 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:38.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.504+0000 7f5611fdbb80 10 client.27592 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:16:38.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.505+0000 7f5611fdbb80 20 client.27592 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:38.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.505+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547e3c010 0x55e547e3e4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.505+0000 7f5611fdbb80 4 client.27592 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:38.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.505+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e547d82d90 con 0x55e547e3c010 2024-05-20T20:16:38.509 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.506+0000 7f5604c00700 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547e3c010 0x55e547e3e4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.510 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.508+0000 7f5604c00700 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547e3c010 0x55e547e3e4a0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=deed29674cc16393 server_cookie=8f2769ac8c7a2fbf in_seq=0 out_seq=0 2024-05-20T20:16:38.510 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.508+0000 7f55e6a00700 10 client.27592 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:16:38.510 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.509+0000 7f55e6a00700 1 -- 172.21.15.28:0/2165394725 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55e547d82d90 con 0x55e547e3c010 2024-05-20T20:16:38.510 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.509+0000 7f55e6a00700 10 client.27592 handle_command_reply: tid=0 2024-05-20T20:16:38.510 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.509+0000 7f5611fdbb80 1 client.27592 shutdown 2024-05-20T20:16:38.510 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.510+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 msgr2=0x55e547e320d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.510+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e320d0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f55f4014c30 tx=0x7f55f4014f40 comp rx=0 tx=0).stop 2024-05-20T20:16:38.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.513+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 shutdown_connections 2024-05-20T20:16:38.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.513+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547e3c010 0x55e547e3e4a0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.513+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e2c390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.515 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.513+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e31b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.515 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.513+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/2165394725 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e320d0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.515 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.513+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 >> 172.21.15.28:0/2165394725 conn(0x55e547d50be0 msgr2=0x55e547e214a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:38.517 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.514+0000 7f55e6a00700 0 client.27592 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:16:38.517 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.514+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 shutdown_connections 2024-05-20T20:16:38.517 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.514+0000 7f5611fdbb80 1 -- 172.21.15.28:0/2165394725 wait complete. 2024-05-20T20:16:38.517 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.515+0000 7f5611fdbb80 20 client.27592 trim_cache size 0 max 16384 2024-05-20T20:16:38.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.579+0000 7f5611fdbb80 1 Processor -- start 2024-05-20T20:16:38.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.580+0000 7f5611fdbb80 1 -- start start 2024-05-20T20:16:38.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.584+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e21bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.585+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e43d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.585+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e442b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.585+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f55e000d610 con 0x55e547e224b0 2024-05-20T20:16:38.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.585+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f55e000f150 con 0x55e547e2bf10 2024-05-20T20:16:38.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.585+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f55e000dce0 con 0x55e547e22e50 2024-05-20T20:16:38.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.589+0000 7f55ffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e43d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.589+0000 7f55ffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e43d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:49990/0 (socket says 172.21.15.28:49990) 2024-05-20T20:16:38.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.589+0000 7f55ffe00700 1 -- 172.21.15.28:0/611122017 learned_addr learned my addr 172.21.15.28:0/611122017 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:38.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f5605600700 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e442b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f5604c00700 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e21bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f5604c00700 1 -- 172.21.15.28:0/611122017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 msgr2=0x55e547e43d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:38.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f5604c00700 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e43d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f5604c00700 1 -- 172.21.15.28:0/611122017 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 msgr2=0x55e547e442b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:38.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f5604c00700 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e442b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f5604c00700 1 -- 172.21.15.28:0/611122017 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e547e238d0 con 0x55e547e224b0 2024-05-20T20:16:38.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.590+0000 7f55ffe00700 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e43d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:38.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.591+0000 7f5604c00700 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e21bb0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x55e547e23620 tx=0x7f55ec005040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e28a7af858fa7d1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:38.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.591+0000 7f55f1600700 1 -- 172.21.15.28:0/611122017 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55ec026040 con 0x55e547e224b0 2024-05-20T20:16:38.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.593+0000 7f55f1600700 1 -- 172.21.15.28:0/611122017 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f55ec0ab140 con 0x55e547e224b0 2024-05-20T20:16:38.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.593+0000 7f55f1600700 1 -- 172.21.15.28:0/611122017 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55ec0b3810 con 0x55e547e224b0 2024-05-20T20:16:38.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.595+0000 7f5611fdbb80 1 -- 172.21.15.28:0/611122017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 msgr2=0x55e547e21bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.595+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e21bb0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x55e547e23620 tx=0x7f55ec005040 comp rx=0 tx=0).stop 2024-05-20T20:16:38.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.596+0000 7f5611fdbb80 1 -- 172.21.15.28:0/611122017 shutdown_connections 2024-05-20T20:16:38.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.596+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e2bf10 0x55e547e442b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.596+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e224b0 0x55e547e21bb0 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.596+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/611122017 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e22e50 0x55e547e43d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.597+0000 7f5611fdbb80 1 -- 172.21.15.28:0/611122017 >> 172.21.15.28:0/611122017 conn(0x55e547d50be0 msgr2=0x55e547e214a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:38.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.599+0000 7f5611fdbb80 1 -- 172.21.15.28:0/611122017 shutdown_connections 2024-05-20T20:16:38.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.600+0000 7f5611fdbb80 1 -- 172.21.15.28:0/611122017 wait complete. 2024-05-20T20:16:38.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.601+0000 7f5611fdbb80 1 -- start start 2024-05-20T20:16:38.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.601+0000 7f5611fdbb80 1 Processor -- start 2024-05-20T20:16:38.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.607+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e224b0 0x55e547e38530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.607+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e22e50 0x55e547e249e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.607+0000 7f5611fdbb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e2bf10 0x55e547e24f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.607+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f55e000ea80 con 0x55e547e22e50 2024-05-20T20:16:38.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.608+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f55e000e3b0 con 0x55e547e224b0 2024-05-20T20:16:38.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.608+0000 7f5611fdbb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f55e000b5d0 con 0x55e547e2bf10 2024-05-20T20:16:38.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.608+0000 7f5611fdbb80 10 client.0 _ll_register_callbacks cb 0x55e547d1ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:38.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.608+0000 7f55ffe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e22e50 0x55e547e249e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.608+0000 7f55ffe00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e22e50 0x55e547e249e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:49708/0 (socket says 172.21.15.28:49708) 2024-05-20T20:16:38.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.608+0000 7f55ffe00700 1 -- 172.21.15.28:0/4294345293 learned_addr learned my addr 172.21.15.28:0/4294345293 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:38.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.609+0000 7f55ffe00700 1 -- 172.21.15.28:0/4294345293 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e2bf10 msgr2=0x55e547e24f20 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:16:38.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.609+0000 7f55ffe00700 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e2bf10 0x55e547e24f20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.609+0000 7f55ffe00700 1 -- 172.21.15.28:0/4294345293 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e224b0 msgr2=0x55e547e38530 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:16:38.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.609+0000 7f55ffe00700 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e224b0 0x55e547e38530 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.609+0000 7f55ffe00700 1 -- 172.21.15.28:0/4294345293 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e547e25710 con 0x55e547e22e50 2024-05-20T20:16:38.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.610+0000 7f55e6a00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:16:38.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.611+0000 7f55ffe00700 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e22e50 0x55e547e249e0 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7f55f4014c30 tx=0x7f55f4009870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a6b77d2016144373 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:38.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.611+0000 7f55e6a00700 1 -- 172.21.15.28:0/4294345293 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f55f4035060 con 0x55e547e22e50 2024-05-20T20:16:38.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.612+0000 7f55e6a00700 1 -- 172.21.15.28:0/4294345293 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f55f4011710 con 0x55e547e22e50 2024-05-20T20:16:38.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.613+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f55ec0ab140 con 0x55e547e22e50 2024-05-20T20:16:38.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.616+0000 7f55e6a00700 1 -- 172.21.15.28:0/4294345293 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f55f40099f0 con 0x55e547e22e50 2024-05-20T20:16:38.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.616+0000 7f5611fdbb80 10 client.17160 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:38.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.616+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e547ea6c70 con 0x55e547e22e50 2024-05-20T20:16:38.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.617+0000 7f55e6a00700 1 -- 172.21.15.28:0/4294345293 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f55ec0ab140 con 0x55e547e22e50 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.618+0000 7f55e6a00700 1 -- 172.21.15.28:0/4294345293 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f55f402f070 con 0x55e547e22e50 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.621+0000 7f5611fdbb80 10 client.17160 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.621+0000 7f5611fdbb80 10 client.17160 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.621+0000 7f5611fdbb80 20 client.17160 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.621+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547ea72c0 0x55e547ea96e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.621+0000 7f5611fdbb80 4 client.17160 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.621+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55e547d7c570 con 0x55e547ea72c0 2024-05-20T20:16:38.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.621+0000 7f5604c00700 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547ea72c0 0x55e547ea96e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.623+0000 7f5604c00700 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547ea72c0 0x55e547ea96e0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=16ba28ccd4b10b39 server_cookie=8c65d1192f7097c8 in_seq=0 out_seq=0 2024-05-20T20:16:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.623+0000 7f55e6a00700 10 client.17160 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:16:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.623+0000 7f55e6a00700 1 -- 172.21.15.28:0/4294345293 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55e547d7c570 con 0x55e547ea72c0 2024-05-20T20:16:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.623+0000 7f55e6a00700 10 client.17160 handle_command_reply: tid=0 2024-05-20T20:16:38.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.623+0000 7f5611fdbb80 1 client.17160 shutdown 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e22e50 msgr2=0x55e547e249e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e22e50 0x55e547e249e0 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7f55f4014c30 tx=0x7f55f4009870 comp rx=0 tx=0).stop 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 shutdown_connections 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x55e547ea72c0 0x55e547ea96e0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e547e2bf10 0x55e547e24f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e547e224b0 0x55e547e38530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 --2- 172.21.15.28:0/4294345293 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e547e22e50 0x55e547e249e0 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.625 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.624+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 >> 172.21.15.28:0/4294345293 conn(0x55e547d50be0 msgr2=0x55e547e21d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:38.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.625+0000 7f55e6a00700 0 client.17160 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:16:38.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.625+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 shutdown_connections 2024-05-20T20:16:38.626 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.625+0000 7f5611fdbb80 1 -- 172.21.15.28:0/4294345293 wait complete. 2024-05-20T20:16:38.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.626+0000 7f5611fdbb80 20 client.17160 trim_cache size 0 max 16384 2024-05-20T20:16:38.631 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:16:38.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.633+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f55e80b2af0 msgr2=0x7f55e80b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.633+0000 7f5606000700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f55e80b2af0 0x7f55e80b4fb0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f55f4011000 tx=0x7f55f401a000 comp rx=0 tx=0).stop 2024-05-20T20:16:38.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.633+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a4dc0 msgr2=0x7f55f81496c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:38.634 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.633+0000 7f5606000700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a4dc0 0x7f55f81496c0 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7f55ec005e80 tx=0x7f55ec00e980 comp rx=0 tx=0).stop 2024-05-20T20:16:38.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.634+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 shutdown_connections 2024-05-20T20:16:38.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.634+0000 7f5606000700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f55f80abd40 0x7f55f8143880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.634+0000 7f5606000700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f55e80b2af0 0x7f55e80b4fb0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.634+0000 7f5606000700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f55f80a4dc0 0x7f55f81496c0 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.634+0000 7f5606000700 1 --2- 172.21.15.28:0/2862700834 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f55f80a5780 0x7f55f8149c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:38.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.634+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 >> 172.21.15.28:0/2862700834 conn(0x7f55f801a680 msgr2=0x7f55f80a2a60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:38.635 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.634+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 shutdown_connections 2024-05-20T20:16:38.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:38.635+0000 7f5606000700 1 -- 172.21.15.28:0/2862700834 wait complete. 2024-05-20T20:16:39.227 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: pgmap v343: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 231 MiB/s wr, 1.71k op/s 2024-05-20T20:16:39.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:16:39.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:16:39.228 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:39 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: pgmap v343: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 231 MiB/s wr, 1.71k op/s 2024-05-20T20:16:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:16:39.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:16:39.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:39 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: pgmap v343: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 231 MiB/s wr, 1.71k op/s 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:16:39.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:39 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:16:40.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:40 smithi028 ceph-mon[31178]: Adjusting osd_memory_target on smithi028 to 177.5M 2024-05-20T20:16:40.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:40 smithi028 ceph-mon[31178]: Unable to set osd_memory_target on smithi028 to 186173030: error parsing value: Value '186173030' is below minimum 939524096 2024-05-20T20:16:40.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:40 smithi154 ceph-mon[36314]: Adjusting osd_memory_target on smithi028 to 177.5M 2024-05-20T20:16:40.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:40 smithi154 ceph-mon[36314]: Unable to set osd_memory_target on smithi028 to 186173030: error parsing value: Value '186173030' is below minimum 939524096 2024-05-20T20:16:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:40 smithi033 ceph-mon[36179]: Adjusting osd_memory_target on smithi028 to 177.5M 2024-05-20T20:16:40.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:40 smithi033 ceph-mon[36179]: Unable to set osd_memory_target on smithi028 to 186173030: error parsing value: Value '186173030' is below minimum 939524096 2024-05-20T20:16:41.359 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:16:41.576 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:41 smithi028 ceph-mon[31178]: pgmap v344: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 324 MiB/s wr, 2.23k op/s 2024-05-20T20:16:41.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:41 smithi154 ceph-mon[36314]: pgmap v344: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 324 MiB/s wr, 2.23k op/s 2024-05-20T20:16:41.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:41 smithi033 ceph-mon[36179]: pgmap v344: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 324 MiB/s wr, 2.23k op/s 2024-05-20T20:16:42.456 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:43.534 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:43 smithi028 ceph-mon[31178]: pgmap v345: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 261 MiB/s wr, 1.71k op/s 2024-05-20T20:16:43.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:43 smithi154 ceph-mon[36314]: pgmap v345: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 261 MiB/s wr, 1.71k op/s 2024-05-20T20:16:43.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:43 smithi033 ceph-mon[36179]: pgmap v345: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 261 MiB/s wr, 1.71k op/s 2024-05-20T20:16:45.292 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:45 smithi028 ceph-mon[31178]: pgmap v346: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 407 MiB/s wr, 2.76k op/s 2024-05-20T20:16:45.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:45 smithi154 ceph-mon[36314]: pgmap v346: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 407 MiB/s wr, 2.76k op/s 2024-05-20T20:16:45.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:45 smithi033 ceph-mon[36179]: pgmap v346: 97 pgs: 97 active+clean; 60 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 407 MiB/s wr, 2.76k op/s 2024-05-20T20:16:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.560+0000 7fa83aa00700 1 -- 172.21.15.28:0/948946067 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8341095b0 msgr2=0x7fa834109990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.561 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.560+0000 7fa83aa00700 1 --2- 172.21.15.28:0/948946067 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8341095b0 0x7fa834109990 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fa82c0098f0 tx=0x7fa82c009c00 comp rx=0 tx=0).stop 2024-05-20T20:16:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.563+0000 7fa83aa00700 1 -- 172.21.15.28:0/948946067 shutdown_connections 2024-05-20T20:16:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.563+0000 7fa83aa00700 1 --2- 172.21.15.28:0/948946067 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa83406bbe0 0x7fa8341103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.563+0000 7fa83aa00700 1 --2- 172.21.15.28:0/948946067 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa8341095b0 0x7fa834109990 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.563+0000 7fa83aa00700 1 --2- 172.21.15.28:0/948946067 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa83406b220 0x7fa83406b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.563+0000 7fa83aa00700 1 -- 172.21.15.28:0/948946067 >> 172.21.15.28:0/948946067 conn(0x7fa8340793f0 msgr2=0x7fa834079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.565+0000 7fa83aa00700 1 -- 172.21.15.28:0/948946067 shutdown_connections 2024-05-20T20:16:46.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.565+0000 7fa83aa00700 1 -- 172.21.15.28:0/948946067 wait complete. 2024-05-20T20:16:46.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.571+0000 7fa83aa00700 1 Processor -- start 2024-05-20T20:16:46.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.571+0000 7fa83aa00700 1 -- start start 2024-05-20T20:16:46.574 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa83aa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa83406b220 0x7fa83419be90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa83aa00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa83406bbe0 0x7fa83419c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa83aa00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8341095b0 0x7fa8341a0760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa83aa00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa834113760 con 0x7fa83406b220 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa83aa00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341135e0 con 0x7fa8341095b0 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa83aa00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341138e0 con 0x7fa83406bbe0 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa833400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa83406bbe0 0x7fa83419c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa838a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8341095b0 0x7fa8341a0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa838a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8341095b0 0x7fa8341a0760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58390/0 (socket says 172.21.15.28:58390) 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.572+0000 7fa838a00700 1 -- 172.21.15.28:0/236205846 learned_addr learned my addr 172.21.15.28:0/236205846 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa838a00700 1 -- 172.21.15.28:0/236205846 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa83406bbe0 msgr2=0x7fa83419c3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa833e00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa83406b220 0x7fa83419be90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa838a00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa83406bbe0 0x7fa83419c3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa838a00700 1 -- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa83406b220 msgr2=0x7fa83419be90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa838a00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa83406b220 0x7fa83419be90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa838a00700 1 -- 172.21.15.28:0/236205846 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa8341a0ea0 con 0x7fa8341095b0 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa833e00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa83406b220 0x7fa83419be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.573+0000 7fa838a00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8341095b0 0x7fa8341a0760 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7fa81c00d980 tx=0x7fa81c00dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.574+0000 7fa830c00700 1 -- 172.21.15.28:0/236205846 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa81c016070 con 0x7fa8341095b0 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.574+0000 7fa830c00700 1 -- 172.21.15.28:0/236205846 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa81c005020 con 0x7fa8341095b0 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.574+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa8341a1190 con 0x7fa8341095b0 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.575+0000 7fa830c00700 1 -- 172.21.15.28:0/236205846 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa81c009ab0 con 0x7fa8341095b0 2024-05-20T20:16:46.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.575+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa8341a8a10 con 0x7fa8341095b0 2024-05-20T20:16:46.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.588+0000 7fa8452a2b80 1 Processor -- start 2024-05-20T20:16:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.600+0000 7fa830c00700 1 -- 172.21.15.28:0/236205846 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa81c009c50 con 0x7fa8341095b0 2024-05-20T20:16:46.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.600+0000 7fa8452a2b80 1 -- start start 2024-05-20T20:16:46.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6794e20 0x55a3c6795200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6795740 0x55a3c678f330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678fdb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a3c66e4e40 con 0x55a3c6794e20 2024-05-20T20:16:46.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a3c66d49e0 con 0x55a3c678f950 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a3c66d8a20 con 0x55a3c6795740 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa833400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6794e20 0x55a3c6795200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa833e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6795740 0x55a3c678f330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa833e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6795740 0x55a3c678f330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50020/0 (socket says 172.21.15.28:50020) 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.605+0000 7fa833e00700 1 -- 172.21.15.28:0/4000636836 learned_addr learned my addr 172.21.15.28:0/4000636836 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.606+0000 7fa833400700 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678fdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.607+0000 7fa830c00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8180c3ab0 0x7fa8180c5f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.607+0000 7fa830c00700 1 -- 172.21.15.28:0/236205846 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fa81c0e20b0 con 0x7fa8341095b0 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.612+0000 7fa833400700 1 -- 172.21.15.28:0/4000636836 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6795740 msgr2=0x55a3c678f330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.612+0000 7fa833400700 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6795740 0x55a3c678f330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.612+0000 7fa833400700 1 -- 172.21.15.28:0/4000636836 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 msgr2=0x55a3c678fdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.612+0000 7fa833400700 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678fdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.613+0000 7fa833400700 1 -- 172.21.15.28:0/4000636836 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a3c669d420 con 0x55a3c6794e20 2024-05-20T20:16:46.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.613+0000 7fa833e00700 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6795740 0x55a3c678f330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:46.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.614+0000 7fa833400700 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6794e20 0x55a3c6795200 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fa8240060b0 tx=0x7fa824007990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=584eb11acb6163fc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.618+0000 7fa829600700 1 -- 172.21.15.28:0/4000636836 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa82400a0d0 con 0x55a3c6794e20 2024-05-20T20:16:46.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.618+0000 7fa829600700 1 -- 172.21.15.28:0/4000636836 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa82400a9f0 con 0x55a3c6794e20 2024-05-20T20:16:46.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.618+0000 7fa829600700 1 -- 172.21.15.28:0/4000636836 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa824004020 con 0x55a3c6794e20 2024-05-20T20:16:46.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.619+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4000636836 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6794e20 msgr2=0x55a3c6795200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.619+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6794e20 0x55a3c6795200 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x7fa8240060b0 tx=0x7fa824007990 comp rx=0 tx=0).stop 2024-05-20T20:16:46.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.620+0000 7fa838a00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8180c3ab0 0x7fa8180c5f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.620 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.621+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4000636836 shutdown_connections 2024-05-20T20:16:46.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.621+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678fdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.621+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6794e20 0x55a3c6795200 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.621+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4000636836 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6795740 0x55a3c678f330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.621+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4000636836 >> 172.21.15.28:0/4000636836 conn(0x55a3c66b4be0 msgr2=0x55a3c6781690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.621+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4000636836 shutdown_connections 2024-05-20T20:16:46.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.621+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4000636836 wait complete. 2024-05-20T20:16:46.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.622+0000 7fa8452a2b80 1 -- start start 2024-05-20T20:16:46.630 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.623+0000 7fa8452a2b80 1 Processor -- start 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.628+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c678f950 0x55a3c6798d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.628+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6794e20 0x55a3c6799240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.628+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c6795740 0x55a3c6799780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.628+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a3c65dafa0 con 0x55a3c678f950 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.628+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a3c6675760 con 0x55a3c6795740 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.628+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a3c6786ca0 con 0x55a3c6794e20 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.628+0000 7fa8452a2b80 10 client.0 _ll_register_callbacks cb 0x55a3c667ec70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.629+0000 7fa833e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6794e20 0x55a3c6799240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.629+0000 7fa833e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6794e20 0x55a3c6799240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50030/0 (socket says 172.21.15.28:50030) 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.629+0000 7fa833e00700 1 -- 172.21.15.28:0/2495077618 learned_addr learned my addr 172.21.15.28:0/2495077618 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.631 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.629+0000 7fa838a00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8180c3ab0 0x7fa8180c5f70 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fa81c0a9310 tx=0x7fa81c005880 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.631+0000 7fa833400700 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c6795740 0x55a3c6799780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.631+0000 7fa833400700 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c678f950 0x55a3c6798d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.633+0000 7fa833e00700 1 -- 172.21.15.28:0/2495077618 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c6795740 msgr2=0x55a3c6799780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.633+0000 7fa833e00700 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c6795740 0x55a3c6799780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.633+0000 7fa833e00700 1 -- 172.21.15.28:0/2495077618 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c678f950 msgr2=0x55a3c6798d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.636 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.633+0000 7fa833e00700 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c678f950 0x55a3c6798d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.633+0000 7fa833e00700 1 -- 172.21.15.28:0/2495077618 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a3c66248d0 con 0x55a3c6794e20 2024-05-20T20:16:46.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.634+0000 7fa833e00700 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6794e20 0x55a3c6799240 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fa82c00bd10 tx=0x7fa82c00bdf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=543e150c1eb99d2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.637 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.634+0000 7fa822000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:16:46.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.637+0000 7fa822000700 1 -- 172.21.15.28:0/2495077618 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa82c00f660 con 0x55a3c6794e20 2024-05-20T20:16:46.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.637+0000 7fa822000700 1 -- 172.21.15.28:0/2495077618 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fa82c00f800 con 0x55a3c6794e20 2024-05-20T20:16:46.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.637+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa82400a9f0 con 0x55a3c6794e20 2024-05-20T20:16:46.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.638+0000 7fa822000700 1 -- 172.21.15.28:0/2495077618 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa82c030050 con 0x55a3c6794e20 2024-05-20T20:16:46.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.640+0000 7fa8452a2b80 10 client.27607 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:46.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.640+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a3c66a5060 con 0x55a3c6794e20 2024-05-20T20:16:46.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa822000700 1 -- 172.21.15.28:0/2495077618 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa82400a9f0 con 0x55a3c6794e20 2024-05-20T20:16:46.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa822000700 1 -- 172.21.15.28:0/2495077618 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa82c02b030 con 0x55a3c6794e20 2024-05-20T20:16:46.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa8452a2b80 10 client.27607 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:46.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa8452a2b80 10 client.27607 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:16:46.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa8452a2b80 20 client.27607 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:46.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c67a04f0 0x55a3c67a28e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa8452a2b80 4 client.27607 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:46.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.641+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a3c66e6d90 con 0x55a3c67a04f0 2024-05-20T20:16:46.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.642+0000 7fa833400700 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c67a04f0 0x55a3c67a28e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.643+0000 7fa822000700 10 client.27607 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:16:46.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.643+0000 7fa833400700 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c67a04f0 0x55a3c67a28e0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=b8285a2661ce8cb8 server_cookie=b66a2c258f4abba4 in_seq=0 out_seq=0 2024-05-20T20:16:46.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.644+0000 7fa822000700 1 -- 172.21.15.28:0/2495077618 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a3c66e6d90 con 0x55a3c67a04f0 2024-05-20T20:16:46.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.644+0000 7fa822000700 10 client.27607 handle_command_reply: tid=0 2024-05-20T20:16:46.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.644+0000 7fa8452a2b80 1 client.27607 shutdown 2024-05-20T20:16:46.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.644+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6794e20 msgr2=0x55a3c6799240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.644+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6794e20 0x55a3c6799240 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fa82c00bd10 tx=0x7fa82c00bdf0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.645 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.645+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 shutdown_connections 2024-05-20T20:16:46.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.645+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c6795740 0x55a3c6799780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.645+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c67a04f0 0x55a3c67a28e0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.645+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c678f950 0x55a3c6798d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.645+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/2495077618 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c6794e20 0x55a3c6799240 secure :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fa82c00bd10 tx=0x7fa82c00bdf0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.645+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 >> 172.21.15.28:0/2495077618 conn(0x55a3c66b4be0 msgr2=0x55a3c6780d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.648+0000 7fa822000700 0 client.27607 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:16:46.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.652+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 shutdown_connections 2024-05-20T20:16:46.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.652+0000 7fa8452a2b80 1 -- 172.21.15.28:0/2495077618 wait complete. 2024-05-20T20:16:46.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.652+0000 7fa8452a2b80 20 client.27607 trim_cache size 0 max 16384 2024-05-20T20:16:46.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.717+0000 7fa8452a2b80 1 Processor -- start 2024-05-20T20:16:46.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.717+0000 7fa8452a2b80 1 -- start start 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.717+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c678f950 0x55a3c6795200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 0x55a3c6797e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c679b310 0x55a3c67814f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341abde0 con 0x55a3c6795740 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341ad920 con 0x55a3c679b310 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341ac4b0 con 0x55a3c678f950 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa833400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c678f950 0x55a3c6795200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa833400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c678f950 0x55a3c6795200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50036/0 (socket says 172.21.15.28:50036) 2024-05-20T20:16:46.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa833400700 1 -- 172.21.15.28:0/4020157090 learned_addr learned my addr 172.21.15.28:0/4020157090 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.718+0000 7fa833400700 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c679b310 0x55a3c67814f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.721+0000 7fa833400700 1 -- 172.21.15.28:0/4020157090 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c679b310 msgr2=0x55a3c67814f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.721+0000 7fa833400700 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c679b310 0x55a3c67814f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.721+0000 7fa833400700 1 -- 172.21.15.28:0/4020157090 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 msgr2=0x55a3c6797e80 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:16:46.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.721+0000 7fa833400700 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 0x55a3c6797e80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.723 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.721+0000 7fa833400700 1 -- 172.21.15.28:0/4020157090 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a3c6781a30 con 0x55a3c678f950 2024-05-20T20:16:46.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.723+0000 7fa833400700 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c678f950 0x55a3c6795200 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fa82400ad00 tx=0x7fa82400a720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=27a05294fe9aca04 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.724+0000 7fa828c00700 1 -- 172.21.15.28:0/4020157090 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa824003a00 con 0x55a3c678f950 2024-05-20T20:16:46.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.725+0000 7fa828c00700 1 -- 172.21.15.28:0/4020157090 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa82402e040 con 0x55a3c678f950 2024-05-20T20:16:46.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.725+0000 7fa828c00700 1 -- 172.21.15.28:0/4020157090 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa824003a00 con 0x55a3c678f950 2024-05-20T20:16:46.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.726+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4020157090 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c678f950 msgr2=0x55a3c6795200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.726 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.726+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c678f950 0x55a3c6795200 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fa82400ad00 tx=0x7fa82400a720 comp rx=0 tx=0).stop 2024-05-20T20:16:46.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.726+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4020157090 shutdown_connections 2024-05-20T20:16:46.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.726+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c679b310 0x55a3c67814f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.726+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c678f950 0x55a3c6795200 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.727+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4020157090 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 0x55a3c6797e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.727+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4020157090 >> 172.21.15.28:0/4020157090 conn(0x55a3c66b4be0 msgr2=0x55a3c6798cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.727+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4020157090 shutdown_connections 2024-05-20T20:16:46.727 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.727+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4020157090 wait complete. 2024-05-20T20:16:46.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.728+0000 7fa8452a2b80 1 -- start start 2024-05-20T20:16:46.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.728+0000 7fa8452a2b80 1 Processor -- start 2024-05-20T20:16:46.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.732+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678d800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.732+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 0x55a3c6786f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.732+0000 7fa8452a2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c679b310 0x55a3c6787460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.732+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341ad250 con 0x55a3c6795740 2024-05-20T20:16:46.733 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.732+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341acb80 con 0x55a3c678f950 2024-05-20T20:16:46.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.732+0000 7fa8452a2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa8341a9df0 con 0x55a3c679b310 2024-05-20T20:16:46.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.732+0000 7fa8452a2b80 10 client.0 _ll_register_callbacks cb 0x55a3c667ec70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:46.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.733+0000 7fa833400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.733+0000 7fa833400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678d800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58414/0 (socket says 172.21.15.28:58414) 2024-05-20T20:16:46.734 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.733+0000 7fa833400700 1 -- 172.21.15.28:0/4289128779 learned_addr learned my addr 172.21.15.28:0/4289128779 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.735+0000 7fa833400700 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c679b310 0x55a3c6787460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.735+0000 7fa833400700 1 -- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c679b310 msgr2=0x55a3c6787460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.735+0000 7fa833400700 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c679b310 0x55a3c6787460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.735+0000 7fa833400700 1 -- 172.21.15.28:0/4289128779 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 msgr2=0x55a3c6786f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.735+0000 7fa833400700 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 0x55a3c6786f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.736 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.735+0000 7fa833400700 1 -- 172.21.15.28:0/4289128779 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a3c6787af0 con 0x55a3c678f950 2024-05-20T20:16:46.737 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.737+0000 7fa833400700 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678d800 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7fa824004d40 tx=0x7fa82400a0d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f3a5029cde0ce470 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.737+0000 7fa822000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:46.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.738+0000 7fa822000700 1 -- 172.21.15.28:0/4289128779 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa8240188b0 con 0x55a3c678f950 2024-05-20T20:16:46.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.738+0000 7fa822000700 1 -- 172.21.15.28:0/4289128779 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fa824018a50 con 0x55a3c678f950 2024-05-20T20:16:46.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.738+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa82402e040 con 0x55a3c678f950 2024-05-20T20:16:46.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.739+0000 7fa822000700 1 -- 172.21.15.28:0/4289128779 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa824018c50 con 0x55a3c678f950 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.739+0000 7fa8452a2b80 10 client.27671 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.739+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a3c680b080 con 0x55a3c678f950 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.739+0000 7fa822000700 1 -- 172.21.15.28:0/4289128779 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa82402e040 con 0x55a3c678f950 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.740+0000 7fa822000700 1 -- 172.21.15.28:0/4289128779 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa824019040 con 0x55a3c678f950 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.740+0000 7fa8452a2b80 10 client.27671 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.740+0000 7fa8452a2b80 10 client.27671 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.740+0000 7fa8452a2b80 20 client.27671 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.740+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c680b3e0 0x55a3c680d800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.741 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.741+0000 7fa8452a2b80 4 client.27671 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:16:46.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.741+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55a3c66e0570 con 0x55a3c680b3e0 2024-05-20T20:16:46.742 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.741+0000 7fa833e00700 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c680b3e0 0x55a3c680d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.742+0000 7fa822000700 10 client.27671 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:16:46.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.744+0000 7fa833e00700 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c680b3e0 0x55a3c680d800 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=6cde0cdd18c6c88 server_cookie=a12b3467b250ceed in_seq=0 out_seq=0 2024-05-20T20:16:46.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.749+0000 7fa822000700 1 -- 172.21.15.28:0/4289128779 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55a3c66e0570 con 0x55a3c680b3e0 2024-05-20T20:16:46.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.749+0000 7fa822000700 10 client.27671 handle_command_reply: tid=0 2024-05-20T20:16:46.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.750+0000 7fa8452a2b80 1 client.27671 shutdown 2024-05-20T20:16:46.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 msgr2=0x55a3c678d800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678d800 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7fa824004d40 tx=0x7fa82400a0d0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 shutdown_connections 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a3c679b310 0x55a3c6787460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a3c678f950 0x55a3c678d800 secure :-1 s=CLOSED pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7fa824004d40 tx=0x7fa82400a0d0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a3c680b3e0 0x55a3c680d800 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 --2- 172.21.15.28:0/4289128779 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a3c6795740 0x55a3c6786f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 >> 172.21.15.28:0/4289128779 conn(0x55a3c66b4be0 msgr2=0x55a3c679ab90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.751+0000 7fa822000700 0 client.27671 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.752+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 shutdown_connections 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.752+0000 7fa8452a2b80 1 -- 172.21.15.28:0/4289128779 wait complete. 2024-05-20T20:16:46.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.754+0000 7fa8452a2b80 20 client.27671 trim_cache size 0 max 16384 2024-05-20T20:16:46.758 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8180c3ab0 msgr2=0x7fa8180c5f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8180c3ab0 0x7fa8180c5f70 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fa81c0a9310 tx=0x7fa81c005880 comp rx=0 tx=0).stop 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8341095b0 msgr2=0x7fa8341a0760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8341095b0 0x7fa8341a0760 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7fa81c00d980 tx=0x7fa81c00dc90 comp rx=0 tx=0).stop 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 shutdown_connections 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa8180c3ab0 0x7fa8180c5f70 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa8341095b0 0x7fa8341a0760 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa83406b220 0x7fa83419be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 --2- 172.21.15.28:0/236205846 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa83406bbe0 0x7fa83419c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 >> 172.21.15.28:0/236205846 conn(0x7fa8340793f0 msgr2=0x7fa834100f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.762+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 shutdown_connections 2024-05-20T20:16:46.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:46.763+0000 7fa83aa00700 1 -- 172.21.15.28:0/236205846 wait complete. 2024-05-20T20:16:46.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.772+0000 7fbf33200640 1 -- 172.21.15.28:0/2789453328 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf34104a90 msgr2=0x7fbf34104f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.772 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.772+0000 7fbf33200640 1 --2- 172.21.15.28:0/2789453328 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf34104a90 0x7fbf34104f10 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7fbf2c009ca0 tx=0x7fbf2c02a8b0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.774+0000 7fbf33200640 1 -- 172.21.15.28:0/2789453328 shutdown_connections 2024-05-20T20:16:46.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.774+0000 7fbf33200640 1 --2- 172.21.15.28:0/2789453328 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf34104a90 0x7fbf34104f10 unknown :-1 s=CLOSED pgs=552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.774+0000 7fbf33200640 1 --2- 172.21.15.28:0/2789453328 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf34105450 0x7fbf34111960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.774+0000 7fbf33200640 1 --2- 172.21.15.28:0/2789453328 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf3410aaa0 0x7fbf3410ae80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.774 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.774+0000 7fbf33200640 1 -- 172.21.15.28:0/2789453328 >> 172.21.15.28:0/2789453328 conn(0x7fbf34100560 msgr2=0x7fbf34102980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.775 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.775+0000 7fbf33200640 1 -- 172.21.15.28:0/2789453328 shutdown_connections 2024-05-20T20:16:46.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.779+0000 7fbf33200640 1 -- 172.21.15.28:0/2789453328 wait complete. 2024-05-20T20:16:46.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.781+0000 7fbf33200640 1 Processor -- start 2024-05-20T20:16:46.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.781+0000 7fbf33200640 1 -- start start 2024-05-20T20:16:46.781 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.781+0000 7fbf33200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf34104a90 0x7fbf340742d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.781+0000 7fbf33200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf34105450 0x7fbf3406d3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.781+0000 7fbf33200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf3410aaa0 0x7fbf3406d8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.782+0000 7fbf33200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbf34114e30 con 0x7fbf34104a90 2024-05-20T20:16:46.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.782+0000 7fbf33200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fbf34114cb0 con 0x7fbf3410aaa0 2024-05-20T20:16:46.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.782+0000 7fbf33200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fbf34114fb0 con 0x7fbf34105450 2024-05-20T20:16:46.782 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.782+0000 7fbf31400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf34105450 0x7fbf3406d3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.782+0000 7fbf31400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf34105450 0x7fbf3406d3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:50056/0 (socket says 172.21.15.28:50056) 2024-05-20T20:16:46.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.782+0000 7fbf31400640 1 -- 172.21.15.28:0/2108213454 learned_addr learned my addr 172.21.15.28:0/2108213454 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.782+0000 7fbf32800640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf3410aaa0 0x7fbf3406d8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.783+0000 7fbf32800640 1 -- 172.21.15.28:0/2108213454 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf34105450 msgr2=0x7fbf3406d3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.784 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.783+0000 7fbf32800640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf34105450 0x7fbf3406d3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.783+0000 7fbf32800640 1 -- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf34104a90 msgr2=0x7fbf340742d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.783+0000 7fbf32800640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf34104a90 0x7fbf340742d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.783+0000 7fbf32800640 1 -- 172.21.15.28:0/2108213454 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbf3406df20 con 0x7fbf3410aaa0 2024-05-20T20:16:46.785 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.784+0000 7fbf32800640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf3410aaa0 0x7fbf3406d8e0 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7fbf2400b440 tx=0x7fbf2400b900 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.784+0000 7fbf2aa00640 1 -- 172.21.15.28:0/2108213454 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf24015020 con 0x7fbf3410aaa0 2024-05-20T20:16:46.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.784+0000 7fbf2aa00640 1 -- 172.21.15.28:0/2108213454 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbf24011ce0 con 0x7fbf3410aaa0 2024-05-20T20:16:46.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.784+0000 7fbf2aa00640 1 -- 172.21.15.28:0/2108213454 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf24010d80 con 0x7fbf3410aaa0 2024-05-20T20:16:46.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.785+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbf34071df0 con 0x7fbf3410aaa0 2024-05-20T20:16:46.786 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.785+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbf340722f0 con 0x7fbf3410aaa0 2024-05-20T20:16:46.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.802+0000 7fbf2aa00640 1 -- 172.21.15.28:0/2108213454 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fbf24018020 con 0x7fbf3410aaa0 2024-05-20T20:16:46.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.803+0000 7fbf3bbba740 1 Processor -- start 2024-05-20T20:16:46.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.805+0000 7fbf3bbba740 1 -- start start 2024-05-20T20:16:46.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.806+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98e540 0x55752d98e920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.806+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.807+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9a46c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.807+0000 7fbf3bbba740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55752d8f6b90 con 0x55752d98ee60 2024-05-20T20:16:46.810 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.807+0000 7fbf3bbba740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55752d8e6730 con 0x55752d9a22d0 2024-05-20T20:16:46.810 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.807+0000 7fbf3bbba740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55752d8ea770 con 0x55752d98e540 2024-05-20T20:16:46.811 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.811+0000 7fbf2aa00640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbf180b2aa0 0x7fbf180b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.811 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.812+0000 7fbf2aa00640 1 -- 172.21.15.28:0/2108213454 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fbf240e1d10 con 0x7fbf3410aaa0 2024-05-20T20:16:46.813 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.812+0000 7fbf31e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.813 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.812+0000 7fbf31e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:49768/0 (socket says 172.21.15.28:49768) 2024-05-20T20:16:46.813 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.812+0000 7fbf31e00640 1 -- 172.21.15.28:0/884392462 learned_addr learned my addr 172.21.15.28:0/884392462 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.813 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.813+0000 7fbf32800640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbf180b2aa0 0x7fbf180b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.814+0000 7fbf31e00640 1 -- 172.21.15.28:0/884392462 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98e540 msgr2=0x55752d98e920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.815+0000 7fbf31e00640 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98e540 0x55752d98e920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.818 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.815+0000 7fbf31e00640 1 -- 172.21.15.28:0/884392462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 msgr2=0x55752d9a46c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.814+0000 7fbf31400640 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9a46c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.818+0000 7fbf31e00640 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9a46c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.818+0000 7fbf31e00640 1 -- 172.21.15.28:0/884392462 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55752d87f8b0 con 0x55752d98ee60 2024-05-20T20:16:46.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.818+0000 7fbf31400640 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9a46c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:46.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.819+0000 7fbf31e00640 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989d90 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fbf20009820 tx=0x7fbf20009ce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6f8946d67ceff828 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.822+0000 7fbf1f000640 1 -- 172.21.15.28:0/884392462 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf20015020 con 0x55752d98ee60 2024-05-20T20:16:46.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.822+0000 7fbf1f000640 1 -- 172.21.15.28:0/884392462 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbf20011420 con 0x55752d98ee60 2024-05-20T20:16:46.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.822+0000 7fbf1f000640 1 -- 172.21.15.28:0/884392462 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf200105e0 con 0x55752d98ee60 2024-05-20T20:16:46.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.824+0000 7fbf3bbba740 1 -- 172.21.15.28:0/884392462 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 msgr2=0x55752d989d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.825 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.824+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989d90 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fbf20009820 tx=0x7fbf20009ce0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.825+0000 7fbf32800640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbf180b2aa0 0x7fbf180b4f60 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7fbf240df560 tx=0x7fbf240106e0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.827+0000 7fbf3bbba740 1 -- 172.21.15.28:0/884392462 shutdown_connections 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.827+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9a46c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.827+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989d90 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.827+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/884392462 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98e540 0x55752d98e920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.827+0000 7fbf3bbba740 1 -- 172.21.15.28:0/884392462 >> 172.21.15.28:0/884392462 conn(0x55752d98ab90 msgr2=0x55752d98af70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.827+0000 7fbf3bbba740 1 -- 172.21.15.28:0/884392462 shutdown_connections 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.827+0000 7fbf3bbba740 1 -- 172.21.15.28:0/884392462 wait complete. 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.828+0000 7fbf3bbba740 1 -- start start 2024-05-20T20:16:46.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.828+0000 7fbf3bbba740 1 Processor -- start 2024-05-20T20:16:46.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.831+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98e540 0x55752d989ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.832 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.832+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98ee60 0x55752d986fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.832+0000 7fbf31400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98e540 0x55752d989ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.832+0000 7fbf31400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98e540 0x55752d989ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:49776/0 (socket says 172.21.15.28:49776) 2024-05-20T20:16:46.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.833+0000 7fbf31400640 1 -- 172.21.15.28:0/898382753 learned_addr learned my addr 172.21.15.28:0/898382753 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.833+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9875e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.833+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55752d5cc790 con 0x55752d98e540 2024-05-20T20:16:46.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.833+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55752d88ebc0 con 0x55752d9a22d0 2024-05-20T20:16:46.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.833+0000 7fbf31e00640 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98ee60 0x55752d986fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.835 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.833+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55752d9a4c60 con 0x55752d98ee60 2024-05-20T20:16:46.835 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.835+0000 7fbf3bbba740 10 client.0 _ll_register_callbacks cb 0x55752d86c400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:46.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.835+0000 7fbf31e00640 1 -- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 msgr2=0x55752d9875e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.835+0000 7fbf31e00640 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9875e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.835+0000 7fbf31e00640 1 -- 172.21.15.28:0/898382753 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98e540 msgr2=0x55752d989ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.835+0000 7fbf31e00640 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98e540 0x55752d989ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.836 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.835+0000 7fbf31e00640 1 -- 172.21.15.28:0/898382753 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55752d5fb210 con 0x55752d98ee60 2024-05-20T20:16:46.837 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.837+0000 7fbf31e00640 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98ee60 0x55752d986fd0 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fbf20011da0 tx=0x7fbf20009560 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5baa3ec29d5551fe server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.837 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.837+0000 7fbf13400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:16:46.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.837+0000 7fbf13400640 1 -- 172.21.15.28:0/898382753 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf20027060 con 0x55752d98ee60 2024-05-20T20:16:46.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.838+0000 7fbf13400640 1 -- 172.21.15.28:0/898382753 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fbf2000b8d0 con 0x55752d98ee60 2024-05-20T20:16:46.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.840+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbf20011420 con 0x55752d98ee60 2024-05-20T20:16:46.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.840+0000 7fbf13400640 1 -- 172.21.15.28:0/898382753 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fbf2000bad0 con 0x55752d98ee60 2024-05-20T20:16:46.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.840+0000 7fbf3bbba740 10 client.27619 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:46.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.840+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55752d6c7620 con 0x55752d98ee60 2024-05-20T20:16:46.843 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.841+0000 7fbf13400640 1 -- 172.21.15.28:0/898382753 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbf20011420 con 0x55752d98ee60 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.842+0000 7fbf13400640 1 -- 172.21.15.28:0/898382753 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fbf2001ab80 con 0x55752d98ee60 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.843+0000 7fbf3bbba740 10 client.27619 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.843+0000 7fbf3bbba740 10 client.27619 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.843+0000 7fbf3bbba740 20 client.27619 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.843+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752d9b1be0 0x55752d9b4020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.843+0000 7fbf3bbba740 4 client.27619 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.843+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55752d8f8ae0 con 0x55752d9b1be0 2024-05-20T20:16:46.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.843+0000 7fbf31400640 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752d9b1be0 0x55752d9b4020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.845 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.844+0000 7fbf31400640 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752d9b1be0 0x55752d9b4020 crc :-1 s=READY pgs=233 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b47eba9986214334 server_cookie=8acab0add7027635 in_seq=0 out_seq=0 2024-05-20T20:16:46.845 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.844+0000 7fbf13400640 10 client.27619 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:46.845 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.845+0000 7fbf13400640 1 -- 172.21.15.28:0/898382753 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55752d8f8ae0 con 0x55752d9b1be0 2024-05-20T20:16:46.845 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.845+0000 7fbf13400640 10 client.27619 handle_command_reply: tid=0 2024-05-20T20:16:46.845 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.845+0000 7fbf3bbba740 1 client.27619 shutdown 2024-05-20T20:16:46.846 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.845+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98ee60 msgr2=0x55752d986fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.846 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.845+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98ee60 0x55752d986fd0 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fbf20011da0 tx=0x7fbf20009560 comp rx=0 tx=0).stop 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 shutdown_connections 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752d9b1be0 0x55752d9b4020 crc :-1 s=READY pgs=233 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d9a22d0 0x55752d9875e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d98ee60 0x55752d986fd0 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/898382753 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98e540 0x55752d989ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 >> 172.21.15.28:0/898382753 conn(0x55752d98ab90 msgr2=0x55752d990930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf13400640 0 client.27619 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 shutdown_connections 2024-05-20T20:16:46.848 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.847+0000 7fbf3bbba740 1 -- 172.21.15.28:0/898382753 wait complete. 2024-05-20T20:16:46.849 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.849+0000 7fbf3bbba740 20 client.27619 trim_cache size 0 max 16384 2024-05-20T20:16:46.922 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.920+0000 7fbf3bbba740 1 Processor -- start 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.921+0000 7fbf3bbba740 1 -- start start 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.921+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752d9813b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.921+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.922+0000 7fbf31e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.922+0000 7fbf31e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:55034/0 (socket says 172.21.15.28:55034) 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.922+0000 7fbf31e00640 1 -- 172.21.15.28:0/1673793062 learned_addr learned my addr 172.21.15.28:0/1673793062 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.922+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9907b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.923+0000 7fbf31400640 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752d9813b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.923 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.923+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbf341b4c20 con 0x55752d98ee60 2024-05-20T20:16:46.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.923+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fbf341b6760 con 0x55752d98e540 2024-05-20T20:16:46.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.923+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fbf341b52f0 con 0x55752d9a22d0 2024-05-20T20:16:46.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.924+0000 7fbf31400640 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9907b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.924 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.924+0000 7fbf31400640 1 -- 172.21.15.28:0/1673793062 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 msgr2=0x55752d9907b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.924+0000 7fbf31400640 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9907b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.924+0000 7fbf31400640 1 -- 172.21.15.28:0/1673793062 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 msgr2=0x55752d989810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.924+0000 7fbf31400640 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.925+0000 7fbf31400640 1 -- 172.21.15.28:0/1673793062 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55752d987910 con 0x55752d98e540 2024-05-20T20:16:46.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.925+0000 7fbf31e00640 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:46.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.926+0000 7fbf31400640 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752d9813b0 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7fbf2c005fe0 tx=0x7fbf2c030d40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=888a5252401e194d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.926+0000 7fbf1e600640 1 -- 172.21.15.28:0/1673793062 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf2c003970 con 0x55752d98e540 2024-05-20T20:16:46.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.927+0000 7fbf1e600640 1 -- 172.21.15.28:0/1673793062 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbf2c036040 con 0x55752d98e540 2024-05-20T20:16:46.927 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.927+0000 7fbf1e600640 1 -- 172.21.15.28:0/1673793062 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf2c0046e0 con 0x55752d98e540 2024-05-20T20:16:46.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.928+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 msgr2=0x55752d9813b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.928+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752d9813b0 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7fbf2c005fe0 tx=0x7fbf2c030d40 comp rx=0 tx=0).stop 2024-05-20T20:16:46.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.928+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 shutdown_connections 2024-05-20T20:16:46.928 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.928+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9907b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.929 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.929+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752d989810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.929 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.929+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/1673793062 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752d9813b0 unknown :-1 s=CLOSED pgs=554 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.929 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.929+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 >> 172.21.15.28:0/1673793062 conn(0x55752d98ab90 msgr2=0x55752d9900a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.929 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.929+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 shutdown_connections 2024-05-20T20:16:46.929 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.929+0000 7fbf3bbba740 1 -- 172.21.15.28:0/1673793062 wait complete. 2024-05-20T20:16:46.930 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.930+0000 7fbf3bbba740 1 -- start start 2024-05-20T20:16:46.930 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.930+0000 7fbf3bbba740 1 Processor -- start 2024-05-20T20:16:46.931 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.932+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752da14b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.931 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.932+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752da15040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.932 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.932+0000 7fbf31400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752da14b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.932 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.932+0000 7fbf31400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752da14b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51472/0 (socket says 172.21.15.28:51472) 2024-05-20T20:16:46.932 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.932+0000 7fbf3bbba740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9a54b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.932 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.933+0000 7fbf3bbba740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbf341b6090 con 0x55752d98ee60 2024-05-20T20:16:46.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.933+0000 7fbf3bbba740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fbf341b59c0 con 0x55752d98e540 2024-05-20T20:16:46.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.933+0000 7fbf3bbba740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fbf3407b0e0 con 0x55752d9a22d0 2024-05-20T20:16:46.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.933+0000 7fbf3bbba740 10 client.0 _ll_register_callbacks cb 0x7fbf20009670 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:46.933 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.933+0000 7fbf31400640 1 -- 172.21.15.28:0/835608626 learned_addr learned my addr 172.21.15.28:0/835608626 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:46.934 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.934+0000 7fbf31400640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9a54b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.934 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.934+0000 7fbf31400640 1 -- 172.21.15.28:0/835608626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 msgr2=0x55752d9a54b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.934 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.934+0000 7fbf31400640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9a54b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.934 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.934+0000 7fbf31400640 1 -- 172.21.15.28:0/835608626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 msgr2=0x55752da15040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:46.934 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.934+0000 7fbf31e00640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752da15040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.934 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.934+0000 7fbf31400640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752da15040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.935 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.934+0000 7fbf31400640 1 -- 172.21.15.28:0/835608626 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55752d9a5ba0 con 0x55752d98e540 2024-05-20T20:16:46.935 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.935+0000 7fbf31e00640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752da15040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:46.935 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.936+0000 7fbf13400640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:46.935 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.936+0000 7fbf31400640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752da14b00 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7fbf2c030bd0 tx=0x7fbf2c030db0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=524f73c473365f54 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:46.936 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.936+0000 7fbf13400640 1 -- 172.21.15.28:0/835608626 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf2c057020 con 0x55752d98e540 2024-05-20T20:16:46.936 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.936+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbf2c036040 con 0x55752d98e540 2024-05-20T20:16:46.937 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.937+0000 7fbf13400640 1 -- 172.21.15.28:0/835608626 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7fbf2c004520 con 0x55752d98e540 2024-05-20T20:16:46.938 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.938+0000 7fbf13400640 1 -- 172.21.15.28:0/835608626 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fbf2c057020 con 0x55752d98e540 2024-05-20T20:16:46.938 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.938+0000 7fbf3bbba740 10 client.27683 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:46.938 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.938+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55752da1c2b0 con 0x55752d98e540 2024-05-20T20:16:46.940 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.939+0000 7fbf13400640 1 -- 172.21.15.28:0/835608626 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbf2c036040 con 0x55752d98e540 2024-05-20T20:16:46.940 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.941+0000 7fbf13400640 1 -- 172.21.15.28:0/835608626 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fbf2c003970 con 0x55752d98e540 2024-05-20T20:16:46.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.941+0000 7fbf3bbba740 10 client.27683 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:46.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.941+0000 7fbf3bbba740 10 client.27683 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:46.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.941+0000 7fbf3bbba740 20 client.27683 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:46.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.941+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752da1c8d0 0x55752da1ecf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:46.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.941+0000 7fbf3bbba740 4 client.27683 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:16:46.941 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.941+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55752d8f22c0 con 0x55752da1c8d0 2024-05-20T20:16:46.942 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.942+0000 7fbf31e00640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752da1c8d0 0x55752da1ecf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:46.943 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.943+0000 7fbf13400640 10 client.27683 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:46.943 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.943+0000 7fbf31e00640 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752da1c8d0 0x55752da1ecf0 crc :-1 s=READY pgs=234 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d991c9387e1f837e server_cookie=e127b30d661d21d8 in_seq=0 out_seq=0 2024-05-20T20:16:46.983 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.983+0000 7fbf13400640 1 -- 172.21.15.28:0/835608626 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55752d8f22c0 con 0x55752da1c8d0 2024-05-20T20:16:46.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.983+0000 7fbf13400640 10 client.27683 handle_command_reply: tid=0 2024-05-20T20:16:46.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.983+0000 7fbf3bbba740 1 client.27683 shutdown 2024-05-20T20:16:46.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.983+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 msgr2=0x55752da14b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.983+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752da14b00 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7fbf2c030bd0 tx=0x7fbf2c030db0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 shutdown_connections 2024-05-20T20:16:46.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55752da1c8d0 0x55752da1ecf0 crc :-1 s=READY pgs=234 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55752d9a22d0 0x55752d9a54b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55752d98ee60 0x55752da15040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf3bbba740 1 --2- 172.21.15.28:0/835608626 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55752d98e540 0x55752da14b00 unknown :-1 s=CLOSED pgs=555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 >> 172.21.15.28:0/835608626 conn(0x55752d98ab90 msgr2=0x55752d990880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf13400640 0 client.27683 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:46.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.984+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 shutdown_connections 2024-05-20T20:16:46.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.985+0000 7fbf3bbba740 1 -- 172.21.15.28:0/835608626 wait complete. 2024-05-20T20:16:46.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.985+0000 7fbf3bbba740 20 client.27683 trim_cache size 0 max 16384 2024-05-20T20:16:46.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:16:46.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:16:46.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbf180b2aa0 msgr2=0x7fbf180b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbf180b2aa0 0x7fbf180b4f60 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7fbf240df560 tx=0x7fbf240106e0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf3410aaa0 msgr2=0x7fbf3406d8e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf3410aaa0 0x7fbf3406d8e0 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7fbf2400b440 tx=0x7fbf2400b900 comp rx=0 tx=0).stop 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 shutdown_connections 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbf3410aaa0 0x7fbf3406d8e0 unknown :-1 s=CLOSED pgs=553 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbf180b2aa0 0x7fbf180b4f60 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf34105450 0x7fbf3406d3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.993 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 --2- 172.21.15.28:0/2108213454 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbf34104a90 0x7fbf340742d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:46.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 >> 172.21.15.28:0/2108213454 conn(0x7fbf34100560 msgr2=0x7fbf34100db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:46.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 shutdown_connections 2024-05-20T20:16:46.994 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:16:46.992+0000 7fbf33200640 1 -- 172.21.15.28:0/2108213454 wait complete. 2024-05-20T20:16:47.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:47 smithi154 ceph-mon[36314]: pgmap v347: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 338 MiB/s wr, 2.34k op/s 2024-05-20T20:16:47.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:47 smithi033 ceph-mon[36179]: pgmap v347: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 338 MiB/s wr, 2.34k op/s 2024-05-20T20:16:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:47 smithi028 ceph-mon[31178]: pgmap v347: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 338 MiB/s wr, 2.34k op/s 2024-05-20T20:16:48.117 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:16:48.965 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:49.441 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:49 smithi028 ceph-mon[31178]: pgmap v348: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 344 MiB/s wr, 2.41k op/s 2024-05-20T20:16:49.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:49 smithi154 ceph-mon[36314]: pgmap v348: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 344 MiB/s wr, 2.41k op/s 2024-05-20T20:16:49.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:49 smithi033 ceph-mon[36179]: pgmap v348: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 344 MiB/s wr, 2.41k op/s 2024-05-20T20:16:51.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:51 smithi028 ceph-mon[31178]: pgmap v349: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 397 MiB/s wr, 2.91k op/s 2024-05-20T20:16:51.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:51 smithi154 ceph-mon[36314]: pgmap v349: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 397 MiB/s wr, 2.91k op/s 2024-05-20T20:16:51.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:51 smithi033 ceph-mon[36179]: pgmap v349: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 397 MiB/s wr, 2.91k op/s 2024-05-20T20:16:53.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.368+0000 7f6fa5200700 1 -- 172.21.15.28:0/1039813605 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0108470 msgr2=0x7f6fa006b250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.369+0000 7f6fa5200700 1 --2- 172.21.15.28:0/1039813605 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0108470 0x7f6fa006b250 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c009990 tx=0x7f6f8c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.369+0000 7f6fa5200700 1 -- 172.21.15.28:0/1039813605 shutdown_connections 2024-05-20T20:16:53.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.369+0000 7f6fa5200700 1 --2- 172.21.15.28:0/1039813605 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6fa0105f80 0x7f6fa0106410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.369+0000 7f6fa5200700 1 --2- 172.21.15.28:0/1039813605 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0108470 0x7f6fa006b250 unknown :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.369+0000 7f6fa5200700 1 --2- 172.21.15.28:0/1039813605 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6fa006b790 0x7f6fa006bbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.370+0000 7f6fa5200700 1 -- 172.21.15.28:0/1039813605 >> 172.21.15.28:0/1039813605 conn(0x7f6fa00793f0 msgr2=0x7f6fa0079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:53.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.370+0000 7f6fa5200700 1 -- 172.21.15.28:0/1039813605 shutdown_connections 2024-05-20T20:16:53.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.370+0000 7f6fa5200700 1 -- 172.21.15.28:0/1039813605 wait complete. 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.372+0000 7f6fa5200700 1 Processor -- start 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.372+0000 7f6fa5200700 1 -- start start 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.373+0000 7f6fa5200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6fa006b790 0x7f6fa01a0200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.373+0000 7f6fa5200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0105f80 0x7f6fa01a0740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.373+0000 7f6fa5200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6fa0108470 0x7f6fa01a4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.373+0000 7f6fa5200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa0119d50 con 0x7f6fa0108470 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.373+0000 7f6fa5200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa0119bd0 con 0x7f6fa0105f80 2024-05-20T20:16:53.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.373+0000 7f6fa5200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa0119ed0 con 0x7f6fa006b790 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.374+0000 7f6f9d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0105f80 0x7f6fa01a0740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.374+0000 7f6f9d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0105f80 0x7f6fa01a0740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51494/0 (socket says 172.21.15.28:51494) 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.374+0000 7f6f9d400700 1 -- 172.21.15.28:0/2686024024 learned_addr learned my addr 172.21.15.28:0/2686024024 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.374+0000 7f6f9de00700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6fa006b790 0x7f6fa01a0200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.375+0000 7f6f9de00700 1 -- 172.21.15.28:0/2686024024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0105f80 msgr2=0x7f6fa01a0740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.375+0000 7f6f9de00700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0105f80 0x7f6fa01a0740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.375+0000 7f6f9de00700 1 -- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6fa0108470 msgr2=0x7f6fa01a4ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.375+0000 7f6f9de00700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6fa0108470 0x7f6fa01a4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.378 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.375+0000 7f6f9de00700 1 -- 172.21.15.28:0/2686024024 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6fa01a5210 con 0x7f6fa006b790 2024-05-20T20:16:53.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.382+0000 7f6f9de00700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6fa006b790 0x7f6fa01a0200 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c006010 tx=0x7f6f8c0057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:53.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.383+0000 7f6f96a00700 1 -- 172.21.15.28:0/2686024024 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c026070 con 0x7f6fa006b790 2024-05-20T20:16:53.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.383+0000 7f6f96a00700 1 -- 172.21.15.28:0/2686024024 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6f8c021410 con 0x7f6fa006b790 2024-05-20T20:16:53.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.383+0000 7f6f96a00700 1 -- 172.21.15.28:0/2686024024 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c017650 con 0x7f6fa006b790 2024-05-20T20:16:53.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.384+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6fa01a54a0 con 0x7f6fa006b790 2024-05-20T20:16:53.384 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.384+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6fa006cf60 con 0x7f6fa006b790 2024-05-20T20:16:53.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.391+0000 7f6f96a00700 1 -- 172.21.15.28:0/2686024024 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f6f8c009d10 con 0x7f6fa006b790 2024-05-20T20:16:53.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.392+0000 7f6fafaa0b80 1 Processor -- start 2024-05-20T20:16:53.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.392+0000 7f6fafaa0b80 1 -- start start 2024-05-20T20:16:53.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.393+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b5398e290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.393+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53988260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.393+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53988ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.393+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558b538e6e40 con 0x558b5398deb0 2024-05-20T20:16:53.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.393+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558b538d69e0 con 0x558b5398e7d0 2024-05-20T20:16:53.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.393+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558b538daa20 con 0x558b53988880 2024-05-20T20:16:53.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.394+0000 7f6f9de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53988260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.394 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.394+0000 7f6f9de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53988260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51510/0 (socket says 172.21.15.28:51510) 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.394+0000 7f6f9de00700 1 -- 172.21.15.28:0/2243122500 learned_addr learned my addr 172.21.15.28:0/2243122500 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.394+0000 7f6f9d400700 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53988ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.395+0000 7f6f9e800700 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b5398e290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.395+0000 7f6f9de00700 1 -- 172.21.15.28:0/2243122500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 msgr2=0x558b53988ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.395+0000 7f6f9de00700 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53988ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.395+0000 7f6f9de00700 1 -- 172.21.15.28:0/2243122500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 msgr2=0x558b5398e290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.395+0000 7f6f9de00700 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b5398e290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.395+0000 7f6f9de00700 1 -- 172.21.15.28:0/2243122500 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558b5389f420 con 0x558b5398e7d0 2024-05-20T20:16:53.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.396+0000 7f6f9de00700 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53988260 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c0e01a0 tx=0x7f6f8c0e6b80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=35ad0b6890dc8010 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:53.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.397+0000 7f6f8b400700 1 -- 172.21.15.28:0/2243122500 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c0f6070 con 0x558b5398e7d0 2024-05-20T20:16:53.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.397+0000 7f6f8b400700 1 -- 172.21.15.28:0/2243122500 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6f8c0ebc90 con 0x558b5398e7d0 2024-05-20T20:16:53.397 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.397+0000 7f6f8b400700 1 -- 172.21.15.28:0/2243122500 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c0f6070 con 0x558b5398e7d0 2024-05-20T20:16:53.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.397+0000 7f6f96a00700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f840b2aa0 0x7f6f840b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.398+0000 7f6f96a00700 1 -- 172.21.15.28:0/2686024024 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f6f8c0e29e0 con 0x7f6fa006b790 2024-05-20T20:16:53.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.398+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/2243122500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 msgr2=0x558b53988260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.398+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53988260 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c0e01a0 tx=0x7f6f8c0e6b80 comp rx=0 tx=0).stop 2024-05-20T20:16:53.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.399+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/2243122500 shutdown_connections 2024-05-20T20:16:53.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.399+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53988ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.399+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b5398e290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.399+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/2243122500 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53988260 unknown :-1 s=CLOSED pgs=557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.399+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/2243122500 >> 172.21.15.28:0/2243122500 conn(0x558b538b6be0 msgr2=0x558b53983010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:53.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.400+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/2243122500 shutdown_connections 2024-05-20T20:16:53.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.400+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/2243122500 wait complete. 2024-05-20T20:16:53.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.400+0000 7f6fafaa0b80 1 -- start start 2024-05-20T20:16:53.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.401+0000 7f6fafaa0b80 1 Processor -- start 2024-05-20T20:16:53.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.401+0000 7f6f9e800700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f840b2aa0 0x7f6f840b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.402+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53994510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.402+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539918b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.402+0000 7f6f9de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53994510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.402+0000 7f6f9de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53994510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56988/0 (socket says 172.21.15.28:56988) 2024-05-20T20:16:53.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.402+0000 7f6f9de00700 1 -- 172.21.15.28:0/3977807499 learned_addr learned my addr 172.21.15.28:0/3977807499 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:53.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.403+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53991df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.403+0000 7f6f9e800700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f840b2aa0 0x7f6f840b4f60 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f6f98009360 tx=0x7f6f9800d040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:53.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.403+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x558b537dcfa0 con 0x558b5398deb0 2024-05-20T20:16:53.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.403+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x558b53877760 con 0x558b5398e7d0 2024-05-20T20:16:53.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.403+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x558b538d69e0 con 0x558b53988880 2024-05-20T20:16:53.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.403+0000 7f6f9d400700 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539918b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.403+0000 7f6fafaa0b80 10 client.0 _ll_register_callbacks cb 0x558b53880c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:53.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f9d400700 1 -- 172.21.15.28:0/3977807499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 msgr2=0x558b53994510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f9d400700 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53994510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f9d400700 1 -- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 msgr2=0x558b53991df0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f9d400700 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53991df0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f9d400700 1 -- 172.21.15.28:0/3977807499 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558b538268d0 con 0x558b5398deb0 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f9de00700 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53994510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f9d400700 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539918b0 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f6f90005950 tx=0x7f6f90011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b73c104333b20728 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.404+0000 7f6f7fe00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.405+0000 7f6f7fe00700 1 -- 172.21.15.28:0/3977807499 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f90004240 con 0x558b5398deb0 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.405+0000 7f6f7fe00700 1 -- 172.21.15.28:0/3977807499 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f6f900043e0 con 0x558b5398deb0 2024-05-20T20:16:53.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.405+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6f8c0ebc90 con 0x558b5398deb0 2024-05-20T20:16:53.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.406+0000 7f6f7fe00700 1 -- 172.21.15.28:0/3977807499 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6f90003680 con 0x558b5398deb0 2024-05-20T20:16:53.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.406+0000 7f6fafaa0b80 10 client.17187 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:53.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.406+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558b538a7060 con 0x558b5398deb0 2024-05-20T20:16:53.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.407+0000 7f6f7fe00700 1 -- 172.21.15.28:0/3977807499 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6f8c0ebc90 con 0x558b5398deb0 2024-05-20T20:16:53.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.407+0000 7f6f7fe00700 1 -- 172.21.15.28:0/3977807499 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f6f90025a30 con 0x558b5398deb0 2024-05-20T20:16:53.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.409+0000 7f6fafaa0b80 10 client.17187 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:53.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.409+0000 7f6fafaa0b80 10 client.17187 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:16:53.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.409+0000 7f6fafaa0b80 20 client.17187 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:53.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.409+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b539a1780 0x558b539a3b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.409+0000 7f6fafaa0b80 4 client.17187 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:53.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.409+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x558b538e8d90 con 0x558b539a1780 2024-05-20T20:16:53.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.412+0000 7f6f9de00700 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b539a1780 0x558b539a3b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.413+0000 7f6f9de00700 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b539a1780 0x558b539a3b70 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=25227bc5614e3f55 server_cookie=1aa58213e56832c4 in_seq=0 out_seq=0 2024-05-20T20:16:53.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.413+0000 7f6f7fe00700 10 client.17187 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:16:53.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.414+0000 7f6f7fe00700 1 -- 172.21.15.28:0/3977807499 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x558b538e8d90 con 0x558b539a1780 2024-05-20T20:16:53.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.414+0000 7f6f7fe00700 10 client.17187 handle_command_reply: tid=0 2024-05-20T20:16:53.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.415+0000 7f6fafaa0b80 1 client.17187 shutdown 2024-05-20T20:16:53.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.415+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 msgr2=0x558b539918b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.415+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539918b0 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f6f90005950 tx=0x7f6f90011720 comp rx=0 tx=0).stop 2024-05-20T20:16:53.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 shutdown_connections 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b539a1780 0x558b539a3b70 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6f7fe00700 0 client.17187 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b5398e7d0 0x558b53991df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b53988880 0x558b53994510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/3977807499 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539918b0 unknown :-1 s=CLOSED pgs=454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 >> 172.21.15.28:0/3977807499 conn(0x558b538b6be0 msgr2=0x558b53987250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 shutdown_connections 2024-05-20T20:16:53.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.416+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/3977807499 wait complete. 2024-05-20T20:16:53.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.417+0000 7f6fafaa0b80 20 client.17187 trim_cache size 0 max 16384 2024-05-20T20:16:53.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.451+0000 7f6fafaa0b80 1 Processor -- start 2024-05-20T20:16:53.451 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.451+0000 7f6fafaa0b80 1 -- start start 2024-05-20T20:16:53.452 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.452+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b53988260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.452+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b53994510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.452+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539829e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.452+0000 7f6f9de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b53988260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.453+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa00745d0 con 0x558b5398deb0 2024-05-20T20:16:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.453+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa01be730 con 0x558b53988880 2024-05-20T20:16:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.453+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa0074ca0 con 0x558b5398e7d0 2024-05-20T20:16:53.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.453+0000 7f6f9d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b53994510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.453+0000 7f6f9e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539829e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.454+0000 7f6f9e800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539829e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57000/0 (socket says 172.21.15.28:57000) 2024-05-20T20:16:53.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.454+0000 7f6f9e800700 1 -- 172.21.15.28:0/1861592875 learned_addr learned my addr 172.21.15.28:0/1861592875 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:53.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.453+0000 7f6f9d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b53994510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:55076/0 (socket says 172.21.15.28:55076) 2024-05-20T20:16:53.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.455+0000 7f6f9de00700 1 -- 172.21.15.28:0/1861592875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 msgr2=0x558b539829e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:53.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.455+0000 7f6f9de00700 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539829e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.455+0000 7f6f9de00700 1 -- 172.21.15.28:0/1861592875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 msgr2=0x558b53994510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:53.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.455+0000 7f6f9e800700 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539829e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:53.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.455+0000 7f6f9de00700 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b53994510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.455+0000 7f6f9de00700 1 -- 172.21.15.28:0/1861592875 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x558b53988c60 con 0x558b53988880 2024-05-20T20:16:53.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.456+0000 7f6f9de00700 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b53988260 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c0a5ed0 tx=0x7f6f8c005040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b71cd6d430479ff2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:53.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.456+0000 7f6f8aa00700 1 -- 172.21.15.28:0/1861592875 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c0a5d10 con 0x558b53988880 2024-05-20T20:16:53.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.457+0000 7f6f8aa00700 1 -- 172.21.15.28:0/1861592875 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6f8c0ac070 con 0x558b53988880 2024-05-20T20:16:53.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.457+0000 7f6f8aa00700 1 -- 172.21.15.28:0/1861592875 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c0a95c0 con 0x558b53988880 2024-05-20T20:16:53.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.457+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1861592875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 msgr2=0x558b53988260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.457+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b53988260 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c0a5ed0 tx=0x7f6f8c005040 comp rx=0 tx=0).stop 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1861592875 shutdown_connections 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539829e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b53988260 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1861592875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b53994510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1861592875 >> 172.21.15.28:0/1861592875 conn(0x558b538b6be0 msgr2=0x558b539901c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1861592875 shutdown_connections 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1861592875 wait complete. 2024-05-20T20:16:53.458 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.458+0000 7f6fafaa0b80 1 -- start start 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 1 Processor -- start 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b5399a860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539a8c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539a9180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa01be060 con 0x558b5398deb0 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa0075370 con 0x558b53988880 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6fa0072590 con 0x558b5398e7d0 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.459+0000 7f6fafaa0b80 10 client.0 _ll_register_callbacks cb 0x558b53880c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:53.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b5399a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b5399a860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:51534/0 (socket says 172.21.15.28:51534) 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 -- 172.21.15.28:0/1286648878 learned_addr learned my addr 172.21.15.28:0/1286648878 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9d400700 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539a8c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 -- 172.21.15.28:0/1286648878 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 msgr2=0x558b539a9180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539a9180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 -- 172.21.15.28:0/1286648878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 msgr2=0x558b539a8c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539a8c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9de00700 1 -- 172.21.15.28:0/1286648878 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x558b539a9a10 con 0x558b53988880 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.460+0000 7f6f9d400700 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539a8c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6f7fe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6f9de00700 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b5399a860 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c0a9b50 tx=0x7f6f8c021c80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bca7537328150897 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6f7fe00700 1 -- 172.21.15.28:0/1286648878 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f8c0c8020 con 0x558b53988880 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6f7fe00700 1 -- 172.21.15.28:0/1286648878 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f6f8c0a6440 con 0x558b53988880 2024-05-20T20:16:53.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f6f8c0ac070 con 0x558b53988880 2024-05-20T20:16:53.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6f7fe00700 1 -- 172.21.15.28:0/1286648878 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f6f8c0c8020 con 0x558b53988880 2024-05-20T20:16:53.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6fafaa0b80 10 client.27698 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:53.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.461+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x558b53a0c340 con 0x558b53988880 2024-05-20T20:16:53.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6f7fe00700 1 -- 172.21.15.28:0/1286648878 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6f8c0aa550 con 0x558b53988880 2024-05-20T20:16:53.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6f7fe00700 1 -- 172.21.15.28:0/1286648878 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f6f8c0218e0 con 0x558b53988880 2024-05-20T20:16:53.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6fafaa0b80 10 client.27698 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:53.464 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6fafaa0b80 10 client.27698 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:16:53.465 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6fafaa0b80 20 client.27698 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:53.465 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b53a0c840 0x558b53a0ec30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:53.465 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6fafaa0b80 4 client.27698 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:16:53.465 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x7f6f8c0ac070 con 0x558b53a0c840 2024-05-20T20:16:53.465 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.462+0000 7f6f9d400700 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b53a0c840 0x558b53a0ec30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:53.465 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.463+0000 7f6f7fe00700 10 client.27698 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:16:53.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.465+0000 7f6f9d400700 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b53a0c840 0x558b53a0ec30 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=fb84651d74072728 server_cookie=8a7ae786ec035b9d in_seq=0 out_seq=0 2024-05-20T20:16:53.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.465+0000 7f6f7fe00700 1 -- 172.21.15.28:0/1286648878 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x7f6f8c0ac070 con 0x558b53a0c840 2024-05-20T20:16:53.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.465+0000 7f6f7fe00700 10 client.27698 handle_command_reply: tid=0 2024-05-20T20:16:53.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.465+0000 7f6fafaa0b80 1 client.27698 shutdown 2024-05-20T20:16:53.466 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 msgr2=0x558b5399a860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b5399a860 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c0a9b50 tx=0x7f6f8c021c80 comp rx=0 tx=0).stop 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 shutdown_connections 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x558b53a0c840 0x558b53a0ec30 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x558b5398e7d0 0x558b539a9180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x558b53988880 0x558b5399a860 unknown :-1 s=CLOSED pgs=559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 --2- 172.21.15.28:0/1286648878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x558b5398deb0 0x558b539a8c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6f7fe00700 0 client.27698 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.466+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 >> 172.21.15.28:0/1286648878 conn(0x558b538b6be0 msgr2=0x558b539829e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:53.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.467+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 shutdown_connections 2024-05-20T20:16:53.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.467+0000 7f6fafaa0b80 1 -- 172.21.15.28:0/1286648878 wait complete. 2024-05-20T20:16:53.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.467+0000 7f6fafaa0b80 20 client.27698 trim_cache size 0 max 16384 2024-05-20T20:16:53.468 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:16:53.469 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.469+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f840b2aa0 msgr2=0x7f6f840b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.469+0000 7f6fa5200700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f840b2aa0 0x7f6f840b4f60 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f6f98009360 tx=0x7f6f9800d040 comp rx=0 tx=0).stop 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.469+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6fa006b790 msgr2=0x7f6fa01a0200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.469+0000 7f6fa5200700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6fa006b790 0x7f6fa01a0200 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7f6f8c006010 tx=0x7f6f8c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 shutdown_connections 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f6fa0108470 0x7f6fa01a4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6fa006b790 0x7f6fa01a0200 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f6f840b2aa0 0x7f6f840b4f60 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 --2- 172.21.15.28:0/2686024024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6fa0105f80 0x7f6fa01a0740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 >> 172.21.15.28:0/2686024024 conn(0x7f6fa00793f0 msgr2=0x7f6fa0101430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 shutdown_connections 2024-05-20T20:16:53.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:53.470+0000 7f6fa5200700 1 -- 172.21.15.28:0/2686024024 wait complete. 2024-05-20T20:16:53.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:53 smithi028 ceph-mon[31178]: pgmap v350: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 274 MiB/s wr, 2.10k op/s 2024-05-20T20:16:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:53 smithi154 ceph-mon[36314]: pgmap v350: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 274 MiB/s wr, 2.10k op/s 2024-05-20T20:16:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:53 smithi033 ceph-mon[36179]: pgmap v350: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 274 MiB/s wr, 2.10k op/s 2024-05-20T20:16:54.785 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:16:55.578 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:16:55.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:55 smithi154 ceph-mon[36314]: pgmap v351: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 363 MiB/s wr, 2.76k op/s 2024-05-20T20:16:55.630 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:55 smithi033 ceph-mon[36179]: pgmap v351: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 363 MiB/s wr, 2.76k op/s 2024-05-20T20:16:55.769 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:55 smithi028 ceph-mon[31178]: pgmap v351: 97 pgs: 97 active+clean; 61 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 363 MiB/s wr, 2.76k op/s 2024-05-20T20:16:56.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:56 smithi028 ceph-mon[31178]: pgmap v352: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 235 MiB/s wr, 1.83k op/s 2024-05-20T20:16:56.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:56 smithi154 ceph-mon[36314]: pgmap v352: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 235 MiB/s wr, 1.83k op/s 2024-05-20T20:16:56.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:56 smithi033 ceph-mon[36179]: pgmap v352: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 44 KiB/s rd, 235 MiB/s wr, 1.83k op/s 2024-05-20T20:16:59.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.139+0000 7ffaade00700 1 -- 172.21.15.28:0/490186933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 msgr2=0x7ffaa810a540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.139+0000 7ffaade00700 1 --2- 172.21.15.28:0/490186933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 0x7ffaa810a540 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7ffa980098f0 tx=0x7ffa98009c00 comp rx=0 tx=0).stop 2024-05-20T20:16:59.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.141+0000 7ffaade00700 1 -- 172.21.15.28:0/490186933 shutdown_connections 2024-05-20T20:16:59.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.141+0000 7ffaade00700 1 --2- 172.21.15.28:0/490186933 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffaa8118670 0x7ffaa811aa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.141+0000 7ffaade00700 1 --2- 172.21.15.28:0/490186933 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 0x7ffaa810a540 unknown :-1 s=CLOSED pgs=560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.141+0000 7ffaade00700 1 --2- 172.21.15.28:0/490186933 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa810ab10 0x7ffaa810af90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.141+0000 7ffaade00700 1 -- 172.21.15.28:0/490186933 >> 172.21.15.28:0/490186933 conn(0x7ffaa806e6b0 msgr2=0x7ffaa806eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:59.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.142+0000 7ffaade00700 1 -- 172.21.15.28:0/490186933 shutdown_connections 2024-05-20T20:16:59.144 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.143+0000 7ffaade00700 1 -- 172.21.15.28:0/490186933 wait complete. 2024-05-20T20:16:59.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.145+0000 7ffaade00700 1 Processor -- start 2024-05-20T20:16:59.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.145+0000 7ffaade00700 1 -- start start 2024-05-20T20:16:59.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.146+0000 7ffaade00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffaa810ab10 0x7ffaa807bf20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.146+0000 7ffaade00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 0x7ffaa807c460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.146+0000 7ffaade00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa8118670 0x7ffaa8077040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.146+0000 7ffaade00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa8075c40 con 0x7ffaa810ab10 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.146+0000 7ffaade00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa8075ac0 con 0x7ffaa810dcc0 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.146+0000 7ffaade00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffaa8075dc0 con 0x7ffaa8118670 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.147+0000 7ffaa7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa8118670 0x7ffaa8077040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.147+0000 7ffaa7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa8118670 0x7ffaa8077040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35296/0 (socket says 172.21.15.28:35296) 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.147+0000 7ffaa7400700 1 -- 172.21.15.28:0/3827420840 learned_addr learned my addr 172.21.15.28:0/3827420840 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:59.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.147+0000 7ffaa6000700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 0x7ffaa807c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.148+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffaa810ab10 0x7ffaa807bf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.151+0000 7ffaa7400700 1 -- 172.21.15.28:0/3827420840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 msgr2=0x7ffaa807c460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.151+0000 7ffaa7400700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 0x7ffaa807c460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.151+0000 7ffaa7400700 1 -- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffaa810ab10 msgr2=0x7ffaa807bf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.152+0000 7ffaa7400700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffaa810ab10 0x7ffaa807bf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.154 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.152+0000 7ffaa7400700 1 -- 172.21.15.28:0/3827420840 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffaa8077940 con 0x7ffaa8118670 2024-05-20T20:16:59.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.152+0000 7ffaa6000700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 0x7ffaa807c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:16:59.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.152+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffaa810ab10 0x7ffaa807bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:59.155 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.153+0000 7ffaa7400700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa8118670 0x7ffaa8077040 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7ffaa000b970 tx=0x7ffaa000bd30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:59.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.153+0000 7ffaaca00700 1 -- 172.21.15.28:0/3827420840 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffaa0019070 con 0x7ffaa8118670 2024-05-20T20:16:59.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.153+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffaa8077c30 con 0x7ffaa8118670 2024-05-20T20:16:59.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.153+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ffaa8083e20 con 0x7ffaa8118670 2024-05-20T20:16:59.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.154+0000 7ffaaca00700 1 -- 172.21.15.28:0/3827420840 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffaa0014440 con 0x7ffaa8118670 2024-05-20T20:16:59.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.154+0000 7ffaaca00700 1 -- 172.21.15.28:0/3827420840 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffaa000ce60 con 0x7ffaa8118670 2024-05-20T20:16:59.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.160+0000 7ffab87f6b80 1 Processor -- start 2024-05-20T20:16:59.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.161+0000 7ffaaca00700 1 -- 172.21.15.28:0/3827420840 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7ffaa001c070 con 0x7ffaa8118670 2024-05-20T20:16:59.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.162+0000 7ffab87f6b80 1 -- start start 2024-05-20T20:16:59.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.163+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e96f290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.163+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e969260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.163+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e969ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.163+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56040e8c7e40 con 0x56040e96eeb0 2024-05-20T20:16:59.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.163+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56040e8b79e0 con 0x56040e969880 2024-05-20T20:16:59.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.163+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56040e8bba20 con 0x56040e96f7d0 2024-05-20T20:16:59.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.164+0000 7ffaaca00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffa8c0b2b40 0x7ffa8c0b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.165+0000 7ffaaca00700 1 -- 172.21.15.28:0/3827420840 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7ffaa00e1790 con 0x7ffaa8118670 2024-05-20T20:16:59.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.165+0000 7ffaa7400700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffa8c0b2b40 0x7ffa8c0b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.165+0000 7ffaa6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e969260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.166+0000 7ffaa6a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e969260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35312/0 (socket says 172.21.15.28:35312) 2024-05-20T20:16:59.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.166+0000 7ffaa6a00700 1 -- 172.21.15.28:0/381076192 learned_addr learned my addr 172.21.15.28:0/381076192 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:59.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.166+0000 7ffaa6000700 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e969ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.166+0000 7ffaa6000700 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e96f290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.166+0000 7ffaa7400700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffa8c0b2b40 0x7ffa8c0b5000 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7ffaa00a8be0 tx=0x7ffaa0004b10 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:59.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.167+0000 7ffaa6a00700 1 -- 172.21.15.28:0/381076192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 msgr2=0x56040e969ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.167+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e969ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.167+0000 7ffaa6a00700 1 -- 172.21.15.28:0/381076192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 msgr2=0x56040e96f290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.167+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e96f290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.167+0000 7ffaa6a00700 1 -- 172.21.15.28:0/381076192 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56040e880420 con 0x56040e96f7d0 2024-05-20T20:16:59.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.167+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e969260 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7ffa98009fd0 tx=0x7ffa980056a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7814268a858dc6f3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.171+0000 7ffa9cc00700 1 -- 172.21.15.28:0/381076192 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa98025070 con 0x56040e96f7d0 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.171+0000 7ffa9cc00700 1 -- 172.21.15.28:0/381076192 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffa9800fdf0 con 0x56040e96f7d0 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.171+0000 7ffa9cc00700 1 -- 172.21.15.28:0/381076192 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa98003970 con 0x56040e96f7d0 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.171+0000 7ffab87f6b80 1 -- 172.21.15.28:0/381076192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 msgr2=0x56040e969260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.171+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e969260 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7ffa98009fd0 tx=0x7ffa980056a0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.172+0000 7ffab87f6b80 1 -- 172.21.15.28:0/381076192 shutdown_connections 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.172+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e969ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.172+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e96f290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.172+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/381076192 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e969260 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.172+0000 7ffab87f6b80 1 -- 172.21.15.28:0/381076192 >> 172.21.15.28:0/381076192 conn(0x56040e897be0 msgr2=0x56040e964010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.172+0000 7ffab87f6b80 1 -- 172.21.15.28:0/381076192 shutdown_connections 2024-05-20T20:16:59.173 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.172+0000 7ffab87f6b80 1 -- 172.21.15.28:0/381076192 wait complete. 2024-05-20T20:16:59.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.173+0000 7ffab87f6b80 1 -- start start 2024-05-20T20:16:59.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.173+0000 7ffab87f6b80 1 Processor -- start 2024-05-20T20:16:59.174 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e975440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e9727e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e972d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56040e7bdfa0 con 0x56040e96eeb0 2024-05-20T20:16:59.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56040e858760 con 0x56040e969880 2024-05-20T20:16:59.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56040e97e9f0 con 0x56040e96f7d0 2024-05-20T20:16:59.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffab87f6b80 10 client.0 _ll_register_callbacks cb 0x56040e861c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffaa6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e975440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffaa6000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e975440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43100/0 (socket says 172.21.15.28:43100) 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffaa6000700 1 -- 172.21.15.28:0/2750847368 learned_addr learned my addr 172.21.15.28:0/2750847368 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffaa6000700 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e972d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.174+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e9727e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffaa6000700 1 -- 172.21.15.28:0/2750847368 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 msgr2=0x56040e972d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffaa6000700 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e972d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffaa6000700 1 -- 172.21.15.28:0/2750847368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 msgr2=0x56040e9727e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffaa6000700 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e9727e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffaa6000700 1 -- 172.21.15.28:0/2750847368 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56040e8078d0 con 0x56040e969880 2024-05-20T20:16:59.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffaa6000700 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e975440 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7ffa94011e40 tx=0x7ffa940047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ade8734c3833f78b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:59.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffa91600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:16:59.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffa91600700 1 -- 172.21.15.28:0/2750847368 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa94010da0 con 0x56040e969880 2024-05-20T20:16:59.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffa91600700 1 -- 172.21.15.28:0/2750847368 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7ffa94004030 con 0x56040e969880 2024-05-20T20:16:59.177 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.175+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffa9800fdf0 con 0x56040e969880 2024-05-20T20:16:59.178 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffa91600700 1 -- 172.21.15.28:0/2750847368 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffa94004230 con 0x56040e969880 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 10 client.27710 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56040e888060 con 0x56040e969880 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffa91600700 1 -- 172.21.15.28:0/2750847368 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffa9800fdf0 con 0x56040e969880 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffa91600700 1 -- 172.21.15.28:0/2750847368 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffa94021420 con 0x56040e969880 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 10 client.27710 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 10 client.27710 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 20 client.27710 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e98b530 0x56040e98d940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 4 client.27710 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.176+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56040e8c9d90 con 0x56040e98b530 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.177+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e98b530 0x56040e98d940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.177+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e98b530 0x56040e98d940 crc :-1 s=READY pgs=235 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=76c0b6ef715331c9 server_cookie=e35195c3753c1052 in_seq=0 out_seq=0 2024-05-20T20:16:59.179 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.177+0000 7ffa91600700 10 client.27710 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:59.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.178+0000 7ffa91600700 1 -- 172.21.15.28:0/2750847368 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56040e8c9d90 con 0x56040e98b530 2024-05-20T20:16:59.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.178+0000 7ffa91600700 10 client.27710 handle_command_reply: tid=0 2024-05-20T20:16:59.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.178+0000 7ffab87f6b80 1 client.27710 shutdown 2024-05-20T20:16:59.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.178+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 msgr2=0x56040e975440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.178+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e975440 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7ffa94011e40 tx=0x7ffa940047c0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.179+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 shutdown_connections 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.179+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e98b530 0x56040e98d940 crc :-1 s=READY pgs=235 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.179+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96f7d0 0x56040e972d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.179+0000 7ffa91600700 0 client.27710 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.179+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e969880 0x56040e975440 unknown :-1 s=CLOSED pgs=561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.179+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/2750847368 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e9727e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.179+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 >> 172.21.15.28:0/2750847368 conn(0x56040e897be0 msgr2=0x56040e968250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.180+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 shutdown_connections 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.180+0000 7ffab87f6b80 1 -- 172.21.15.28:0/2750847368 wait complete. 2024-05-20T20:16:59.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.181+0000 7ffab87f6b80 20 client.27710 trim_cache size 0 max 16384 2024-05-20T20:16:59.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.200+0000 7ffab87f6b80 1 Processor -- start 2024-05-20T20:16:59.202 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffab87f6b80 1 -- start start 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e969880 0x56040e969260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e969c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e9710f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffa8800d610 con 0x56040e96eeb0 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffa8800f150 con 0x56040e96f7d0 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffa8800dce0 con 0x56040e969880 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffaa6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e969880 0x56040e969260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.201+0000 7ffaa6000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e969880 0x56040e969260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35332/0 (socket says 172.21.15.28:35332) 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 -- 172.21.15.28:0/1588569405 learned_addr learned my addr 172.21.15.28:0/1588569405 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:59.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e9710f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e969c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 -- 172.21.15.28:0/1588569405 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 msgr2=0x56040e9710f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e9710f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 -- 172.21.15.28:0/1588569405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 msgr2=0x56040e969c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e969c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 -- 172.21.15.28:0/1588569405 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56040e970d70 con 0x56040e969880 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e969c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:16:59.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.202+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e969880 0x56040e969260 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7ffa94009fd0 tx=0x7ffa94009400 comp rx=0 tx=0).ready entity=mon.1 client_cookie=820da6e7145cc4b8 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:59.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.203+0000 7ffa93e00700 1 -- 172.21.15.28:0/1588569405 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa9402e040 con 0x56040e969880 2024-05-20T20:16:59.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.203+0000 7ffa93e00700 1 -- 172.21.15.28:0/1588569405 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffa940043d0 con 0x56040e969880 2024-05-20T20:16:59.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.203+0000 7ffa93e00700 1 -- 172.21.15.28:0/1588569405 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa94021e60 con 0x56040e969880 2024-05-20T20:16:59.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1588569405 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e969880 msgr2=0x56040e969260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e969880 0x56040e969260 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7ffa94009fd0 tx=0x7ffa94009400 comp rx=0 tx=0).stop 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1588569405 shutdown_connections 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e9710f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e969880 0x56040e969260 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1588569405 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e96eeb0 0x56040e969c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1588569405 >> 172.21.15.28:0/1588569405 conn(0x56040e897be0 msgr2=0x56040e96c950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1588569405 shutdown_connections 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.204+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1588569405 wait complete. 2024-05-20T20:16:59.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.205+0000 7ffab87f6b80 1 -- start start 2024-05-20T20:16:59.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.205+0000 7ffab87f6b80 1 Processor -- start 2024-05-20T20:16:59.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e969880 0x56040e9843e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96eeb0 0x56040e992a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffab87f6b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e992f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7ffa8800ea80 con 0x56040e969880 2024-05-20T20:16:59.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7ffa8800e3b0 con 0x56040e96f7d0 2024-05-20T20:16:59.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffab87f6b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7ffa8800b5d0 con 0x56040e96eeb0 2024-05-20T20:16:59.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffab87f6b80 10 client.0 _ll_register_callbacks cb 0x56040e861c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:16:59.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffaa6000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e969880 0x56040e9843e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffaa6000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e969880 0x56040e9843e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58582/0 (socket says 172.21.15.28:58582) 2024-05-20T20:16:59.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffaa6000700 1 -- 172.21.15.28:0/1680489041 learned_addr learned my addr 172.21.15.28:0/1680489041 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:16:59.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.206+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e992f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96eeb0 0x56040e992a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffaa6000700 1 -- 172.21.15.28:0/1680489041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96eeb0 msgr2=0x56040e992a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96eeb0 0x56040e992a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffaa6000700 1 -- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 msgr2=0x56040e992f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e992f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffaa6000700 1 -- 172.21.15.28:0/1680489041 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56040e9937c0 con 0x56040e969880 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffaa6000700 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e969880 0x56040e9843e0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7ffa940097f0 tx=0x7ffa94009530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=72ab56d1d2f64c92 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffa91600700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.207+0000 7ffa91600700 1 -- 172.21.15.28:0/1680489041 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffa9402e070 con 0x56040e969880 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.208+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7ffa940043d0 con 0x56040e969880 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.208+0000 7ffa91600700 1 -- 172.21.15.28:0/1680489041 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7ffa94005220 con 0x56040e969880 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.208+0000 7ffa91600700 1 -- 172.21.15.28:0/1680489041 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7ffa94005420 con 0x56040e969880 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.208+0000 7ffab87f6b80 10 client.17202 fetch_fsmap learned FSMap version 29 2024-05-20T20:16:59.210 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.208+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56040e9f6040 con 0x56040e969880 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.208+0000 7ffa91600700 1 -- 172.21.15.28:0/1680489041 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7ffa940043d0 con 0x56040e969880 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.208+0000 7ffa91600700 1 -- 172.21.15.28:0/1680489041 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7ffa94034040 con 0x56040e969880 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.209+0000 7ffab87f6b80 10 client.17202 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.209+0000 7ffab87f6b80 10 client.17202 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.209+0000 7ffab87f6b80 20 client.17202 populate_metadata read hostname 'smithi028' 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.209+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e9f6530 0x56040e9f8920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.209+0000 7ffab87f6b80 4 client.17202 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.209+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56040e8c3570 con 0x56040e9f6530 2024-05-20T20:16:59.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.209+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e9f6530 0x56040e9f8920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffaa6a00700 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e9f6530 0x56040e9f8920 crc :-1 s=READY pgs=236 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=be058511c0b37d4f server_cookie=9f3d654f7299665d in_seq=0 out_seq=0 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffa91600700 10 client.17202 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffa91600700 1 -- 172.21.15.28:0/1680489041 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56040e8c3570 con 0x56040e9f6530 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffa91600700 10 client.17202 handle_command_reply: tid=0 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 client.17202 shutdown 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e969880 msgr2=0x56040e9843e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e969880 0x56040e9843e0 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7ffa940097f0 tx=0x7ffa94009530 comp rx=0 tx=0).stop 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 shutdown_connections 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56040e9f6530 0x56040e9f8920 crc :-1 s=READY pgs=236 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.212 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56040e96f7d0 0x56040e992f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56040e969880 0x56040e9843e0 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 --2- 172.21.15.28:0/1680489041 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56040e96eeb0 0x56040e992a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 >> 172.21.15.28:0/1680489041 conn(0x56040e897be0 msgr2=0x56040e96c950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:59.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.210+0000 7ffa91600700 0 client.17202 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:16:59.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.213+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 shutdown_connections 2024-05-20T20:16:59.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.213+0000 7ffab87f6b80 1 -- 172.21.15.28:0/1680489041 wait complete. 2024-05-20T20:16:59.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.213+0000 7ffab87f6b80 20 client.17202 trim_cache size 0 max 16384 2024-05-20T20:16:59.214 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:16:59.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.215+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffa8c0b2b40 msgr2=0x7ffa8c0b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.215+0000 7ffaade00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffa8c0b2b40 0x7ffa8c0b5000 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7ffaa00a8be0 tx=0x7ffaa0004b10 comp rx=0 tx=0).stop 2024-05-20T20:16:59.216 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.215+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa8118670 msgr2=0x7ffaa8077040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:16:59.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.215+0000 7ffaade00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa8118670 0x7ffaa8077040 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7ffaa000b970 tx=0x7ffaa000bd30 comp rx=0 tx=0).stop 2024-05-20T20:16:59.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.216+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 shutdown_connections 2024-05-20T20:16:59.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.216+0000 7ffaade00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ffaa8118670 0x7ffaa8077040 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.216+0000 7ffaade00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7ffaa810ab10 0x7ffaa807bf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.216+0000 7ffaade00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7ffa8c0b2b40 0x7ffa8c0b5000 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.216+0000 7ffaade00700 1 --2- 172.21.15.28:0/3827420840 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7ffaa810dcc0 0x7ffaa807c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:16:59.217 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.216+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 >> 172.21.15.28:0/3827420840 conn(0x7ffaa806e6b0 msgr2=0x7ffaa8119bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:16:59.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.217+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 shutdown_connections 2024-05-20T20:16:59.218 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:16:59.218+0000 7ffaade00700 1 -- 172.21.15.28:0/3827420840 wait complete. 2024-05-20T20:16:59.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:16:59 smithi028 ceph-mon[31178]: pgmap v353: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 211 MiB/s wr, 1.64k op/s 2024-05-20T20:16:59.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:16:59 smithi154 ceph-mon[36314]: pgmap v353: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 211 MiB/s wr, 1.64k op/s 2024-05-20T20:16:59.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:16:59 smithi033 ceph-mon[36179]: pgmap v353: 97 pgs: 97 active+clean; 62 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 211 MiB/s wr, 1.64k op/s 2024-05-20T20:17:00.262 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.260+0000 7f275ba00640 1 -- 172.21.15.28:0/1139985260 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275410f340 msgr2=0x7f2754111730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.262 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.260+0000 7f275ba00640 1 --2- 172.21.15.28:0/1139985260 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275410f340 0x7f2754111730 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7f274c009ca0 tx=0x7f274c02a950 comp rx=0 tx=0).stop 2024-05-20T20:17:00.262 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.261+0000 7f275ba00640 1 -- 172.21.15.28:0/1139985260 shutdown_connections 2024-05-20T20:17:00.262 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.261+0000 7f275ba00640 1 --2- 172.21.15.28:0/1139985260 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f275410f340 0x7f2754111730 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.262 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.261+0000 7f275ba00640 1 --2- 172.21.15.28:0/1139985260 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2754075640 0x7f275410ee00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.263 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.261+0000 7f275ba00640 1 --2- 172.21.15.28:0/1139985260 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f2754074d20 0x7f2754075100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.263 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.261+0000 7f275ba00640 1 -- 172.21.15.28:0/1139985260 >> 172.21.15.28:0/1139985260 conn(0x7f2754079ee0 msgr2=0x7f275407a2f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:00.263 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.261+0000 7f275ba00640 1 -- 172.21.15.28:0/1139985260 shutdown_connections 2024-05-20T20:17:00.263 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.261+0000 7f275ba00640 1 -- 172.21.15.28:0/1139985260 wait complete. 2024-05-20T20:17:00.264 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.263+0000 7f275ba00640 1 Processor -- start 2024-05-20T20:17:00.265 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f275ba00640 1 -- start start 2024-05-20T20:17:00.265 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f275ba00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2754074d20 0x7f275419b450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.265 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f275ba00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2754075640 0x7f275419b990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.265 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f275ba00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275410f340 0x7f275419fcc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.265 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f275ba00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2754114f30 con 0x7f2754075640 2024-05-20T20:17:00.265 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f275ba00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2754114db0 con 0x7f275410f340 2024-05-20T20:17:00.265 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f275ba00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f27541150b0 con 0x7f2754074d20 2024-05-20T20:17:00.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f2759000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2754074d20 0x7f275419b450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f2759000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2754074d20 0x7f275419b450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35354/0 (socket says 172.21.15.28:35354) 2024-05-20T20:17:00.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f2759000640 1 -- 172.21.15.28:0/2413418378 learned_addr learned my addr 172.21.15.28:0/2413418378 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:00.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.264+0000 7f2753e00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2754075640 0x7f275419b990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.266 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.265+0000 7f2759a00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275410f340 0x7f275419fcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.265+0000 7f2759a00640 1 -- 172.21.15.28:0/2413418378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2754074d20 msgr2=0x7f275419b450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.265+0000 7f2759a00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2754074d20 0x7f275419b450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.265+0000 7f2759a00640 1 -- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2754075640 msgr2=0x7f275419b990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.265+0000 7f2759a00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2754075640 0x7f275419b990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.265+0000 7f2759a00640 1 -- 172.21.15.28:0/2413418378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f27541a03c0 con 0x7f275410f340 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.265+0000 7f2759a00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275410f340 0x7f275419fcc0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f274c006010 tx=0x7f274c031f40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.266+0000 7f2751600640 1 -- 172.21.15.28:0/2413418378 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f274c034040 con 0x7f275410f340 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.266+0000 7f2751600640 1 -- 172.21.15.28:0/2413418378 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f274c0308f0 con 0x7f275410f340 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.266+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f27541a0650 con 0x7f275410f340 2024-05-20T20:17:00.267 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.266+0000 7f2751600640 1 -- 172.21.15.28:0/2413418378 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f274c0044d0 con 0x7f275410f340 2024-05-20T20:17:00.268 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.266+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f275406c470 con 0x7f275410f340 2024-05-20T20:17:00.270 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.269+0000 7f275e7da740 1 Processor -- start 2024-05-20T20:17:00.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f2751600640 1 -- 172.21.15.28:0/2413418378 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f274c037020 con 0x7f275410f340 2024-05-20T20:17:00.272 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f275e7da740 1 -- start start 2024-05-20T20:17:00.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f275e7da740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x562214415880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f275e7da740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x562214420140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f275e7da740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 0x562214422a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f275e7da740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56221437db90 con 0x562214420680 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f275e7da740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56221436d730 con 0x5622144154a0 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.272+0000 7f275e7da740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562214371770 con 0x562214415dc0 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x562214415880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x562214415880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43136/0 (socket says 172.21.15.28:43136) 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 -- 172.21.15.28:0/1054228752 learned_addr learned my addr 172.21.15.28:0/1054228752 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:00.274 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 0x562214422a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2759000640 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x562214420140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 -- 172.21.15.28:0/1054228752 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 msgr2=0x562214420140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:00.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x562214420140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 -- 172.21.15.28:0/1054228752 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 msgr2=0x562214422a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:00.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 0x562214422a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.273+0000 7f2753e00640 1 -- 172.21.15.28:0/1054228752 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5622143068b0 con 0x5622144154a0 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.274+0000 7f2751600640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f273c0b2a90 0x7f273c0b4f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.274+0000 7f2753e00640 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x562214415880 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f2740016500 tx=0x7f274000a2b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=406999d5283fbf3d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.274+0000 7f2751600640 1 -- 172.21.15.28:0/2413418378 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f274c1032c0 con 0x7f275410f340 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.274+0000 7f2746000640 1 -- 172.21.15.28:0/1054228752 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2740018430 con 0x5622144154a0 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.274+0000 7f2759000640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f273c0b2a90 0x7f273c0b4f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.274+0000 7f2746000640 1 -- 172.21.15.28:0/1054228752 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2740018a80 con 0x5622144154a0 2024-05-20T20:17:00.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.274+0000 7f2746000640 1 -- 172.21.15.28:0/1054228752 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f27400178b0 con 0x5622144154a0 2024-05-20T20:17:00.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 -- 172.21.15.28:0/1054228752 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 msgr2=0x562214415880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x562214415880 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f2740016500 tx=0x7f274000a2b0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f2759000640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f273c0b2a90 0x7f273c0b4f50 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x562214416a40 tx=0x7f2748005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:00.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 -- 172.21.15.28:0/1054228752 shutdown_connections 2024-05-20T20:17:00.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 0x562214422a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x562214420140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 --2- 172.21.15.28:0/1054228752 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x562214415880 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 -- 172.21.15.28:0/1054228752 >> 172.21.15.28:0/1054228752 conn(0x5622144119b0 msgr2=0x562214411d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:00.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.275+0000 7f275e7da740 1 -- 172.21.15.28:0/1054228752 shutdown_connections 2024-05-20T20:17:00.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.276+0000 7f275e7da740 1 -- 172.21.15.28:0/1054228752 wait complete. 2024-05-20T20:17:00.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.276+0000 7f275e7da740 1 -- start start 2024-05-20T20:17:00.278 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.276+0000 7f275e7da740 1 Processor -- start 2024-05-20T20:17:00.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.277+0000 7f275e7da740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5622144154a0 0x5622144200d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.277+0000 7f275e7da740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441d3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.277+0000 7f275e7da740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441db00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.277+0000 7f275e7da740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562214053790 con 0x5622144154a0 2024-05-20T20:17:00.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.277+0000 7f275e7da740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562214315bc0 con 0x562214420680 2024-05-20T20:17:00.279 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.277+0000 7f275e7da740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562214423250 con 0x562214415dc0 2024-05-20T20:17:00.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.277+0000 7f275e7da740 10 client.0 _ll_register_callbacks cb 0x5622142f3400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:00.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.279+0000 7f2759000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5622144154a0 0x5622144200d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.279+0000 7f2753e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441d3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.279+0000 7f2759000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5622144154a0 0x5622144200d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58630/0 (socket says 172.21.15.28:58630) 2024-05-20T20:17:00.280 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.279+0000 7f2759000640 1 -- 172.21.15.28:0/1048289747 learned_addr learned my addr 172.21.15.28:0/1048289747 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:00.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.279+0000 7f2759a00640 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441db00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.280+0000 7f2759000640 1 -- 172.21.15.28:0/1048289747 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 msgr2=0x56221441d3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:00.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.280+0000 7f2759000640 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441d3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.280+0000 7f2759000640 1 -- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 msgr2=0x56221441db00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:00.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.280+0000 7f2759000640 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441db00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.280+0000 7f2759000640 1 -- 172.21.15.28:0/1048289747 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562214082210 con 0x5622144154a0 2024-05-20T20:17:00.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.280+0000 7f2753e00640 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441d3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:17:00.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.281+0000 7f2759000640 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5622144154a0 0x5622144200d0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7f274801e9d0 tx=0x7f2748004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=66d87e6183421e6c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:00.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.281+0000 7f273aa00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:17:00.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.281+0000 7f273aa00640 1 -- 172.21.15.28:0/1048289747 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2748009e00 con 0x5622144154a0 2024-05-20T20:17:00.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.282+0000 7f273aa00640 1 -- 172.21.15.28:0/1048289747 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f2748018030 con 0x5622144154a0 2024-05-20T20:17:00.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.282+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2740018a80 con 0x5622144154a0 2024-05-20T20:17:00.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.282+0000 7f273aa00640 1 -- 172.21.15.28:0/1048289747 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f2748004020 con 0x5622144154a0 2024-05-20T20:17:00.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.283+0000 7f275e7da740 10 client.17214 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:00.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.283+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56221414e620 con 0x5622144154a0 2024-05-20T20:17:00.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.283+0000 7f273aa00640 1 -- 172.21.15.28:0/1048289747 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2740018a80 con 0x5622144154a0 2024-05-20T20:17:00.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f273aa00640 1 -- 172.21.15.28:0/1048289747 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f27480334c0 con 0x5622144154a0 2024-05-20T20:17:00.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f275e7da740 10 client.17214 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:00.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f275e7da740 10 client.17214 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:00.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f275e7da740 20 client.17214 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:00.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f275e7da740 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562214430140 0x5622144325b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f275e7da740 4 client.17214 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:00.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56221437fae0 con 0x562214430140 2024-05-20T20:17:00.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.284+0000 7f2753e00640 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562214430140 0x5622144325b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.285+0000 7f2753e00640 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562214430140 0x5622144325b0 crc :-1 s=READY pgs=237 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=963c7c90e13c9a90 server_cookie=1c634296ffc5252 in_seq=0 out_seq=0 2024-05-20T20:17:00.285 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.285+0000 7f273aa00640 10 client.17214 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:00.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f273aa00640 1 -- 172.21.15.28:0/1048289747 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56221437fae0 con 0x562214430140 2024-05-20T20:17:00.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f273aa00640 10 client.17214 handle_command_reply: tid=0 2024-05-20T20:17:00.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 client.17214 shutdown 2024-05-20T20:17:00.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5622144154a0 msgr2=0x5622144200d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5622144154a0 0x5622144200d0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7f274801e9d0 tx=0x7f2748004730 comp rx=0 tx=0).stop 2024-05-20T20:17:00.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 shutdown_connections 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x562214430140 0x5622144325b0 crc :-1 s=READY pgs=237 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441db00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441d3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 --2- 172.21.15.28:0/1048289747 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5622144154a0 0x5622144200d0 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f273aa00640 0 client.17214 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.286+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 >> 172.21.15.28:0/1048289747 conn(0x5622144119b0 msgr2=0x562214408310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.287+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 shutdown_connections 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.287+0000 7f275e7da740 1 -- 172.21.15.28:0/1048289747 wait complete. 2024-05-20T20:17:00.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.287+0000 7f275e7da740 20 client.17214 trim_cache size 0 max 16384 2024-05-20T20:17:00.332 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.329+0000 7f275e7da740 1 Processor -- start 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.329+0000 7f275e7da740 1 -- start start 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.329+0000 7f275e7da740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x56221442c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.329+0000 7f275e7da740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441fc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.330+0000 7f275e7da740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 0x562214420140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.330+0000 7f275e7da740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2754070940 con 0x562214420680 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.330+0000 7f275e7da740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2754072480 con 0x5622144154a0 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.330+0000 7f275e7da740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2754071010 con 0x562214415dc0 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.330+0000 7f2753e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441fc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.330+0000 7f2753e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441fc00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35376/0 (socket says 172.21.15.28:35376) 2024-05-20T20:17:00.333 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.330+0000 7f2753e00640 1 -- 172.21.15.28:0/981851772 learned_addr learned my addr 172.21.15.28:0/981851772 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:00.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.331+0000 7f2753e00640 1 -- 172.21.15.28:0/981851772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 msgr2=0x56221442c2b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:17:00.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.331+0000 7f2753e00640 1 --2- 172.21.15.28:0/981851772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x56221442c2b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.331+0000 7f2753e00640 1 -- 172.21.15.28:0/981851772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 msgr2=0x562214420140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:00.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.331+0000 7f2753e00640 1 --2- 172.21.15.28:0/981851772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 0x562214420140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.331+0000 7f2753e00640 1 -- 172.21.15.28:0/981851772 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56221441de30 con 0x562214415dc0 2024-05-20T20:17:00.334 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.332+0000 7f2753e00640 1 --2- 172.21.15.28:0/981851772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441fc00 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f27400164d0 tx=0x7f27400047d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=abb80fded35670cb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.332+0000 7f2745600640 1 -- 172.21.15.28:0/981851772 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2740055070 con 0x562214415dc0 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.332+0000 7f2745600640 1 -- 172.21.15.28:0/981851772 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2740017d00 con 0x562214415dc0 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.332+0000 7f2745600640 1 -- 172.21.15.28:0/981851772 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2740018670 con 0x562214415dc0 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.333+0000 7f275e7da740 1 -- 172.21.15.28:0/981851772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 msgr2=0x56221441fc00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.333+0000 7f275e7da740 1 --2- 172.21.15.28:0/981851772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441fc00 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f27400164d0 tx=0x7f27400047d0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.334+0000 7f275e7da740 1 -- 172.21.15.28:0/981851772 shutdown_connections 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.334+0000 7f275e7da740 1 --2- 172.21.15.28:0/981851772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214420680 0x562214420140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.334+0000 7f275e7da740 1 --2- 172.21.15.28:0/981851772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562214415dc0 0x56221441fc00 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.334+0000 7f275e7da740 1 --2- 172.21.15.28:0/981851772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5622144154a0 0x56221442c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.335 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.334+0000 7f275e7da740 1 -- 172.21.15.28:0/981851772 >> 172.21.15.28:0/981851772 conn(0x5622144119b0 msgr2=0x5622144176e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:00.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.334+0000 7f275e7da740 1 -- 172.21.15.28:0/981851772 shutdown_connections 2024-05-20T20:17:00.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.334+0000 7f275e7da740 1 -- 172.21.15.28:0/981851772 wait complete. 2024-05-20T20:17:00.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.335+0000 7f275e7da740 1 -- start start 2024-05-20T20:17:00.336 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.335+0000 7f275e7da740 1 Processor -- start 2024-05-20T20:17:00.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.336+0000 7f275e7da740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5622144154a0 0x56221441a410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.336+0000 7f275e7da740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214415dc0 0x56221441a950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.336+0000 7f275e7da740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441ae90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.337 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.336+0000 7f275e7da740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f2754071db0 con 0x562214415dc0 2024-05-20T20:17:00.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.336+0000 7f275e7da740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f27540716e0 con 0x562214420680 2024-05-20T20:17:00.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.336+0000 7f275e7da740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f275406e940 con 0x5622144154a0 2024-05-20T20:17:00.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.336+0000 7f275e7da740 10 client.0 _ll_register_callbacks cb 0x7f2748009830 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:00.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.337+0000 7f2759a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441ae90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.337+0000 7f2759a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441ae90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:43156/0 (socket says 172.21.15.28:43156) 2024-05-20T20:17:00.338 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.337+0000 7f2759a00640 1 -- 172.21.15.28:0/3841871168 learned_addr learned my addr 172.21.15.28:0/3841871168 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:00.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.337+0000 7f2759000640 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5622144154a0 0x56221441a410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.337+0000 7f2753e00640 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214415dc0 0x56221441a950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.338+0000 7f2759a00640 1 -- 172.21.15.28:0/3841871168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5622144154a0 msgr2=0x56221441a410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:00.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.338+0000 7f2759a00640 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5622144154a0 0x56221441a410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.338+0000 7f2759a00640 1 -- 172.21.15.28:0/3841871168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214415dc0 msgr2=0x56221441a950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:00.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.338+0000 7f2759a00640 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214415dc0 0x56221441a950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.339 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.338+0000 7f2759a00640 1 -- 172.21.15.28:0/3841871168 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562214499e10 con 0x562214420680 2024-05-20T20:17:00.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.338+0000 7f2759a00640 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441ae90 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f274c0ca630 tx=0x7f274c0c9e10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=66efb890ecbf3ebf server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:00.340 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.338+0000 7f273aa00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:17:00.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.342+0000 7f273aa00640 1 -- 172.21.15.28:0/3841871168 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f274c02c040 con 0x562214420680 2024-05-20T20:17:00.343 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.343+0000 7f273aa00640 1 -- 172.21.15.28:0/3841871168 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(91..91 src has 1..91) ==== 8542+0+0 (secure 0 0 0) 0x7f274c0ca900 con 0x562214420680 2024-05-20T20:17:00.344 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.343+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f2740017d00 con 0x562214420680 2024-05-20T20:17:00.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.348+0000 7f273aa00640 1 -- 172.21.15.28:0/3841871168 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f274c103110 con 0x562214420680 2024-05-20T20:17:00.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.348+0000 7f275e7da740 10 client.27731 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:00.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.348+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56221449b160 con 0x562214420680 2024-05-20T20:17:00.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.350+0000 7f273aa00640 1 -- 172.21.15.28:0/3841871168 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2740017d00 con 0x562214420680 2024-05-20T20:17:00.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.350+0000 7f273aa00640 1 -- 172.21.15.28:0/3841871168 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f274c0d3070 con 0x562214420680 2024-05-20T20:17:00.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.350+0000 7f275e7da740 10 client.27731 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:00.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.350+0000 7f275e7da740 10 client.27731 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:00.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.350+0000 7f275e7da740 20 client.27731 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:00.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.351+0000 7f275e7da740 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56221449b870 0x56221449dc90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:00.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.351+0000 7f275e7da740 4 client.27731 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:17:00.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.351+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5622143792c0 con 0x56221449b870 2024-05-20T20:17:00.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.352+0000 7f2759000640 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56221449b870 0x56221449dc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:00.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.353+0000 7f2759000640 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56221449b870 0x56221449dc90 crc :-1 s=READY pgs=238 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=49a045f34c20e35c server_cookie=70fcc101ffd7585 in_seq=0 out_seq=0 2024-05-20T20:17:00.353 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.353+0000 7f273aa00640 10 client.27731 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:00.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.379+0000 7f273aa00640 1 -- 172.21.15.28:0/3841871168 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5622143792c0 con 0x56221449b870 2024-05-20T20:17:00.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.379+0000 7f273aa00640 10 client.27731 handle_command_reply: tid=0 2024-05-20T20:17:00.380 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.379+0000 7f275e7da740 1 client.27731 shutdown 2024-05-20T20:17:00.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 msgr2=0x56221441ae90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441ae90 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7f274c0ca630 tx=0x7f274c0c9e10 comp rx=0 tx=0).stop 2024-05-20T20:17:00.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 shutdown_connections 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56221449b870 0x56221449dc90 crc :-1 s=READY pgs=238 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562214420680 0x56221441ae90 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f273aa00640 0 client.27731 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562214415dc0 0x56221441a950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 --2- 172.21.15.28:0/3841871168 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5622144154a0 0x56221441a410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.380+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 >> 172.21.15.28:0/3841871168 conn(0x5622144119b0 msgr2=0x562214407a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.381+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 shutdown_connections 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.381+0000 7f275e7da740 1 -- 172.21.15.28:0/3841871168 wait complete. 2024-05-20T20:17:00.382 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.381+0000 7f275e7da740 20 client.27731 trim_cache size 0 max 16384 2024-05-20T20:17:00.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:17:00.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:17:00.383 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f273c0b2a90 msgr2=0x7f273c0b4f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f273c0b2a90 0x7f273c0b4f50 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x562214416a40 tx=0x7f2748005f90 comp rx=0 tx=0).stop 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275410f340 msgr2=0x7f275419fcc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275410f340 0x7f275419fcc0 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f274c006010 tx=0x7f274c031f40 comp rx=0 tx=0).stop 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 shutdown_connections 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f275410f340 0x7f275419fcc0 unknown :-1 s=CLOSED pgs=562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f273c0b2a90 0x7f273c0b4f50 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f2754075640 0x7f275419b990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 --2- 172.21.15.28:0/2413418378 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2754074d20 0x7f275419b450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 >> 172.21.15.28:0/2413418378 conn(0x7f2754079ee0 msgr2=0x7f27540ff910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:00.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 shutdown_connections 2024-05-20T20:17:00.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:00.382+0000 7f275ba00640 1 -- 172.21.15.28:0/2413418378 wait complete. 2024-05-20T20:17:01.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:01 smithi028 ceph-mon[31178]: pgmap v354: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 303 MiB/s wr, 2.12k op/s 2024-05-20T20:17:01.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:01 smithi154 ceph-mon[36314]: pgmap v354: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 303 MiB/s wr, 2.12k op/s 2024-05-20T20:17:01.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:01 smithi033 ceph-mon[36179]: pgmap v354: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 303 MiB/s wr, 2.12k op/s 2024-05-20T20:17:01.964 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:17:01.965 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:17:02.972 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:03.519 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:03 smithi028 ceph-mon[31178]: pgmap v355: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 237 MiB/s wr, 1.55k op/s 2024-05-20T20:17:03.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:03 smithi154 ceph-mon[36314]: pgmap v355: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 237 MiB/s wr, 1.55k op/s 2024-05-20T20:17:03.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:03 smithi033 ceph-mon[36179]: pgmap v355: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 237 MiB/s wr, 1.55k op/s 2024-05-20T20:17:05.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:05 smithi028 ceph-mon[31178]: pgmap v356: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 336 MiB/s wr, 2.03k op/s 2024-05-20T20:17:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:05 smithi154 ceph-mon[36314]: pgmap v356: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 336 MiB/s wr, 2.03k op/s 2024-05-20T20:17:05.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:05 smithi033 ceph-mon[36179]: pgmap v356: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 336 MiB/s wr, 2.03k op/s 2024-05-20T20:17:07.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:07 smithi028 ceph-mon[31178]: pgmap v357: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 267 MiB/s wr, 1.46k op/s 2024-05-20T20:17:07.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:07 smithi154 ceph-mon[36314]: pgmap v357: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 267 MiB/s wr, 1.46k op/s 2024-05-20T20:17:07.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:07 smithi033 ceph-mon[36179]: pgmap v357: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 267 MiB/s wr, 1.46k op/s 2024-05-20T20:17:08.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.519+0000 7fd3ee200700 1 -- 172.21.15.28:0/1584085242 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 msgr2=0x7fd3e810a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.519+0000 7fd3ee200700 1 --2- 172.21.15.28:0/1584085242 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 0x7fd3e810a1b0 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7fd3d8009990 tx=0x7fd3d8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.520+0000 7fd3ee200700 1 -- 172.21.15.28:0/1584085242 shutdown_connections 2024-05-20T20:17:08.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.520+0000 7fd3ee200700 1 --2- 172.21.15.28:0/1584085242 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd3e810eeb0 0x7fd3e8116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.520+0000 7fd3ee200700 1 --2- 172.21.15.28:0/1584085242 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 0x7fd3e810a1b0 unknown :-1 s=CLOSED pgs=565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.520+0000 7fd3ee200700 1 --2- 172.21.15.28:0/1584085242 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 0x7fd3e810e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.520+0000 7fd3ee200700 1 -- 172.21.15.28:0/1584085242 >> 172.21.15.28:0/1584085242 conn(0x7fd3e8076d10 msgr2=0x7fd3e8079130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:08.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.521+0000 7fd3ee200700 1 -- 172.21.15.28:0/1584085242 shutdown_connections 2024-05-20T20:17:08.522 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.521+0000 7fd3ee200700 1 -- 172.21.15.28:0/1584085242 wait complete. 2024-05-20T20:17:08.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.526+0000 7fd3ee200700 1 Processor -- start 2024-05-20T20:17:08.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.527+0000 7fd3ee200700 1 -- start start 2024-05-20T20:17:08.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.528+0000 7fd3ee200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 0x7fd3e807ec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.528+0000 7fd3ee200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 0x7fd3e807f160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.528+0000 7fd3ee200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd3e810eeb0 0x7fd3e8083540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.528+0000 7fd3ee200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd3e807d4b0 con 0x7fd3e810eeb0 2024-05-20T20:17:08.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.529+0000 7fd3ee200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd3e807d330 con 0x7fd3e8109dd0 2024-05-20T20:17:08.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.529+0000 7fd3dea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 0x7fd3e807f160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.529+0000 7fd3e7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 0x7fd3e807ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.529+0000 7fd3dea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 0x7fd3e807f160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38142/0 (socket says 172.21.15.28:38142) 2024-05-20T20:17:08.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.529+0000 7fd3dea00700 1 -- 172.21.15.28:0/760528421 learned_addr learned my addr 172.21.15.28:0/760528421 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:08.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.529+0000 7fd3ee200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd3e807d630 con 0x7fd3e810a780 2024-05-20T20:17:08.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.532+0000 7fd3dea00700 1 -- 172.21.15.28:0/760528421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 msgr2=0x7fd3e807ec00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.532+0000 7fd3dea00700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 0x7fd3e807ec00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.532+0000 7fd3dea00700 1 -- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd3e810eeb0 msgr2=0x7fd3e8083540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.532+0000 7fd3dea00700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd3e810eeb0 0x7fd3e8083540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.532+0000 7fd3dea00700 1 -- 172.21.15.28:0/760528421 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd3e8083ca0 con 0x7fd3e810a780 2024-05-20T20:17:08.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.533+0000 7fd3dea00700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 0x7fd3e807f160 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7fd3d000b550 tx=0x7fd3d000b910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:08.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.533+0000 7fd3e7400700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 0x7fd3e807ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:08.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.534+0000 7fd3e4c00700 1 -- 172.21.15.28:0/760528421 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3d0015020 con 0x7fd3e810a780 2024-05-20T20:17:08.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.534+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd3e8083f90 con 0x7fd3e810a780 2024-05-20T20:17:08.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.534+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd3e81b5680 con 0x7fd3e810a780 2024-05-20T20:17:08.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.535+0000 7fd3e4c00700 1 -- 172.21.15.28:0/760528421 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd3d0011ce0 con 0x7fd3e810a780 2024-05-20T20:17:08.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.535+0000 7fd3e4c00700 1 -- 172.21.15.28:0/760528421 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3d0010da0 con 0x7fd3e810a780 2024-05-20T20:17:08.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.554+0000 7fd3f8c58b80 1 Processor -- start 2024-05-20T20:17:08.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.556+0000 7fd3e4c00700 1 -- 172.21.15.28:0/760528421 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd3d0018020 con 0x7fd3e810a780 2024-05-20T20:17:08.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.560+0000 7fd3e4c00700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd3cc0b2b30 0x7fd3cc0b4ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.560+0000 7fd3e4c00700 1 -- 172.21.15.28:0/760528421 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fd3d00e1c40 con 0x7fd3e810a780 2024-05-20T20:17:08.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.562+0000 7fd3f8c58b80 1 -- start start 2024-05-20T20:17:08.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.562+0000 7fd3e7e00700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd3cc0b2b30 0x7fd3cc0b4ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.564+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249c310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.565+0000 7fd3e7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.565+0000 7fd3e7400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249c310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38146/0 (socket says 172.21.15.28:38146) 2024-05-20T20:17:08.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.566+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 0x55d2d24962e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.566+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d2496d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.566+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d2d23f4ec0 con 0x55d2d249c850 2024-05-20T20:17:08.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.566+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d2d23e4a60 con 0x55d2d2496900 2024-05-20T20:17:08.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.566+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d2d23e8aa0 con 0x55d2d249bf30 2024-05-20T20:17:08.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.568+0000 7fd3e7400700 1 -- 172.21.15.28:0/2031125581 learned_addr learned my addr 172.21.15.28:0/2031125581 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:08.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.568+0000 7fd3e7e00700 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d2496d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.569+0000 7fd3e7400700 1 -- 172.21.15.28:0/2031125581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 msgr2=0x55d2d2496d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.569+0000 7fd3e7400700 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d2496d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.569+0000 7fd3e7400700 1 -- 172.21.15.28:0/2031125581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 msgr2=0x55d2d24962e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.569+0000 7fd3e7400700 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 0x55d2d24962e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.571 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.569+0000 7fd3e7400700 1 -- 172.21.15.28:0/2031125581 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d2d23ad4a0 con 0x55d2d249bf30 2024-05-20T20:17:08.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.571+0000 7fd3e7400700 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249c310 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fd3d8005f30 tx=0x7fd3d8009630 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d334a4207d377c07 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:08.572 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.572+0000 7fd3e7e00700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd3cc0b2b30 0x7fd3cc0b4ff0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fd3e0007f80 tx=0x7fd3e0007550 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:08.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.573+0000 7fd3dcc00700 1 -- 172.21.15.28:0/2031125581 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3d80038f0 con 0x55d2d249bf30 2024-05-20T20:17:08.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.573+0000 7fd3dcc00700 1 -- 172.21.15.28:0/2031125581 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd3d80050c0 con 0x55d2d249bf30 2024-05-20T20:17:08.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.573+0000 7fd3dcc00700 1 -- 172.21.15.28:0/2031125581 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3d800bcd0 con 0x55d2d249bf30 2024-05-20T20:17:08.575 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.575+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2031125581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 msgr2=0x55d2d249c310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.575+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249c310 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fd3d8005f30 tx=0x7fd3d8009630 comp rx=0 tx=0).stop 2024-05-20T20:17:08.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.575+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2031125581 shutdown_connections 2024-05-20T20:17:08.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.575+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d2496d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.575+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249c310 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.575+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2031125581 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 0x55d2d24962e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.576 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.575+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2031125581 >> 172.21.15.28:0/2031125581 conn(0x55d2d23c4c60 msgr2=0x55d2d2491090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:08.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.576+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2031125581 shutdown_connections 2024-05-20T20:17:08.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.576+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2031125581 wait complete. 2024-05-20T20:17:08.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.577+0000 7fd3f8c58b80 1 -- start start 2024-05-20T20:17:08.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.579+0000 7fd3f8c58b80 1 Processor -- start 2024-05-20T20:17:08.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.580+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.580+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249f970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.580+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 0x55d2d249feb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.580+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55d2d22eb020 con 0x55d2d249c850 2024-05-20T20:17:08.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.580+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55d2d23857e0 con 0x55d2d2496900 2024-05-20T20:17:08.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.580+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55d2d24a3260 con 0x55d2d249bf30 2024-05-20T20:17:08.582 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.580+0000 7fd3f8c58b80 10 client.0 _ll_register_callbacks cb 0x55d2d238ecf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:08.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.582+0000 7fd3e7e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 0x55d2d249feb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.582+0000 7fd3dea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249f970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.582+0000 7fd3dea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249f970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38150/0 (socket says 172.21.15.28:38150) 2024-05-20T20:17:08.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.582+0000 7fd3dea00700 1 -- 172.21.15.28:0/3685080287 learned_addr learned my addr 172.21.15.28:0/3685080287 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:08.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.583+0000 7fd3e7400700 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.583+0000 7fd3dea00700 1 -- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 msgr2=0x55d2d24a25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.583+0000 7fd3dea00700 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a25d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.583+0000 7fd3dea00700 1 -- 172.21.15.28:0/3685080287 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 msgr2=0x55d2d249feb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.583+0000 7fd3dea00700 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 0x55d2d249feb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.583+0000 7fd3dea00700 1 -- 172.21.15.28:0/3685080287 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d2d2334950 con 0x55d2d249bf30 2024-05-20T20:17:08.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.585+0000 7fd3dea00700 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249f970 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fd3d00a5010 tx=0x7fd3d00df850 comp rx=0 tx=0).ready entity=mon.1 client_cookie=aa83f144ce9c8765 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:08.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.585+0000 7fd3d5600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:08.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.594+0000 7fd3d5600700 1 -- 172.21.15.28:0/3685080287 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3d00a8f90 con 0x55d2d249bf30 2024-05-20T20:17:08.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.595+0000 7fd3d5600700 1 -- 172.21.15.28:0/3685080287 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fd3d00b1b70 con 0x55d2d249bf30 2024-05-20T20:17:08.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.599+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd3d80050c0 con 0x55d2d249bf30 2024-05-20T20:17:08.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.600+0000 7fd3d5600700 1 -- 172.21.15.28:0/3685080287 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd3d00ad340 con 0x55d2d249bf30 2024-05-20T20:17:08.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.600+0000 7fd3f8c58b80 10 client.27676 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:08.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.600+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d2d23b50e0 con 0x55d2d249bf30 2024-05-20T20:17:08.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.601+0000 7fd3d5600700 1 -- 172.21.15.28:0/3685080287 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd3d80050c0 con 0x55d2d249bf30 2024-05-20T20:17:08.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.602+0000 7fd3d5600700 1 -- 172.21.15.28:0/3685080287 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd3d00a4900 con 0x55d2d249bf30 2024-05-20T20:17:08.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.603+0000 7fd3f8c58b80 10 client.27676 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:08.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.603+0000 7fd3f8c58b80 10 client.27676 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:08.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.603+0000 7fd3f8c58b80 20 client.27676 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:08.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.603+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24afef0 0x55d2d24b22e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.603+0000 7fd3f8c58b80 4 client.27676 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:08.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.603+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55d2d23f6e10 con 0x55d2d24afef0 2024-05-20T20:17:08.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.605+0000 7fd3e7400700 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24afef0 0x55d2d24b22e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.606+0000 7fd3e7400700 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24afef0 0x55d2d24b22e0 crc :-1 s=READY pgs=239 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=25ecf9a8ba420e68 server_cookie=6ea08b24a19d9fc5 in_seq=0 out_seq=0 2024-05-20T20:17:08.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.606+0000 7fd3d5600700 10 client.27676 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.607+0000 7fd3d5600700 1 -- 172.21.15.28:0/3685080287 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55d2d23f6e10 con 0x55d2d24afef0 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.607+0000 7fd3d5600700 10 client.27676 handle_command_reply: tid=0 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.607+0000 7fd3f8c58b80 1 client.27676 shutdown 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 msgr2=0x55d2d249f970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249f970 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fd3d00a5010 tx=0x7fd3d00df850 comp rx=0 tx=0).stop 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 shutdown_connections 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24afef0 0x55d2d24b22e0 crc :-1 s=READY pgs=239 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249c850 0x55d2d249feb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a25d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3685080287 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d249f970 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 >> 172.21.15.28:0/3685080287 conn(0x55d2d23c4c60 msgr2=0x55d2d24952d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:08.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3d5600700 0 client.27676 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:08.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 shutdown_connections 2024-05-20T20:17:08.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.608+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3685080287 wait complete. 2024-05-20T20:17:08.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.609+0000 7fd3f8c58b80 20 client.27676 trim_cache size 0 max 16384 2024-05-20T20:17:08.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.655+0000 7fd3f8c58b80 1 Processor -- start 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.659+0000 7fd3f8c58b80 1 -- start start 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.659+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d2496900 0x55d2d24ac390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.659+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d24962e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.659+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d249c850 0x55d2d249a150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.659+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c400d780 con 0x55d2d2496900 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.659+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c400f2c0 con 0x55d2d249c850 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.659+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c400de50 con 0x55d2d249bf30 2024-05-20T20:17:08.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.660+0000 7fd3dea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d24962e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.660+0000 7fd3e7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d249c850 0x55d2d249a150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.661+0000 7fd3e7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d249c850 0x55d2d249a150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57862/0 (socket says 172.21.15.28:57862) 2024-05-20T20:17:08.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.661+0000 7fd3dea00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d24962e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38152/0 (socket says 172.21.15.28:38152) 2024-05-20T20:17:08.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.661+0000 7fd3dea00700 1 -- 172.21.15.28:0/2709140763 learned_addr learned my addr 172.21.15.28:0/2709140763 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:08.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.662+0000 7fd3e7e00700 1 -- 172.21.15.28:0/2709140763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 msgr2=0x55d2d24962e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.662+0000 7fd3e7e00700 1 --2- 172.21.15.28:0/2709140763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d24962e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.662+0000 7fd3e7e00700 1 -- 172.21.15.28:0/2709140763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d2496900 msgr2=0x55d2d24ac390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.662+0000 7fd3e7e00700 1 --2- 172.21.15.28:0/2709140763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d2496900 0x55d2d24ac390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.662+0000 7fd3e7e00700 1 -- 172.21.15.28:0/2709140763 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55d2d2491db0 con 0x55d2d249c850 2024-05-20T20:17:08.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.664+0000 7fd3e7e00700 1 --2- 172.21.15.28:0/2709140763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d249c850 0x55d2d249a150 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7fd3e0015550 tx=0x7fd3e0013000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=455f7243e510dfc4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:08.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.665+0000 7fd3d7e00700 1 -- 172.21.15.28:0/2709140763 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3e0026070 con 0x55d2d249c850 2024-05-20T20:17:08.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.665+0000 7fd3d7e00700 1 -- 172.21.15.28:0/2709140763 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd3e0015b70 con 0x55d2d249c850 2024-05-20T20:17:08.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.666+0000 7fd3d7e00700 1 -- 172.21.15.28:0/2709140763 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3e0014600 con 0x55d2d249c850 2024-05-20T20:17:08.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2709140763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d249c850 msgr2=0x55d2d249a150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2709140763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d249c850 0x55d2d249a150 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7fd3e0015550 tx=0x7fd3e0013000 comp rx=0 tx=0).stop 2024-05-20T20:17:08.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2709140763 shutdown_connections 2024-05-20T20:17:08.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2709140763 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d249c850 0x55d2d249a150 unknown :-1 s=CLOSED pgs=566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2709140763 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d2496900 0x55d2d24ac390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/2709140763 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249bf30 0x55d2d24962e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2709140763 >> 172.21.15.28:0/2709140763 conn(0x55d2d23c4c60 msgr2=0x55d2d2499a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:08.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2709140763 shutdown_connections 2024-05-20T20:17:08.668 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.667+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/2709140763 wait complete. 2024-05-20T20:17:08.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.668+0000 7fd3f8c58b80 1 -- start start 2024-05-20T20:17:08.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.669+0000 7fd3f8c58b80 1 Processor -- start 2024-05-20T20:17:08.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.670+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a87e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.670+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249bf30 0x55d2d24a8d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.670+0000 7fd3f8c58b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249c850 0x55d2d2515ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.670+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c400ebf0 con 0x55d2d249bf30 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.670+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c400e520 con 0x55d2d2496900 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.670+0000 7fd3f8c58b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd3c400b740 con 0x55d2d249c850 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.670+0000 7fd3f8c58b80 10 client.0 _ll_register_callbacks cb 0x55d2d238ecf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.671+0000 7fd3e7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a87e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.671+0000 7fd3e7400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a87e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57872/0 (socket says 172.21.15.28:57872) 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.671+0000 7fd3e7400700 1 -- 172.21.15.28:0/3836811564 learned_addr learned my addr 172.21.15.28:0/3836811564 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.671+0000 7fd3e7e00700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249c850 0x55d2d2515ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.672+0000 7fd3e7400700 1 -- 172.21.15.28:0/3836811564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249c850 msgr2=0x55d2d2515ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.672+0000 7fd3e7400700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249c850 0x55d2d2515ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.672+0000 7fd3e7400700 1 -- 172.21.15.28:0/3836811564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249bf30 msgr2=0x55d2d24a8d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:08.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.672+0000 7fd3dea00700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249bf30 0x55d2d24a8d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.672+0000 7fd3e7400700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249bf30 0x55d2d24a8d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.672+0000 7fd3e7400700 1 -- 172.21.15.28:0/3836811564 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55d2d25164e0 con 0x55d2d2496900 2024-05-20T20:17:08.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.672+0000 7fd3dea00700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249bf30 0x55d2d24a8d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:08.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.673+0000 7fd3d5600700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:17:08.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.673+0000 7fd3e7400700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a87e0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7fd3d8009750 tx=0x7fd3d8009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5545405515f175a2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:08.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.673+0000 7fd3d5600700 1 -- 172.21.15.28:0/3836811564 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd3d800b5c0 con 0x55d2d2496900 2024-05-20T20:17:08.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.674+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd3e0015b70 con 0x55d2d2496900 2024-05-20T20:17:08.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.674+0000 7fd3d5600700 1 -- 172.21.15.28:0/3836811564 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fd3d8031060 con 0x55d2d2496900 2024-05-20T20:17:08.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.675+0000 7fd3d5600700 1 -- 172.21.15.28:0/3836811564 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd3d8005570 con 0x55d2d2496900 2024-05-20T20:17:08.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.675+0000 7fd3f8c58b80 10 client.27743 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:08.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.675+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55d2d2515820 con 0x55d2d2496900 2024-05-20T20:17:08.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.676+0000 7fd3d5600700 1 -- 172.21.15.28:0/3836811564 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd3e0015b70 con 0x55d2d2496900 2024-05-20T20:17:08.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.676+0000 7fd3d5600700 1 -- 172.21.15.28:0/3836811564 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd3d802c620 con 0x55d2d2496900 2024-05-20T20:17:08.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.676+0000 7fd3f8c58b80 10 client.27743 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:08.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.676+0000 7fd3f8c58b80 10 client.27743 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:08.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.676+0000 7fd3f8c58b80 20 client.27743 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:08.677 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.677+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24a75e0 0x55d2d24a79f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:08.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.677+0000 7fd3f8c58b80 4 client.27743 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:17:08.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.677+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x55d2d23f05f0 con 0x55d2d24a75e0 2024-05-20T20:17:08.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.678+0000 7fd3dea00700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24a75e0 0x55d2d24a79f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:08.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.680+0000 7fd3dea00700 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24a75e0 0x55d2d24a79f0 crc :-1 s=READY pgs=240 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fec7dde16c545f29 server_cookie=500c4f54702884a2 in_seq=0 out_seq=0 2024-05-20T20:17:08.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.680+0000 7fd3d5600700 10 client.27743 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:08.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.682+0000 7fd3d5600700 1 -- 172.21.15.28:0/3836811564 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x55d2d23f05f0 con 0x55d2d24a75e0 2024-05-20T20:17:08.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.682+0000 7fd3d5600700 10 client.27743 handle_command_reply: tid=0 2024-05-20T20:17:08.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.682+0000 7fd3f8c58b80 1 client.27743 shutdown 2024-05-20T20:17:08.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.682+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 msgr2=0x55d2d24a87e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.682+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a87e0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7fd3d8009750 tx=0x7fd3d8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 shutdown_connections 2024-05-20T20:17:08.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55d2d24a75e0 0x55d2d24a79f0 crc :-1 s=READY pgs=240 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55d2d249c850 0x55d2d2515ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55d2d2496900 0x55d2d24a87e0 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 --2- 172.21.15.28:0/3836811564 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55d2d249bf30 0x55d2d24a8d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 >> 172.21.15.28:0/3836811564 conn(0x55d2d23c4c60 msgr2=0x55d2d249f750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:08.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3d5600700 0 client.27743 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:08.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 shutdown_connections 2024-05-20T20:17:08.692 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:17:08.692 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:17:08.692 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "ac73e0cd-45d4-428a-8d06-3ca1736db730", 2024-05-20T20:17:08.692 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:17:08.692 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.683+0000 7fd3f8c58b80 1 -- 172.21.15.28:0/3836811564 wait complete. 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.684+0000 7fd3f8c58b80 20 client.27743 trim_cache size 0 max 16384 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.688+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd3cc0b2b30 msgr2=0x7fd3cc0b4ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.688+0000 7fd3ee200700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd3cc0b2b30 0x7fd3cc0b4ff0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fd3e0007f80 tx=0x7fd3e0007550 comp rx=0 tx=0).stop 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.688+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 msgr2=0x7fd3e807f160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.688+0000 7fd3ee200700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 0x7fd3e807f160 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7fd3d000b550 tx=0x7fd3d000b910 comp rx=0 tx=0).stop 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 shutdown_connections 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd3e810eeb0 0x7fd3e8083540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd3cc0b2b30 0x7fd3cc0b4ff0 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd3e8109dd0 0x7fd3e807ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 --2- 172.21.15.28:0/760528421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3e810a780 0x7fd3e807f160 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:08.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 >> 172.21.15.28:0/760528421 conn(0x7fd3e8076d10 msgr2=0x7fd3e8079010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:08.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 shutdown_connections 2024-05-20T20:17:08.694 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:08.689+0000 7fd3ee200700 1 -- 172.21.15.28:0/760528421 wait complete. 2024-05-20T20:17:09.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:09 smithi028 ceph-mon[31178]: pgmap v358: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 264 MiB/s wr, 1.42k op/s 2024-05-20T20:17:09.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:09 smithi028 ceph-mon[31178]: osdmap e92: 12 total, 12 up, 12 in 2024-05-20T20:17:09.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:09 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:17:09.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:09 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:17:09.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:09 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:17:09.558 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:09 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:17:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:09 smithi154 ceph-mon[36314]: pgmap v358: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 264 MiB/s wr, 1.42k op/s 2024-05-20T20:17:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:09 smithi154 ceph-mon[36314]: osdmap e92: 12 total, 12 up, 12 in 2024-05-20T20:17:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:09 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:17:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:09 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:17:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:09 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:17:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:09 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:17:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:09 smithi033 ceph-mon[36179]: pgmap v358: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 264 MiB/s wr, 1.42k op/s 2024-05-20T20:17:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:09 smithi033 ceph-mon[36179]: osdmap e92: 12 total, 12 up, 12 in 2024-05-20T20:17:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:09 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:17:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:09 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:17:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:09 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:17:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:09 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:17:11.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:11 smithi028 ceph-mon[31178]: pgmap v360: 97 pgs: 18 active+clean+snaptrim, 29 active+clean+snaptrim_wait, 50 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 301 MiB/s wr, 1.61k op/s 2024-05-20T20:17:11.581 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:17:11.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:11 smithi154 ceph-mon[36314]: pgmap v360: 97 pgs: 18 active+clean+snaptrim, 29 active+clean+snaptrim_wait, 50 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 301 MiB/s wr, 1.61k op/s 2024-05-20T20:17:11.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:11 smithi033 ceph-mon[36179]: pgmap v360: 97 pgs: 18 active+clean+snaptrim, 29 active+clean+snaptrim_wait, 50 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 301 MiB/s wr, 1.61k op/s 2024-05-20T20:17:12.798 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:13.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:13 smithi028 ceph-mon[31178]: pgmap v361: 97 pgs: 20 active+clean+snaptrim, 30 active+clean+snaptrim_wait, 47 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 299 MiB/s wr, 1.61k op/s 2024-05-20T20:17:13.286 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:13 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:17:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:13 smithi154 ceph-mon[36314]: pgmap v361: 97 pgs: 20 active+clean+snaptrim, 30 active+clean+snaptrim_wait, 47 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 299 MiB/s wr, 1.61k op/s 2024-05-20T20:17:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:13 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:17:13.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:13 smithi033 ceph-mon[36179]: pgmap v361: 97 pgs: 20 active+clean+snaptrim, 30 active+clean+snaptrim_wait, 47 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 299 MiB/s wr, 1.61k op/s 2024-05-20T20:17:13.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:13 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:17:13.726 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.721+0000 7fd473c00640 1 -- 172.21.15.28:0/604148364 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c100ee0 msgr2=0x7fd46c1012c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:13.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.721+0000 7fd473c00640 1 --2- 172.21.15.28:0/604148364 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c100ee0 0x7fd46c1012c0 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7fd464009900 tx=0x7fd46402a400 comp rx=0 tx=0).stop 2024-05-20T20:17:13.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.724+0000 7fd473c00640 1 -- 172.21.15.28:0/604148364 shutdown_connections 2024-05-20T20:17:13.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.724+0000 7fd473c00640 1 --2- 172.21.15.28:0/604148364 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46c105fb0 0x7fd46c10db80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.725+0000 7fd473c00640 1 --2- 172.21.15.28:0/604148364 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c101890 0x7fd46c1058d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.725+0000 7fd473c00640 1 --2- 172.21.15.28:0/604148364 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c100ee0 0x7fd46c1012c0 unknown :-1 s=CLOSED pgs=569 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.725+0000 7fd473c00640 1 -- 172.21.15.28:0/604148364 >> 172.21.15.28:0/604148364 conn(0x7fd46c077a50 msgr2=0x7fd46c0fee30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:13.727 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.725+0000 7fd473c00640 1 -- 172.21.15.28:0/604148364 shutdown_connections 2024-05-20T20:17:13.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.725+0000 7fd473c00640 1 -- 172.21.15.28:0/604148364 wait complete. 2024-05-20T20:17:13.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.729+0000 7fd473c00640 1 Processor -- start 2024-05-20T20:17:13.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.731+0000 7fd473c00640 1 -- start start 2024-05-20T20:17:13.738 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.737+0000 7fd473c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c100ee0 0x7fd46c118270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.738+0000 7fd473c00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c105fb0 0x7fd46c1187b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.740 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.739+0000 7fd473c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46c111400 0x7fd46c1118b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.740+0000 7fd46be00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c105fb0 0x7fd46c1187b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.741+0000 7fd471200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c100ee0 0x7fd46c118270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.741+0000 7fd471200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c100ee0 0x7fd46c118270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38192/0 (socket says 172.21.15.28:38192) 2024-05-20T20:17:13.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.743+0000 7fd46be00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c105fb0 0x7fd46c1187b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:57900/0 (socket says 172.21.15.28:57900) 2024-05-20T20:17:13.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.743+0000 7fd473c00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c119380 con 0x7fd46c111400 2024-05-20T20:17:13.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.743+0000 7fd473c00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c119200 con 0x7fd46c105fb0 2024-05-20T20:17:13.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.743+0000 7fd473c00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c119500 con 0x7fd46c100ee0 2024-05-20T20:17:13.745 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.743+0000 7fd471200640 1 -- 172.21.15.28:0/930664258 learned_addr learned my addr 172.21.15.28:0/930664258 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:13.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.746+0000 7fd471200640 1 -- 172.21.15.28:0/930664258 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c105fb0 msgr2=0x7fd46c1187b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:13.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.746+0000 7fd471200640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c105fb0 0x7fd46c1187b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.746+0000 7fd471200640 1 -- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46c111400 msgr2=0x7fd46c1118b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:13.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.746+0000 7fd471200640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46c111400 0x7fd46c1118b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.748 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.746+0000 7fd471200640 1 -- 172.21.15.28:0/930664258 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd46c115d90 con 0x7fd46c100ee0 2024-05-20T20:17:13.750 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.749+0000 7fd471200640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c100ee0 0x7fd46c118270 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fd464005ab0 tx=0x7fd464003d50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:13.751 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.750+0000 7fd469600640 1 -- 172.21.15.28:0/930664258 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd46403f070 con 0x7fd46c100ee0 2024-05-20T20:17:13.751 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.751+0000 7fd469600640 1 -- 172.21.15.28:0/930664258 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd464031d60 con 0x7fd46c100ee0 2024-05-20T20:17:13.752 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.751+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd46c116020 con 0x7fd46c100ee0 2024-05-20T20:17:13.754 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.753+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd46c116510 con 0x7fd46c100ee0 2024-05-20T20:17:13.754 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.754+0000 7fd469600640 1 -- 172.21.15.28:0/930664258 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd464035030 con 0x7fd46c100ee0 2024-05-20T20:17:13.790 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.785+0000 7fd469600640 1 -- 172.21.15.28:0/930664258 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fd464042070 con 0x7fd46c100ee0 2024-05-20T20:17:13.794 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.793+0000 7fd476a22740 1 Processor -- start 2024-05-20T20:17:13.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.796+0000 7fd476a22740 1 -- start start 2024-05-20T20:17:13.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.804+0000 7fd469600640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4500b2bd0 0x7fd4500b5090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.804+0000 7fd469600640 1 -- 172.21.15.28:0/930664258 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fd464102c80 con 0x7fd46c100ee0 2024-05-20T20:17:13.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.805+0000 7fd476a22740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959bdc90 0x55cc959be070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.806 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.806+0000 7fd471c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4500b2bd0 0x7fd4500b5090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.808 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.807+0000 7fd471200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959bdc90 0x55cc959be070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.808 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.808+0000 7fd471200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959bdc90 0x55cc959be070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38198/0 (socket says 172.21.15.28:38198) 2024-05-20T20:17:13.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.808+0000 7fd471200640 1 -- 172.21.15.28:0/3896614467 learned_addr learned my addr 172.21.15.28:0/3896614467 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:13.809 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.809+0000 7fd476a22740 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959c8930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.811 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.809+0000 7fd476a22740 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959c8e70 0x55cc959cb260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.812 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.809+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cc9592eb90 con 0x55cc959c8e70 2024-05-20T20:17:13.812 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.809+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cc9591e730 con 0x55cc959be5b0 2024-05-20T20:17:13.812 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.809+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cc95922770 con 0x55cc959bdc90 2024-05-20T20:17:13.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.813+0000 7fd46be00640 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959c8930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.814+0000 7fd471200640 1 -- 172.21.15.28:0/3896614467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 msgr2=0x55cc959c8930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:13.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.814+0000 7fd471200640 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959c8930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.814+0000 7fd471200640 1 -- 172.21.15.28:0/3896614467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959c8e70 msgr2=0x55cc959cb260 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:17:13.816 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.816+0000 7fd471c00640 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959c8e70 0x55cc959cb260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.816+0000 7fd471200640 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959c8e70 0x55cc959cb260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.816+0000 7fd471200640 1 -- 172.21.15.28:0/3896614467 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cc958b78b0 con 0x55cc959bdc90 2024-05-20T20:17:13.817 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.816+0000 7fd471c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4500b2bd0 0x7fd4500b5090 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fd45c00ab10 tx=0x7fd45c009250 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:13.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.818+0000 7fd471200640 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959bdc90 0x55cc959be070 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fd4640c5f40 tx=0x7fd464031f10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f6d96a389c3873d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:13.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.820+0000 7fd462000640 1 -- 172.21.15.28:0/3896614467 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd464049040 con 0x55cc959bdc90 2024-05-20T20:17:13.820 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.820+0000 7fd462000640 1 -- 172.21.15.28:0/3896614467 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd4640c9ec0 con 0x55cc959bdc90 2024-05-20T20:17:13.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.822+0000 7fd462000640 1 -- 172.21.15.28:0/3896614467 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd4640ec040 con 0x55cc959bdc90 2024-05-20T20:17:13.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.822+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959bdc90 msgr2=0x55cc959be070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:13.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.822+0000 7fd476a22740 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959bdc90 0x55cc959be070 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fd4640c5f40 tx=0x7fd464031f10 comp rx=0 tx=0).stop 2024-05-20T20:17:13.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.823+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 shutdown_connections 2024-05-20T20:17:13.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.823+0000 7fd476a22740 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959c8930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.823+0000 7fd476a22740 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959c8e70 0x55cc959cb260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.823+0000 7fd476a22740 1 --2- 172.21.15.28:0/3896614467 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959bdc90 0x55cc959be070 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.824 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.823+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 >> 172.21.15.28:0/3896614467 conn(0x55cc959ba1a0 msgr2=0x55cc959ba580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:13.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.823+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 shutdown_connections 2024-05-20T20:17:13.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.824+0000 7fd476a22740 1 -- 172.21.15.28:0/3896614467 wait complete. 2024-05-20T20:17:13.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.825+0000 7fd476a22740 1 -- start start 2024-05-20T20:17:13.827 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.826+0000 7fd476a22740 1 Processor -- start 2024-05-20T20:17:13.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.830+0000 7fd476a22740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959bdc90 0x55cc959c87a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.830 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.830+0000 7fd476a22740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959be5b0 0x55cc959c5a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.830+0000 7fd476a22740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959c5fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd476a22740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55cc95604790 con 0x55cc959be5b0 2024-05-20T20:17:13.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd471200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959bdc90 0x55cc959c87a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.831 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd476a22740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55cc958c6bc0 con 0x55cc959bdc90 2024-05-20T20:17:13.832 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd471c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959c5fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.832 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd476a22740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55cc959cb860 con 0x55cc959c8e70 2024-05-20T20:17:13.837 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd471c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959c5fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38204/0 (socket says 172.21.15.28:38204) 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd471c00640 1 -- 172.21.15.28:0/1630954749 learned_addr learned my addr 172.21.15.28:0/1630954749 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.831+0000 7fd476a22740 10 client.0 _ll_register_callbacks cb 0x55cc958a4400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.832+0000 7fd471c00640 1 -- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959bdc90 msgr2=0x55cc959c87a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.832+0000 7fd46be00640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959be5b0 0x55cc959c5a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.837+0000 7fd471c00640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959bdc90 0x55cc959c87a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.837+0000 7fd471c00640 1 -- 172.21.15.28:0/1630954749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959be5b0 msgr2=0x55cc959c5a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.837+0000 7fd471c00640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959be5b0 0x55cc959c5a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.838 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.837+0000 7fd471c00640 1 -- 172.21.15.28:0/1630954749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cc95633210 con 0x55cc959c8e70 2024-05-20T20:17:13.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.837+0000 7fd46be00640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959be5b0 0x55cc959c5a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:13.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.837+0000 7fd471200640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959bdc90 0x55cc959c87a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:13.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.838+0000 7fd45aa00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:13.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.838+0000 7fd471c00640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959c5fd0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7fd45c00ddf0 tx=0x7fd45c004bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bf435bfe418ea472 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:13.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.838+0000 7fd45aa00640 1 -- 172.21.15.28:0/1630954749 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd45c00cdd0 con 0x55cc959c8e70 2024-05-20T20:17:13.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.839+0000 7fd45aa00640 1 -- 172.21.15.28:0/1630954749 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fd45c01a420 con 0x55cc959c8e70 2024-05-20T20:17:13.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.839+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd4640c9ec0 con 0x55cc959c8e70 2024-05-20T20:17:13.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.840+0000 7fd45aa00640 1 -- 172.21.15.28:0/1630954749 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd45c01a620 con 0x55cc959c8e70 2024-05-20T20:17:13.841 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.841+0000 7fd476a22740 10 client.27691 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:13.845 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.842+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cc956ff620 con 0x55cc959c8e70 2024-05-20T20:17:13.850 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.847+0000 7fd45aa00640 1 -- 172.21.15.28:0/1630954749 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd4640c9ec0 con 0x55cc959c8e70 2024-05-20T20:17:13.850 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.847+0000 7fd45aa00640 1 -- 172.21.15.28:0/1630954749 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd45c017a20 con 0x55cc959c8e70 2024-05-20T20:17:13.850 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.849+0000 7fd476a22740 10 client.27691 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:13.850 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.849+0000 7fd476a22740 10 client.27691 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:13.850 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.849+0000 7fd476a22740 20 client.27691 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:13.850 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.849+0000 7fd476a22740 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc959d86c0 0x55cc959daab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:13.851 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.849+0000 7fd476a22740 4 client.27691 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:13.851 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.849+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55cc95930ae0 con 0x55cc959d86c0 2024-05-20T20:17:13.851 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.851+0000 7fd471200640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc959d86c0 0x55cc959daab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:13.856 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.855+0000 7fd471200640 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc959d86c0 0x55cc959daab0 crc :-1 s=READY pgs=241 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2fff35807826ed2d server_cookie=c30ef77f0afce048 in_seq=0 out_seq=0 2024-05-20T20:17:13.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.856+0000 7fd45aa00640 10 client.27691 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:13.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.857+0000 7fd45aa00640 1 -- 172.21.15.28:0/1630954749 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55cc95930ae0 con 0x55cc959d86c0 2024-05-20T20:17:13.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.857+0000 7fd45aa00640 10 client.27691 handle_command_reply: tid=0 2024-05-20T20:17:13.857 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.857+0000 7fd476a22740 1 client.27691 shutdown 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.858+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 msgr2=0x55cc959c5fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.858+0000 7fd476a22740 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959c5fd0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7fd45c00ddf0 tx=0x7fd45c004bc0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.858+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 shutdown_connections 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.859+0000 7fd476a22740 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc959d86c0 0x55cc959daab0 crc :-1 s=READY pgs=241 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.859+0000 7fd476a22740 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959c5fd0 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.859+0000 7fd476a22740 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959be5b0 0x55cc959c5a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.859+0000 7fd476a22740 1 --2- 172.21.15.28:0/1630954749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959bdc90 0x55cc959c87a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:13.859 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.859+0000 7fd45aa00640 0 client.27691 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:13.860 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.859+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 >> 172.21.15.28:0/1630954749 conn(0x55cc959ba1a0 msgr2=0x55cc959bfed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:13.862 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.861+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 shutdown_connections 2024-05-20T20:17:13.863 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.862+0000 7fd476a22740 1 -- 172.21.15.28:0/1630954749 wait complete. 2024-05-20T20:17:13.863 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:13.862+0000 7fd476a22740 20 client.27691 trim_cache size 0 max 16384 2024-05-20T20:17:14.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.085+0000 7fd476a22740 1 Processor -- start 2024-05-20T20:17:14.086 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.086+0000 7fd476a22740 1 -- start start 2024-05-20T20:17:14.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.086+0000 7fd476a22740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959d4910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:14.087 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.086+0000 7fd476a22740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959e03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:14.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.087+0000 7fd471200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959d4910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:14.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.087+0000 7fd471200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959d4910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60920/0 (socket says 172.21.15.28:60920) 2024-05-20T20:17:14.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.087+0000 7fd476a22740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959e08e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:14.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.087+0000 7fd476a22740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c1bd5a0 con 0x55cc959bdc90 2024-05-20T20:17:14.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.087+0000 7fd476a22740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c1bf0e0 con 0x55cc959be5b0 2024-05-20T20:17:14.088 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.087+0000 7fd476a22740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c1bdc70 con 0x55cc959c8e70 2024-05-20T20:17:14.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.087+0000 7fd471200640 1 -- 172.21.15.28:0/2249728189 learned_addr learned my addr 172.21.15.28:0/2249728189 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:14.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.088+0000 7fd46be00640 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959e03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:14.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.088+0000 7fd471200640 1 -- 172.21.15.28:0/2249728189 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 msgr2=0x55cc959e08e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:17:14.089 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.088+0000 7fd471200640 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959e08e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:14.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.088+0000 7fd471200640 1 -- 172.21.15.28:0/2249728189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 msgr2=0x55cc959e03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:14.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.088+0000 7fd471200640 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959e03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:14.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.088+0000 7fd471200640 1 -- 172.21.15.28:0/2249728189 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55cc959c5a00 con 0x55cc959bdc90 2024-05-20T20:17:14.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.088+0000 7fd471200640 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959d4910 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fd46403b8d0 tx=0x7fd4640ca700 comp rx=0 tx=0).ready entity=mon.0 client_cookie=87d79e029a3251c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:14.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.089+0000 7fd461600640 1 -- 172.21.15.28:0/2249728189 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd464049070 con 0x55cc959bdc90 2024-05-20T20:17:14.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.089+0000 7fd461600640 1 -- 172.21.15.28:0/2249728189 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd4640c64b0 con 0x55cc959bdc90 2024-05-20T20:17:14.090 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.089+0000 7fd461600640 1 -- 172.21.15.28:0/2249728189 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd4640ec040 con 0x55cc959bdc90 2024-05-20T20:17:14.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.091+0000 7fd476a22740 1 -- 172.21.15.28:0/2249728189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 msgr2=0x55cc959d4910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:14.092 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.091+0000 7fd476a22740 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959d4910 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fd46403b8d0 tx=0x7fd4640ca700 comp rx=0 tx=0).stop 2024-05-20T20:17:14.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.092+0000 7fd476a22740 1 -- 172.21.15.28:0/2249728189 shutdown_connections 2024-05-20T20:17:14.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.092+0000 7fd476a22740 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959e03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:14.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.092+0000 7fd476a22740 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc959e08e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:14.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.092+0000 7fd476a22740 1 --2- 172.21.15.28:0/2249728189 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959d4910 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:14.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.092+0000 7fd476a22740 1 -- 172.21.15.28:0/2249728189 >> 172.21.15.28:0/2249728189 conn(0x55cc959ba1a0 msgr2=0x55cc959b8a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:14.093 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.093+0000 7fd476a22740 1 -- 172.21.15.28:0/2249728189 shutdown_connections 2024-05-20T20:17:14.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.093+0000 7fd476a22740 1 -- 172.21.15.28:0/2249728189 wait complete. 2024-05-20T20:17:14.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.093+0000 7fd476a22740 1 -- start start 2024-05-20T20:17:14.094 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.094+0000 7fd476a22740 1 Processor -- start 2024-05-20T20:17:14.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.095+0000 7fd476a22740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959ce0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:14.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.095+0000 7fd476a22740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959ce610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:14.096 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.095+0000 7fd476a22740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc95a41270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:14.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.096+0000 7fd471c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc95a41270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:14.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.096+0000 7fd471c00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc95a41270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38216/0 (socket says 172.21.15.28:38216) 2024-05-20T20:17:14.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.096+0000 7fd471c00640 1 -- 172.21.15.28:0/3413173928 learned_addr learned my addr 172.21.15.28:0/3413173928 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:14.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.096+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c1bea10 con 0x55cc959bdc90 2024-05-20T20:17:14.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.096+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c1be340 con 0x55cc959be5b0 2024-05-20T20:17:14.097 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.096+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fd46c1bb5a0 con 0x55cc959c8e70 2024-05-20T20:17:14.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.097+0000 7fd471200640 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959ce0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:14.098 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.097+0000 7fd476a22740 10 client.0 _ll_register_callbacks cb 0x7fd45c00cb20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:14.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.097+0000 7fd471c00640 1 -- 172.21.15.28:0/3413173928 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 msgr2=0x55cc959ce610 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:17:14.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.097+0000 7fd471c00640 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959ce610 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:14.099 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.097+0000 7fd471c00640 1 -- 172.21.15.28:0/3413173928 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 msgr2=0x55cc959ce0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.097+0000 7fd471c00640 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959ce0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.098+0000 7fd471c00640 1 -- 172.21.15.28:0/3413173928 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55cc95a417b0 con 0x55cc959c8e70 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.098+0000 7fd471200640 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959ce0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.098+0000 7fd471c00640 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc95a41270 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fd45c022440 tx=0x7fd45c00ce00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a2926b026b17a886 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.098+0000 7fd45aa00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.099+0000 7fd45aa00640 1 -- 172.21.15.28:0/3413173928 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd45c01a420 con 0x55cc959c8e70 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.099+0000 7fd45aa00640 1 -- 172.21.15.28:0/3413173928 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fd45c022a80 con 0x55cc959c8e70 2024-05-20T20:17:14.100 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.100+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fd4640c64b0 con 0x55cc959c8e70 2024-05-20T20:17:14.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.100+0000 7fd45aa00640 1 -- 172.21.15.28:0/3413173928 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fd45c022c80 con 0x55cc959c8e70 2024-05-20T20:17:14.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.100+0000 7fd476a22740 10 client.27694 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:14.101 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.100+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55cc95a429a0 con 0x55cc959c8e70 2024-05-20T20:17:14.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.101+0000 7fd45aa00640 1 -- 172.21.15.28:0/3413173928 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fd4640c64b0 con 0x55cc959c8e70 2024-05-20T20:17:14.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.101+0000 7fd45aa00640 1 -- 172.21.15.28:0/3413173928 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fd45c0177d0 con 0x55cc959c8e70 2024-05-20T20:17:14.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.101+0000 7fd476a22740 10 client.27694 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:14.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.101+0000 7fd476a22740 10 client.27694 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:14.102 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.101+0000 7fd476a22740 20 client.27694 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:14.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.102+0000 7fd476a22740 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc95a43010 0x55cc95a45400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:14.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.102+0000 7fd476a22740 4 client.27694 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:17:14.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.102+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55cc9592a2c0 con 0x55cc95a43010 2024-05-20T20:17:14.103 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.103+0000 7fd471200640 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc95a43010 0x55cc95a45400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:14.104 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.104+0000 7fd471200640 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc95a43010 0x55cc95a45400 crc :-1 s=READY pgs=242 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5624ef8d4d44065d server_cookie=28bb9f53c8c475b7 in_seq=0 out_seq=0 2024-05-20T20:17:14.105 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:14.104+0000 7fd45aa00640 10 client.27694 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:15.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:15 smithi028 ceph-mon[31178]: pgmap v362: 97 pgs: 8 active+clean+snaptrim, 17 active+clean+snaptrim_wait, 72 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 279 MiB/s wr, 1.82k op/s 2024-05-20T20:17:15.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:15 smithi154 ceph-mon[36314]: pgmap v362: 97 pgs: 8 active+clean+snaptrim, 17 active+clean+snaptrim_wait, 72 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 279 MiB/s wr, 1.82k op/s 2024-05-20T20:17:15.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:15 smithi033 ceph-mon[36179]: pgmap v362: 97 pgs: 8 active+clean+snaptrim, 17 active+clean+snaptrim_wait, 72 active+clean; 56 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 279 MiB/s wr, 1.82k op/s 2024-05-20T20:17:17.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:17 smithi028 ceph-mon[31178]: pgmap v363: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 272 MiB/s wr, 1.85k op/s 2024-05-20T20:17:17.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:17 smithi154 ceph-mon[36314]: pgmap v363: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 272 MiB/s wr, 1.85k op/s 2024-05-20T20:17:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:17 smithi033 ceph-mon[36179]: pgmap v363: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 272 MiB/s wr, 1.85k op/s 2024-05-20T20:17:18.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.060+0000 7fd45aa00640 1 -- 172.21.15.28:0/3413173928 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55cc9592a2c0 con 0x55cc95a43010 2024-05-20T20:17:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.060+0000 7fd45aa00640 10 client.27694 handle_command_reply: tid=0 2024-05-20T20:17:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.060+0000 7fd476a22740 1 client.27694 shutdown 2024-05-20T20:17:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 msgr2=0x55cc95a41270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.061 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc95a41270 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fd45c022440 tx=0x7fd45c00ce00 comp rx=0 tx=0).stop 2024-05-20T20:17:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 shutdown_connections 2024-05-20T20:17:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55cc95a43010 0x55cc95a45400 crc :-1 s=READY pgs=242 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55cc959be5b0 0x55cc959ce610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55cc959c8e70 0x55cc95a41270 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 --2- 172.21.15.28:0/3413173928 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55cc959bdc90 0x55cc959ce0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 >> 172.21.15.28:0/3413173928 conn(0x55cc959ba1a0 msgr2=0x55cc959d4190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd45aa00640 0 client.27694 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 shutdown_connections 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.061+0000 7fd476a22740 1 -- 172.21.15.28:0/3413173928 wait complete. 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.062+0000 7fd476a22740 20 client.27694 trim_cache size 0 max 16384 2024-05-20T20:17:18.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:17:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:17:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:17:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.062+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4500b2bd0 msgr2=0x7fd4500b5090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.062+0000 7fd473c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4500b2bd0 0x7fd4500b5090 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fd45c00ab10 tx=0x7fd45c009250 comp rx=0 tx=0).stop 2024-05-20T20:17:18.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.062+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c100ee0 msgr2=0x7fd46c118270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.062+0000 7fd473c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c100ee0 0x7fd46c118270 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fd464005ab0 tx=0x7fd464003d50 comp rx=0 tx=0).stop 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 shutdown_connections 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fd46c105fb0 0x7fd46c1187b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fd4500b2bd0 0x7fd4500b5090 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fd46c111400 0x7fd46c1118b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 --2- 172.21.15.28:0/930664258 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd46c100ee0 0x7fd46c118270 unknown :-1 s=CLOSED pgs=467 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 >> 172.21.15.28:0/930664258 conn(0x7fd46c077a50 msgr2=0x7fd46c103ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 shutdown_connections 2024-05-20T20:17:18.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:18.063+0000 7fd473c00640 1 -- 172.21.15.28:0/930664258 wait complete. 2024-05-20T20:17:18.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.398+0000 7fa147a00700 1 -- 172.21.15.28:0/1501823546 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006b220 msgr2=0x7fa14006b6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.398+0000 7fa147a00700 1 --2- 172.21.15.28:0/1501823546 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006b220 0x7fa14006b6a0 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7fa12c009990 tx=0x7fa12c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.399+0000 7fa147a00700 1 -- 172.21.15.28:0/1501823546 shutdown_connections 2024-05-20T20:17:18.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.399+0000 7fa147a00700 1 --2- 172.21.15.28:0/1501823546 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa14006bbe0 0x7fa1401103d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.399+0000 7fa147a00700 1 --2- 172.21.15.28:0/1501823546 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 0x7fa140109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.399+0000 7fa147a00700 1 --2- 172.21.15.28:0/1501823546 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006b220 0x7fa14006b6a0 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.399+0000 7fa147a00700 1 -- 172.21.15.28:0/1501823546 >> 172.21.15.28:0/1501823546 conn(0x7fa1400793f0 msgr2=0x7fa140079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.399+0000 7fa147a00700 1 -- 172.21.15.28:0/1501823546 shutdown_connections 2024-05-20T20:17:18.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.399+0000 7fa147a00700 1 -- 172.21.15.28:0/1501823546 wait complete. 2024-05-20T20:17:18.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.401+0000 7fa147a00700 1 Processor -- start 2024-05-20T20:17:18.401 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.401+0000 7fa147a00700 1 -- start start 2024-05-20T20:17:18.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.402+0000 7fa147a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa14006b220 0x7fa14019f910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.402+0000 7fa147a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006bbe0 0x7fa14019fe50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.402+0000 7fa147a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 0x7fa140199ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.402+0000 7fa147a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa140113730 con 0x7fa1401095b0 2024-05-20T20:17:18.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.402+0000 7fa147a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401135b0 con 0x7fa14006b220 2024-05-20T20:17:18.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.402+0000 7fa147a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401138b0 con 0x7fa14006bbe0 2024-05-20T20:17:18.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.404+0000 7fa145a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 0x7fa140199ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.403 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.404+0000 7fa145a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 0x7fa140199ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:59760/0 (socket says 172.21.15.28:59760) 2024-05-20T20:17:18.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.404+0000 7fa145a00700 1 -- 172.21.15.28:0/3864104951 learned_addr learned my addr 172.21.15.28:0/3864104951 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:18.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.404+0000 7fa13fe00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006bbe0 0x7fa14019fe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.405+0000 7fa145a00700 1 -- 172.21.15.28:0/3864104951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006bbe0 msgr2=0x7fa14019fe50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.405+0000 7fa145a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006bbe0 0x7fa14019fe50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.405+0000 7fa145000700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa14006b220 0x7fa14019f910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.405+0000 7fa145a00700 1 -- 172.21.15.28:0/3864104951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa14006b220 msgr2=0x7fa14019f910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.405+0000 7fa145a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa14006b220 0x7fa14019f910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.405+0000 7fa145a00700 1 -- 172.21.15.28:0/3864104951 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa14019a040 con 0x7fa1401095b0 2024-05-20T20:17:18.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.405+0000 7fa145000700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa14006b220 0x7fa14019f910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:18.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.406+0000 7fa145a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 0x7fa140199ad0 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fa13800ef30 tx=0x7fa13800c450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:18.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.407+0000 7fa13d600700 1 -- 172.21.15.28:0/3864104951 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1380092e0 con 0x7fa1401095b0 2024-05-20T20:17:18.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.407+0000 7fa13d600700 1 -- 172.21.15.28:0/3864104951 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa138004bb0 con 0x7fa1401095b0 2024-05-20T20:17:18.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.407+0000 7fa13d600700 1 -- 172.21.15.28:0/3864104951 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa138004280 con 0x7fa1401095b0 2024-05-20T20:17:18.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.407+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa14019a2d0 con 0x7fa1401095b0 2024-05-20T20:17:18.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.407+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa14019a7c0 con 0x7fa1401095b0 2024-05-20T20:17:18.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.414+0000 7fa15231fb80 1 Processor -- start 2024-05-20T20:17:18.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.415+0000 7fa13d600700 1 -- 172.21.15.28:0/3864104951 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa13801b030 con 0x7fa1401095b0 2024-05-20T20:17:18.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.416+0000 7fa15231fb80 1 -- start start 2024-05-20T20:17:18.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.417+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a1290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.417+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a17d0 0x564d2149b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.417+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d2149b880 0x564d2149bce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.417+0000 7fa15231fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564d213f9e40 con 0x564d214a17d0 2024-05-20T20:17:18.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.417+0000 7fa15231fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564d213e99e0 con 0x564d214a0eb0 2024-05-20T20:17:18.417 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.417+0000 7fa15231fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564d213eda20 con 0x564d2149b880 2024-05-20T20:17:18.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.419+0000 7fa13fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a1290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.419+0000 7fa13fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a1290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54002/0 (socket says 172.21.15.28:54002) 2024-05-20T20:17:18.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.419+0000 7fa13fe00700 1 -- 172.21.15.28:0/1051314583 learned_addr learned my addr 172.21.15.28:0/1051314583 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:18.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.419+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d2149b880 0x564d2149bce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.419+0000 7fa13d600700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa1280b2a70 0x7fa1280b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.419+0000 7fa13d600700 1 -- 172.21.15.28:0/3864104951 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fa1380e2180 con 0x7fa1401095b0 2024-05-20T20:17:18.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.420+0000 7fa145000700 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a17d0 0x564d2149b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.420+0000 7fa145000700 1 -- 172.21.15.28:0/1051314583 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d2149b880 msgr2=0x564d2149bce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.420+0000 7fa145000700 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d2149b880 0x564d2149bce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.420+0000 7fa145000700 1 -- 172.21.15.28:0/1051314583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 msgr2=0x564d214a1290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.421+0000 7fa145000700 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a1290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.421+0000 7fa145000700 1 -- 172.21.15.28:0/1051314583 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564d213b2420 con 0x564d214a17d0 2024-05-20T20:17:18.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.421+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a1290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:18.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.421+0000 7fa145a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa1280b2a70 0x7fa1280b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.422+0000 7fa145000700 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a17d0 0x564d2149b260 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7fa134009b90 tx=0x7fa134009ea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c33e9b04f96fb735 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:18.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.423+0000 7fa132000700 1 -- 172.21.15.28:0/1051314583 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa134016070 con 0x564d214a17d0 2024-05-20T20:17:18.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.423+0000 7fa145a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa1280b2a70 0x7fa1280b4f30 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fa1380df940 tx=0x7fa138000f40 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:18.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.423+0000 7fa132000700 1 -- 172.21.15.28:0/1051314583 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa134004830 con 0x564d214a17d0 2024-05-20T20:17:18.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.423+0000 7fa132000700 1 -- 172.21.15.28:0/1051314583 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1340053e0 con 0x564d214a17d0 2024-05-20T20:17:18.424 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.425+0000 7fa15231fb80 1 -- 172.21.15.28:0/1051314583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a17d0 msgr2=0x564d2149b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.425+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a17d0 0x564d2149b260 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7fa134009b90 tx=0x7fa134009ea0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.425+0000 7fa15231fb80 1 -- 172.21.15.28:0/1051314583 shutdown_connections 2024-05-20T20:17:18.425 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.425+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d2149b880 0x564d2149bce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.425+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a1290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.425+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1051314583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a17d0 0x564d2149b260 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.425+0000 7fa15231fb80 1 -- 172.21.15.28:0/1051314583 >> 172.21.15.28:0/1051314583 conn(0x564d213c9be0 msgr2=0x564d21496010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.426+0000 7fa15231fb80 1 -- 172.21.15.28:0/1051314583 shutdown_connections 2024-05-20T20:17:18.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.426+0000 7fa15231fb80 1 -- 172.21.15.28:0/1051314583 wait complete. 2024-05-20T20:17:18.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.427+0000 7fa15231fb80 1 -- start start 2024-05-20T20:17:18.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.427+0000 7fa15231fb80 1 Processor -- start 2024-05-20T20:17:18.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.428+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d2149b880 0x564d214a76f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.428+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.428 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.428+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a4f30 0x564d214a53b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.429+0000 7fa145000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.429+0000 7fa145000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a49f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54010/0 (socket says 172.21.15.28:54010) 2024-05-20T20:17:18.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.429+0000 7fa145000700 1 -- 172.21.15.28:0/844353676 learned_addr learned my addr 172.21.15.28:0/844353676 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:18.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.429+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x564d212effa0 con 0x564d2149b880 2024-05-20T20:17:18.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.429+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x564d2138a760 con 0x564d214a0eb0 2024-05-20T20:17:18.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.429+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x564d214a8240 con 0x564d214a4f30 2024-05-20T20:17:18.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.430+0000 7fa15231fb80 10 client.0 _ll_register_callbacks cb 0x564d21393c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:18.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.430+0000 7fa13fe00700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a4f30 0x564d214a53b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.430+0000 7fa13fe00700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d2149b880 0x564d214a76f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.430+0000 7fa145000700 1 -- 172.21.15.28:0/844353676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a4f30 msgr2=0x564d214a53b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.430+0000 7fa145000700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a4f30 0x564d214a53b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.430+0000 7fa145000700 1 -- 172.21.15.28:0/844353676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d2149b880 msgr2=0x564d214a76f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.431+0000 7fa145000700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d2149b880 0x564d214a76f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.431+0000 7fa145000700 1 -- 172.21.15.28:0/844353676 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564d213398d0 con 0x564d214a0eb0 2024-05-20T20:17:18.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.431+0000 7fa13fe00700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d2149b880 0x564d214a76f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:17:18.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.431+0000 7fa145000700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a49f0 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7fa134005e60 tx=0x7fa13400eb80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bdebca5b8eb86299 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:18.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.432+0000 7fa126a00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:17:18.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.432+0000 7fa126a00700 1 -- 172.21.15.28:0/844353676 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa134028060 con 0x564d214a0eb0 2024-05-20T20:17:18.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.432+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa134004830 con 0x564d214a0eb0 2024-05-20T20:17:18.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.432+0000 7fa126a00700 1 -- 172.21.15.28:0/844353676 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fa134005210 con 0x564d214a0eb0 2024-05-20T20:17:18.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.432+0000 7fa126a00700 1 -- 172.21.15.28:0/844353676 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa134016040 con 0x564d214a0eb0 2024-05-20T20:17:18.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.432+0000 7fa15231fb80 10 client.27764 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:18.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.432+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564d213ba060 con 0x564d214a0eb0 2024-05-20T20:17:18.433 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa126a00700 1 -- 172.21.15.28:0/844353676 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa134004830 con 0x564d214a0eb0 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa126a00700 1 -- 172.21.15.28:0/844353676 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa13400f040 con 0x564d214a0eb0 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa15231fb80 10 client.27764 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa15231fb80 10 client.27764 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa15231fb80 20 client.27764 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa15231fb80 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d214b5270 0x564d214b7660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa15231fb80 4 client.27764 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.433+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x564d213fbd90 con 0x564d214b5270 2024-05-20T20:17:18.434 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.435+0000 7fa13fe00700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d214b5270 0x564d214b7660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.435+0000 7fa13fe00700 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d214b5270 0x564d214b7660 crc :-1 s=READY pgs=243 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9c0beef8b6beea96 server_cookie=4a457f5c88447905 in_seq=0 out_seq=0 2024-05-20T20:17:18.435 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.435+0000 7fa126a00700 10 client.27764 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:18.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.436+0000 7fa126a00700 1 -- 172.21.15.28:0/844353676 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x564d213fbd90 con 0x564d214b5270 2024-05-20T20:17:18.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.436+0000 7fa126a00700 10 client.27764 handle_command_reply: tid=0 2024-05-20T20:17:18.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.436+0000 7fa15231fb80 1 client.27764 shutdown 2024-05-20T20:17:18.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.437+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 msgr2=0x564d214a49f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.437+0000 7fa15231fb80 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a49f0 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7fa134005e60 tx=0x7fa13400eb80 comp rx=0 tx=0).stop 2024-05-20T20:17:18.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.437+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 shutdown_connections 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.437+0000 7fa15231fb80 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d214b5270 0x564d214b7660 crc :-1 s=READY pgs=243 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa15231fb80 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a4f30 0x564d214a53b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa15231fb80 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d2149b880 0x564d214a76f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa15231fb80 1 --2- 172.21.15.28:0/844353676 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d214a0eb0 0x564d214a49f0 unknown :-1 s=CLOSED pgs=570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 >> 172.21.15.28:0/844353676 conn(0x564d213c9be0 msgr2=0x564d2149a250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa126a00700 0 client.27764 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 shutdown_connections 2024-05-20T20:17:18.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa15231fb80 1 -- 172.21.15.28:0/844353676 wait complete. 2024-05-20T20:17:18.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.438+0000 7fa15231fb80 20 client.27764 trim_cache size 0 max 16384 2024-05-20T20:17:18.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.483+0000 7fa15231fb80 1 Processor -- start 2024-05-20T20:17:18.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.484+0000 7fa15231fb80 1 -- start start 2024-05-20T20:17:18.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.484+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d2149b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.484 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.484+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d2149f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.484+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d2149fb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.484+0000 7fa15231fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401b4700 con 0x564d214a4f30 2024-05-20T20:17:18.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.484+0000 7fa15231fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401b6240 con 0x564d2149b880 2024-05-20T20:17:18.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.484+0000 7fa15231fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401b4dd0 con 0x564d214a0eb0 2024-05-20T20:17:18.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.485+0000 7fa13fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d2149b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.485 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.485+0000 7fa13fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d2149b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54014/0 (socket says 172.21.15.28:54014) 2024-05-20T20:17:18.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.485+0000 7fa13fe00700 1 -- 172.21.15.28:0/1765373696 learned_addr learned my addr 172.21.15.28:0/1765373696 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:18.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.485+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d2149fb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.485+0000 7fa145000700 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d2149f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.486 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.486+0000 7fa13fe00700 1 -- 172.21.15.28:0/1765373696 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 msgr2=0x564d2149f0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.486+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d2149f0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.486+0000 7fa13fe00700 1 -- 172.21.15.28:0/1765373696 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 msgr2=0x564d2149b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.486+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d2149b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.486+0000 7fa13fe00700 1 -- 172.21.15.28:0/1765373696 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x564d214a5720 con 0x564d214a4f30 2024-05-20T20:17:18.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.486+0000 7fa145000700 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d2149f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:18.487 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.487+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d2149fb80 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7fa12c000c00 tx=0x7fa12c019040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d706ef76b48e8a74 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:18.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.488+0000 7fa131600700 1 -- 172.21.15.28:0/1765373696 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa12c021040 con 0x564d214a4f30 2024-05-20T20:17:18.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.488+0000 7fa131600700 1 -- 172.21.15.28:0/1765373696 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa12c003e10 con 0x564d214a4f30 2024-05-20T20:17:18.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.488+0000 7fa131600700 1 -- 172.21.15.28:0/1765373696 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa12c02aa20 con 0x564d214a4f30 2024-05-20T20:17:18.488 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.489+0000 7fa15231fb80 1 -- 172.21.15.28:0/1765373696 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 msgr2=0x564d2149fb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.489+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d2149fb80 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7fa12c000c00 tx=0x7fa12c019040 comp rx=0 tx=0).stop 2024-05-20T20:17:18.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.489+0000 7fa15231fb80 1 -- 172.21.15.28:0/1765373696 shutdown_connections 2024-05-20T20:17:18.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.489+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d2149fb80 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.489+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d2149b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.489+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1765373696 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d2149f0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.489+0000 7fa15231fb80 1 -- 172.21.15.28:0/1765373696 >> 172.21.15.28:0/1765373696 conn(0x564d213c9be0 msgr2=0x564d2149e9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.490+0000 7fa15231fb80 1 -- 172.21.15.28:0/1765373696 shutdown_connections 2024-05-20T20:17:18.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.490+0000 7fa15231fb80 1 -- 172.21.15.28:0/1765373696 wait complete. 2024-05-20T20:17:18.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.491+0000 7fa15231fb80 1 -- start start 2024-05-20T20:17:18.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.491+0000 7fa15231fb80 1 Processor -- start 2024-05-20T20:17:18.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.492+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d214b8560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.492+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d214b8aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa15231fb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d214b8fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa15231fb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401b5b70 con 0x564d214a4f30 2024-05-20T20:17:18.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa15231fb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401b54a0 con 0x564d2149b880 2024-05-20T20:17:18.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa15231fb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa1401b26c0 con 0x564d214a0eb0 2024-05-20T20:17:18.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa15231fb80 10 client.0 _ll_register_callbacks cb 0x564d21393c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:18.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa13fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d214b8560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa13fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d214b8560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:54026/0 (socket says 172.21.15.28:54026) 2024-05-20T20:17:18.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.493+0000 7fa13fe00700 1 -- 172.21.15.28:0/1728001540 learned_addr learned my addr 172.21.15.28:0/1728001540 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:18.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.494+0000 7fa145000700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d214b8aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.494+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d214b8fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.495+0000 7fa145000700 1 -- 172.21.15.28:0/1728001540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 msgr2=0x564d214b8560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.495+0000 7fa145000700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d214b8560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.495+0000 7fa145000700 1 -- 172.21.15.28:0/1728001540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 msgr2=0x564d214b8fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:18.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.495+0000 7fa145000700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d214b8fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.495+0000 7fa145000700 1 -- 172.21.15.28:0/1728001540 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x564d2151ec70 con 0x564d214a0eb0 2024-05-20T20:17:18.495 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.495+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d214b8fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:18.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.495+0000 7fa145000700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d214b8aa0 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x564d21496d90 tx=0x7fa134005980 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2de6de4a687f625 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:18.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.496+0000 7fa126a00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:18.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.496+0000 7fa126a00700 1 -- 172.21.15.28:0/1728001540 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa134028040 con 0x564d214a0eb0 2024-05-20T20:17:18.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.496+0000 7fa126a00700 1 -- 172.21.15.28:0/1728001540 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(92..92 src has 1..92) ==== 8586+0+0 (secure 0 0 0) 0x7fa134011470 con 0x564d214a0eb0 2024-05-20T20:17:18.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.496+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fa12c003e10 con 0x564d214a0eb0 2024-05-20T20:17:18.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.497+0000 7fa126a00700 1 -- 172.21.15.28:0/1728001540 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fa134011670 con 0x564d214a0eb0 2024-05-20T20:17:18.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.500+0000 7fa15231fb80 10 client.27706 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:18.500 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.500+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x564d2151feb0 con 0x564d214a0eb0 2024-05-20T20:17:18.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa126a00700 1 -- 172.21.15.28:0/1728001540 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa134004020 con 0x564d214a0eb0 2024-05-20T20:17:18.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa126a00700 1 -- 172.21.15.28:0/1728001540 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fa134032070 con 0x564d214a0eb0 2024-05-20T20:17:18.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa15231fb80 10 client.27706 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:18.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa15231fb80 10 client.27706 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:18.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa15231fb80 20 client.27706 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:18.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d215205b0 0x564d215229d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:18.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa15231fb80 4 client.27706 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:17:18.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.501+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x7fa12c003e10 con 0x564d215205b0 2024-05-20T20:17:18.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.502+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d215205b0 0x564d215229d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:18.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.502+0000 7fa126a00700 10 client.27706 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:18.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.503+0000 7fa13fe00700 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d215205b0 0x564d215229d0 crc :-1 s=READY pgs=244 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8157b91a47161fbb server_cookie=5b28a84627de1bfc in_seq=0 out_seq=0 2024-05-20T20:17:18.504 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.503+0000 7fa126a00700 1 -- 172.21.15.28:0/1728001540 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x7fa12c003e10 con 0x564d215205b0 2024-05-20T20:17:18.504 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.503+0000 7fa126a00700 10 client.27706 handle_command_reply: tid=0 2024-05-20T20:17:18.504 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.503+0000 7fa15231fb80 1 client.27706 shutdown 2024-05-20T20:17:18.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 msgr2=0x564d214b8aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d214b8aa0 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x564d21496d90 tx=0x7fa134005980 comp rx=0 tx=0).stop 2024-05-20T20:17:18.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 shutdown_connections 2024-05-20T20:17:18.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x564d215205b0 0x564d215229d0 crc :-1 s=READY pgs=244 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x564d214a4f30 0x564d214b8fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa126a00700 0 client.27706 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:18.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x564d2149b880 0x564d214b8560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 --2- 172.21.15.28:0/1728001540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x564d214a0eb0 0x564d214b8aa0 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.506+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 >> 172.21.15.28:0/1728001540 conn(0x564d213c9be0 msgr2=0x564d214b0500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.507+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 shutdown_connections 2024-05-20T20:17:18.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.507+0000 7fa15231fb80 1 -- 172.21.15.28:0/1728001540 wait complete. 2024-05-20T20:17:18.508 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.507+0000 7fa15231fb80 20 client.27706 trim_cache size 0 max 16384 2024-05-20T20:17:18.508 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:17:18.508 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:17:18.509 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:17:18.509 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:17:18.511 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.512+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa1280b2a70 msgr2=0x7fa1280b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.512+0000 7fa147a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa1280b2a70 0x7fa1280b4f30 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fa1380df940 tx=0x7fa138000f40 comp rx=0 tx=0).stop 2024-05-20T20:17:18.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.512+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 msgr2=0x7fa140199ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:18.512 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.512+0000 7fa147a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 0x7fa140199ad0 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fa13800ef30 tx=0x7fa13800c450 comp rx=0 tx=0).stop 2024-05-20T20:17:18.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.513+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 shutdown_connections 2024-05-20T20:17:18.513 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.513+0000 7fa147a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa1401095b0 0x7fa140199ad0 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.513+0000 7fa147a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa1280b2a70 0x7fa1280b4f30 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.513+0000 7fa147a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa14006b220 0x7fa14019f910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.513+0000 7fa147a00700 1 --2- 172.21.15.28:0/3864104951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa14006bbe0 0x7fa14019fe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:18.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.513+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 >> 172.21.15.28:0/3864104951 conn(0x7fa1400793f0 msgr2=0x7fa1401011a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:18.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.513+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 shutdown_connections 2024-05-20T20:17:18.514 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:18.514+0000 7fa147a00700 1 -- 172.21.15.28:0/3864104951 wait complete. 2024-05-20T20:17:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:19 smithi154 ceph-mon[36314]: pgmap v364: 97 pgs: 2 active+clean+snaptrim, 4 active+clean+snaptrim_wait, 91 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 269 MiB/s wr, 1.91k op/s 2024-05-20T20:17:19.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:19 smithi033 ceph-mon[36179]: pgmap v364: 97 pgs: 2 active+clean+snaptrim, 4 active+clean+snaptrim_wait, 91 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 269 MiB/s wr, 1.91k op/s 2024-05-20T20:17:19.809 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:19 smithi028 ceph-mon[31178]: pgmap v364: 97 pgs: 2 active+clean+snaptrim, 4 active+clean+snaptrim_wait, 91 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 269 MiB/s wr, 1.91k op/s 2024-05-20T20:17:20.177 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:17:20.177 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:17:21.427 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:21 smithi154 ceph-mon[36314]: pgmap v365: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 301 MiB/s wr, 2.53k op/s 2024-05-20T20:17:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:21 smithi154 ceph-mon[36314]: osdmap e93: 12 total, 12 up, 12 in 2024-05-20T20:17:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:21 smithi033 ceph-mon[36179]: pgmap v365: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 301 MiB/s wr, 2.53k op/s 2024-05-20T20:17:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:21 smithi033 ceph-mon[36179]: osdmap e93: 12 total, 12 up, 12 in 2024-05-20T20:17:21.651 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:21 smithi028 ceph-mon[31178]: pgmap v365: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 301 MiB/s wr, 2.53k op/s 2024-05-20T20:17:21.651 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:21 smithi028 ceph-mon[31178]: osdmap e93: 12 total, 12 up, 12 in 2024-05-20T20:17:23.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:23 smithi028 ceph-mon[31178]: pgmap v367: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 252 MiB/s wr, 2.42k op/s 2024-05-20T20:17:23.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:23 smithi154 ceph-mon[36314]: pgmap v367: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 252 MiB/s wr, 2.42k op/s 2024-05-20T20:17:23.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:23 smithi033 ceph-mon[36179]: pgmap v367: 97 pgs: 97 active+clean; 54 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 252 MiB/s wr, 2.42k op/s 2024-05-20T20:17:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:25 smithi154 ceph-mon[36314]: pgmap v368: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 229 MiB/s wr, 2.35k op/s 2024-05-20T20:17:25.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:25 smithi033 ceph-mon[36179]: pgmap v368: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 229 MiB/s wr, 2.35k op/s 2024-05-20T20:17:25.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:25 smithi028 ceph-mon[31178]: pgmap v368: 97 pgs: 97 active+clean; 55 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 229 MiB/s wr, 2.35k op/s 2024-05-20T20:17:27.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:27 smithi154 ceph-mon[36314]: pgmap v369: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 225 MiB/s wr, 2.31k op/s 2024-05-20T20:17:27.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:27 smithi033 ceph-mon[36179]: pgmap v369: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 225 MiB/s wr, 2.31k op/s 2024-05-20T20:17:27.719 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:27 smithi028 ceph-mon[31178]: pgmap v369: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 225 MiB/s wr, 2.31k op/s 2024-05-20T20:17:27.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.717+0000 7fedb0000700 1 -- 172.21.15.28:0/4092329262 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feda8078390 msgr2=0x7feda8113ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:27.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.717+0000 7fedb0000700 1 --2- 172.21.15.28:0/4092329262 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feda8078390 0x7feda8113ee0 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7fed94009940 tx=0x7fed94009c50 comp rx=0 tx=0).stop 2024-05-20T20:17:27.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.717+0000 7fedb0000700 1 -- 172.21.15.28:0/4092329262 shutdown_connections 2024-05-20T20:17:27.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.717+0000 7fedb0000700 1 --2- 172.21.15.28:0/4092329262 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feda8114420 0x7feda8116810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.717+0000 7fedb0000700 1 --2- 172.21.15.28:0/4092329262 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8077a70 0x7feda8077e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.717+0000 7fedb0000700 1 --2- 172.21.15.28:0/4092329262 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feda8078390 0x7feda8113ee0 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.717+0000 7fedb0000700 1 -- 172.21.15.28:0/4092329262 >> 172.21.15.28:0/4092329262 conn(0x7feda8101270 msgr2=0x7feda8103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.718+0000 7fedb0000700 1 -- 172.21.15.28:0/4092329262 shutdown_connections 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.718+0000 7fedb0000700 1 -- 172.21.15.28:0/4092329262 wait complete. 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.719+0000 7fedb0000700 1 Processor -- start 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.720+0000 7fedb0000700 1 -- start start 2024-05-20T20:17:27.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.720+0000 7fedb0000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feda8077a70 0x7feda81a0390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.720+0000 7fedb0000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8078390 0x7feda81a08d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.720+0000 7fedb0000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feda8114420 0x7feda81a4c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.720+0000 7fedb0000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7feda8119de0 con 0x7feda8114420 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.720+0000 7fedb0000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7feda8119c60 con 0x7feda8077a70 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.720+0000 7fedb0000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7feda8119f60 con 0x7feda8078390 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedad600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feda8077a70 0x7feda81a0390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8078390 0x7feda81a08d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8078390 0x7feda81a08d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35476/0 (socket says 172.21.15.28:35476) 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 -- 172.21.15.28:0/542100667 learned_addr learned my addr 172.21.15.28:0/542100667 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 -- 172.21.15.28:0/542100667 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feda8077a70 msgr2=0x7feda81a0390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:27.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feda8077a70 0x7feda81a0390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 -- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feda8114420 msgr2=0x7feda81a4c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feda8114420 0x7feda81a4c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.721+0000 7fedacc00700 1 -- 172.21.15.28:0/542100667 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feda81a5350 con 0x7feda8078390 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.722+0000 7fedacc00700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8078390 0x7feda81a08d0 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7fed94009230 tx=0x7fed9400fab0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.722+0000 7feda6000700 1 -- 172.21.15.28:0/542100667 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed94024020 con 0x7feda8078390 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.722+0000 7feda6000700 1 -- 172.21.15.28:0/542100667 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fed94005020 con 0x7feda8078390 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.722+0000 7feda6000700 1 -- 172.21.15.28:0/542100667 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fed94004ad0 con 0x7feda8078390 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.723+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feda81a55e0 con 0x7feda8078390 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.723+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7feda81acec0 con 0x7feda8078390 2024-05-20T20:17:27.766 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.725+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7feda80791b0 con 0x7feda8078390 2024-05-20T20:17:27.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.730+0000 7feda6000700 1 -- 172.21.15.28:0/542100667 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fed94004c70 con 0x7feda8078390 2024-05-20T20:17:27.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.733+0000 7feda6000700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fed900b2af0 0x7fed900b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:27.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.735+0000 7feda6000700 1 -- 172.21.15.28:0/542100667 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fed940e3590 con 0x7feda8078390 2024-05-20T20:17:27.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.736+0000 7fedad600700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fed900b2af0 0x7fed900b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:27.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.737+0000 7feda6000700 1 -- 172.21.15.28:0/542100667 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fed9402e020 con 0x7feda8078390 2024-05-20T20:17:27.767 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:27.738+0000 7fedad600700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fed900b2af0 0x7fed900b4fb0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fed9c005fd0 tx=0x7fed9c009310 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:28.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.372+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7feda81a16a0 con 0x7feda8078390 2024-05-20T20:17:28.377 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.378+0000 7feda6000700 1 -- 172.21.15.28:0/542100667 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7fed94013030 con 0x7feda8078390 2024-05-20T20:17:28.378 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:17:28.378 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.381+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fed900b2af0 msgr2=0x7fed900b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.381+0000 7fedb0000700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fed900b2af0 0x7fed900b4fb0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fed9c005fd0 tx=0x7fed9c009310 comp rx=0 tx=0).stop 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.381+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8078390 msgr2=0x7feda81a08d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.381+0000 7fedb0000700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8078390 0x7feda81a08d0 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7fed94009230 tx=0x7fed9400fab0 comp rx=0 tx=0).stop 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 shutdown_connections 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7feda8114420 0x7feda81a4c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fed900b2af0 0x7fed900b4fb0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7feda8077a70 0x7feda81a0390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 --2- 172.21.15.28:0/542100667 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feda8078390 0x7feda81a08d0 unknown :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:28.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 >> 172.21.15.28:0/542100667 conn(0x7feda8101270 msgr2=0x7feda8103660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:28.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 shutdown_connections 2024-05-20T20:17:28.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:28.382+0000 7fedb0000700 1 -- 172.21.15.28:0/542100667 wait complete. 2024-05-20T20:17:28.383 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:17:29.402 INFO:tasks.quiescer.fs.[cephfs]:Total duration (35.8) is longer than `split_if_longer` (35.0), will split into two consecutive quiesces 2024-05-20T20:17:29.402 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 17.9 2024-05-20T20:17:29.403 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:17:29.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:29 smithi154 ceph-mon[36314]: pgmap v370: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 226 MiB/s wr, 2.27k op/s 2024-05-20T20:17:29.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:29 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/542100667' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:29.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:29 smithi033 ceph-mon[36179]: pgmap v370: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 226 MiB/s wr, 2.27k op/s 2024-05-20T20:17:29.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:29 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/542100667' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:29.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:29 smithi028 ceph-mon[31178]: pgmap v370: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 226 MiB/s wr, 2.27k op/s 2024-05-20T20:17:29.809 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:29 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/542100667' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:30.400 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:17:30.797 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:31.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 -- 172.21.15.28:0/2767616284 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc10a4b0 msgr2=0x7fe7dc10da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 --2- 172.21.15.28:0/2767616284 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc10a4b0 0x7fe7dc10da80 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7fe7c8009900 tx=0x7fe7c802a3f0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 -- 172.21.15.28:0/2767616284 shutdown_connections 2024-05-20T20:17:31.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 --2- 172.21.15.28:0/2767616284 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc10a4b0 0x7fe7dc10da80 unknown :-1 s=CLOSED pgs=571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 --2- 172.21.15.28:0/2767616284 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe7dc10e160 0x7fe7dc115dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 --2- 172.21.15.28:0/2767616284 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7dc109b00 0x7fe7dc109ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.508 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 -- 172.21.15.28:0/2767616284 >> 172.21.15.28:0/2767616284 conn(0x7fe7dc0784a0 msgr2=0x7fe7dc07a8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:31.509 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 -- 172.21.15.28:0/2767616284 shutdown_connections 2024-05-20T20:17:31.509 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.508+0000 7fe7e2400640 1 -- 172.21.15.28:0/2767616284 wait complete. 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.509+0000 7fe7e2400640 1 Processor -- start 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.509+0000 7fe7e2400640 1 -- start start 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e2400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc109b00 0x7fe7dc19f6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e2400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7dc10e160 0x7fe7dc19fbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e2400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe7dc1a3f10 0x7fe7dc1a43c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e2400640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc119180 con 0x7fe7dc1a3f10 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e2400640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc119000 con 0x7fe7dc109b00 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e2400640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc119300 con 0x7fe7dc10e160 2024-05-20T20:17:31.511 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc109b00 0x7fe7dc19f6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.513 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7dbe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7dc10e160 0x7fe7dc19fbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.513 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc109b00 0x7fe7dc19f6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:59518/0 (socket says 172.21.15.28:59518) 2024-05-20T20:17:31.513 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 -- 172.21.15.28:0/3255458902 learned_addr learned my addr 172.21.15.28:0/3255458902 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:31.513 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 -- 172.21.15.28:0/3255458902 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7dc10e160 msgr2=0x7fe7dc19fbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7dc10e160 0x7fe7dc19fbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 -- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe7dc1a3f10 msgr2=0x7fe7dc1a43c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe7dc1a3f10 0x7fe7dc1a43c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.510+0000 7fe7e1000640 1 -- 172.21.15.28:0/3255458902 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe7dc1a4a00 con 0x7fe7dc109b00 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.511+0000 7fe7e1000640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc109b00 0x7fe7dc19f6a0 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7fe7d000e9f0 tx=0x7fe7d000edf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.511+0000 7fe7d9600640 1 -- 172.21.15.28:0/3255458902 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7d000cd80 con 0x7fe7dc109b00 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.511+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe7dc1a4cf0 con 0x7fe7dc109b00 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.511+0000 7fe7d9600640 1 -- 172.21.15.28:0/3255458902 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe7d00054d0 con 0x7fe7dc109b00 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.511+0000 7fe7d9600640 1 -- 172.21.15.28:0/3255458902 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7d0010690 con 0x7fe7dc109b00 2024-05-20T20:17:31.514 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.511+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe7dc1ac4b0 con 0x7fe7dc109b00 2024-05-20T20:17:31.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.515+0000 7fe7e67cc740 1 Processor -- start 2024-05-20T20:17:31.515 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.515+0000 7fe7e67cc740 1 -- start start 2024-05-20T20:17:31.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316226f250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x563162279b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x56316227c440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5631621dfb90 con 0x56316227a050 2024-05-20T20:17:31.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5631621cf730 con 0x56316226ee70 2024-05-20T20:17:31.516 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5631621d3770 con 0x56316226f790 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7d9600640 1 -- 172.21.15.28:0/3255458902 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fe7d0010830 con 0x7fe7dc109b00 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e1a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316226f250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e1a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316226f250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:59520/0 (socket says 172.21.15.28:59520) 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.516+0000 7fe7e1a00640 1 -- 172.21.15.28:0/954383849 learned_addr learned my addr 172.21.15.28:0/954383849 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7e1000640 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x563162279b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7dbe00640 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x56316227c440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7e1a00640 1 -- 172.21.15.28:0/954383849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 msgr2=0x563162279b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:31.517 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x563162279b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7e1a00640 1 -- 172.21.15.28:0/954383849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 msgr2=0x56316227c440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:31.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x56316227c440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7e1a00640 1 -- 172.21.15.28:0/954383849 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5631621688b0 con 0x56316226ee70 2024-05-20T20:17:31.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7dbe00640 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x56316227c440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:31.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.517+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316226f250 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7fe7d4007c60 tx=0x7fe7d400c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6cbaac32741555df server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:31.518 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.518+0000 7fe7ce000640 1 -- 172.21.15.28:0/954383849 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7d4003b70 con 0x56316226ee70 2024-05-20T20:17:31.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.518+0000 7fe7ce000640 1 -- 172.21.15.28:0/954383849 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe7d4003d10 con 0x56316226ee70 2024-05-20T20:17:31.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.518+0000 7fe7ce000640 1 -- 172.21.15.28:0/954383849 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7d400f850 con 0x56316226ee70 2024-05-20T20:17:31.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.519+0000 7fe7e67cc740 1 -- 172.21.15.28:0/954383849 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 msgr2=0x56316226f250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.519 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.519+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316226f250 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x7fe7d4007c60 tx=0x7fe7d400c6a0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.519+0000 7fe7e67cc740 1 -- 172.21.15.28:0/954383849 shutdown_connections 2024-05-20T20:17:31.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.519+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x56316227c440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.519+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x563162279b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.519+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/954383849 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316226f250 unknown :-1 s=CLOSED pgs=573 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.519+0000 7fe7e67cc740 1 -- 172.21.15.28:0/954383849 >> 172.21.15.28:0/954383849 conn(0x56316226b380 msgr2=0x56316226b760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:31.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.520+0000 7fe7d9600640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe7c40b29d0 0x7fe7c40b4e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.521 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.520+0000 7fe7d9600640 1 -- 172.21.15.28:0/3255458902 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fe7d00e2b70 con 0x7fe7dc109b00 2024-05-20T20:17:31.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.522+0000 7fe7dbe00640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe7c40b29d0 0x7fe7c40b4e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.522+0000 7fe7e67cc740 1 -- 172.21.15.28:0/954383849 shutdown_connections 2024-05-20T20:17:31.522 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.523+0000 7fe7e67cc740 1 -- 172.21.15.28:0/954383849 wait complete. 2024-05-20T20:17:31.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.523+0000 7fe7e67cc740 1 -- start start 2024-05-20T20:17:31.523 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.524+0000 7fe7dbe00640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe7c40b29d0 0x7fe7c40b4e90 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fe7c8031440 tx=0x7fe7c8005a20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:31.524 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.524+0000 7fe7e67cc740 1 Processor -- start 2024-05-20T20:17:31.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.525+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226ee70 0x563162279a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.525+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226f790 0x563162276cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.525 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.525+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316227a050 0x563162277290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.525+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563161eb5790 con 0x56316226f790 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.525+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563162177bc0 con 0x56316227a050 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.525+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56316227cc80 con 0x56316226ee70 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.525+0000 7fe7e67cc740 10 client.0 _ll_register_callbacks cb 0x563162155400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.526+0000 7fe7dbe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226f790 0x563162276cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.526+0000 7fe7dbe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226f790 0x563162276cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60816/0 (socket says 172.21.15.28:60816) 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.526+0000 7fe7dbe00640 1 -- 172.21.15.28:0/1929539304 learned_addr learned my addr 172.21.15.28:0/1929539304 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.526+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316227a050 0x563162277290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.526 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.526+0000 7fe7e1000640 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226ee70 0x563162279a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.526+0000 7fe7e1a00640 1 -- 172.21.15.28:0/1929539304 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226ee70 msgr2=0x563162279a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:31.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.527+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226ee70 0x563162279a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.527+0000 7fe7e1a00640 1 -- 172.21.15.28:0/1929539304 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226f790 msgr2=0x563162276cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:31.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.527+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226f790 0x563162276cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.527 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.527+0000 7fe7e1a00640 1 -- 172.21.15.28:0/1929539304 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563161ee4210 con 0x56316227a050 2024-05-20T20:17:31.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.529+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316227a050 0x563162277290 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fe7d4004a80 tx=0x7fe7d4005f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d47db3b83f50c749 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:31.529 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.529+0000 7fe7c2a00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:17:31.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.530+0000 7fe7c2a00640 1 -- 172.21.15.28:0/1929539304 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7d4028060 con 0x56316227a050 2024-05-20T20:17:31.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.530+0000 7fe7c2a00640 1 -- 172.21.15.28:0/1929539304 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fe7d40040d0 con 0x56316227a050 2024-05-20T20:17:31.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.530+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe7d4003d10 con 0x56316227a050 2024-05-20T20:17:31.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.530+0000 7fe7c2a00640 1 -- 172.21.15.28:0/1929539304 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe7d40078a0 con 0x56316227a050 2024-05-20T20:17:31.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.530+0000 7fe7e67cc740 10 client.27788 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:31.530 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.530+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563161fb0620 con 0x56316227a050 2024-05-20T20:17:31.531 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7c2a00640 1 -- 172.21.15.28:0/1929539304 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe7d4003d10 con 0x56316227a050 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7c2a00640 1 -- 172.21.15.28:0/1929539304 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe7d40183f0 con 0x56316227a050 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7e67cc740 10 client.27788 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7e67cc740 10 client.27788 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7e67cc740 20 client.27788 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162289830 0x56316228bc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7e67cc740 4 client.27788 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5631621e1ae0 con 0x563162289830 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.531+0000 7fe7e1000640 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162289830 0x56316228bc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.532+0000 7fe7e1000640 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162289830 0x56316228bc20 crc :-1 s=READY pgs=245 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2b0a20951f9a5984 server_cookie=173c28e480511909 in_seq=0 out_seq=0 2024-05-20T20:17:31.532 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.532+0000 7fe7c2a00640 10 client.27788 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:31.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7c2a00640 1 -- 172.21.15.28:0/1929539304 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5631621e1ae0 con 0x563162289830 2024-05-20T20:17:31.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7c2a00640 10 client.27788 handle_command_reply: tid=0 2024-05-20T20:17:31.533 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 client.27788 shutdown 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316227a050 msgr2=0x563162277290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316227a050 0x563162277290 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7fe7d4004a80 tx=0x7fe7d4005f90 comp rx=0 tx=0).stop 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 shutdown_connections 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162289830 0x56316228bc20 crc :-1 s=READY pgs=245 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316227a050 0x563162277290 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226f790 0x563162276cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/1929539304 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226ee70 0x563162279a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 >> 172.21.15.28:0/1929539304 conn(0x56316226b380 msgr2=0x5631622710b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.533+0000 7fe7c2a00640 0 client.27788 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.534+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 shutdown_connections 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.534+0000 7fe7e67cc740 1 -- 172.21.15.28:0/1929539304 wait complete. 2024-05-20T20:17:31.534 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.534+0000 7fe7e67cc740 20 client.27788 trim_cache size 0 max 16384 2024-05-20T20:17:31.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 Processor -- start 2024-05-20T20:17:31.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 -- start start 2024-05-20T20:17:31.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226ee70 0x563162266250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.558 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226f790 0x563162285ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316227a050 0x56316226a230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc1b4bb0 con 0x56316226ee70 2024-05-20T20:17:31.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc1b66f0 con 0x56316226f790 2024-05-20T20:17:31.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.558+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc1b5280 con 0x56316227a050 2024-05-20T20:17:31.559 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.559+0000 7fe7e1000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226ee70 0x563162266250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.559+0000 7fe7e1000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226ee70 0x563162266250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60830/0 (socket says 172.21.15.28:60830) 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.559+0000 7fe7e1000640 1 -- 172.21.15.28:0/376328529 learned_addr learned my addr 172.21.15.28:0/376328529 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.560+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316227a050 0x56316226a230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.560+0000 7fe7e1000640 1 -- 172.21.15.28:0/376328529 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316227a050 msgr2=0x56316226a230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.560+0000 7fe7e1000640 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316227a050 0x56316226a230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.560+0000 7fe7e1000640 1 -- 172.21.15.28:0/376328529 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226f790 msgr2=0x563162285ce0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.560+0000 7fe7e1000640 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226f790 0x563162285ce0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.560+0000 7fe7e1000640 1 -- 172.21.15.28:0/376328529 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56316227c7b0 con 0x56316226ee70 2024-05-20T20:17:31.560 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.560+0000 7fe7e1000640 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226ee70 0x563162266250 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7fe7d00a9ae0 tx=0x7fe7d0004a90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3499b6595cbf3ba0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:31.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.561+0000 7fe7cd600640 1 -- 172.21.15.28:0/376328529 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7d00ea040 con 0x56316226ee70 2024-05-20T20:17:31.561 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.561+0000 7fe7cd600640 1 -- 172.21.15.28:0/376328529 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe7d00a98f0 con 0x56316226ee70 2024-05-20T20:17:31.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.561+0000 7fe7cd600640 1 -- 172.21.15.28:0/376328529 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7d00ae190 con 0x56316226ee70 2024-05-20T20:17:31.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.562+0000 7fe7e67cc740 1 -- 172.21.15.28:0/376328529 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226ee70 msgr2=0x563162266250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.562+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226ee70 0x563162266250 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7fe7d00a9ae0 tx=0x7fe7d0004a90 comp rx=0 tx=0).stop 2024-05-20T20:17:31.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.562+0000 7fe7e67cc740 1 -- 172.21.15.28:0/376328529 shutdown_connections 2024-05-20T20:17:31.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.562+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226f790 0x563162285ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.562 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.563+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316227a050 0x56316226a230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.563+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/376328529 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316226ee70 0x563162266250 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.563 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.563+0000 7fe7e67cc740 1 -- 172.21.15.28:0/376328529 >> 172.21.15.28:0/376328529 conn(0x56316226b380 msgr2=0x563162269b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:31.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.563+0000 7fe7e67cc740 1 -- 172.21.15.28:0/376328529 shutdown_connections 2024-05-20T20:17:31.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.563+0000 7fe7e67cc740 1 -- 172.21.15.28:0/376328529 wait complete. 2024-05-20T20:17:31.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.564+0000 7fe7e67cc740 1 -- start start 2024-05-20T20:17:31.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.564+0000 7fe7e67cc740 1 Processor -- start 2024-05-20T20:17:31.564 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316227d6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x56316227dbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e67cc740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x5631622fd060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc1b6020 con 0x56316227a050 2024-05-20T20:17:31.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc1b5950 con 0x56316226ee70 2024-05-20T20:17:31.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e67cc740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe7dc1b2bb0 con 0x56316226f790 2024-05-20T20:17:31.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e67cc740 10 client.0 _ll_register_callbacks cb 0x7fe7d00e6480 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:31.565 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7dbe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x56316227dbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.566 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7dbe00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x56316227dbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:35516/0 (socket says 172.21.15.28:35516) 2024-05-20T20:17:31.566 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7dbe00640 1 -- 172.21.15.28:0/3400236963 learned_addr learned my addr 172.21.15.28:0/3400236963 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:31.566 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e1000640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316227d6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.566 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.565+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x5631622fd060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.566+0000 7fe7dbe00640 1 -- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 msgr2=0x56316227d6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.566+0000 7fe7dbe00640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316227d6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.566+0000 7fe7dbe00640 1 -- 172.21.15.28:0/3400236963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 msgr2=0x5631622fd060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.566+0000 7fe7dbe00640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x5631622fd060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.566+0000 7fe7dbe00640 1 -- 172.21.15.28:0/3400236963 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5631622fd6d0 con 0x56316226f790 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.566+0000 7fe7e1000640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316227d6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.567+0000 7fe7e1a00640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x5631622fd060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.567+0000 7fe7dbe00640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x56316227dbe0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7fe7c8009230 tx=0x7fe7c802a3f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=18178c9003bcabfd server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:31.567 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.567+0000 7fe7c2a00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:31.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.567+0000 7fe7c2a00640 1 -- 172.21.15.28:0/3400236963 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe7c804e070 con 0x56316226f790 2024-05-20T20:17:31.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.567+0000 7fe7c2a00640 1 -- 172.21.15.28:0/3400236963 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fe7c8004b60 con 0x56316226f790 2024-05-20T20:17:31.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.567+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe7d00a98f0 con 0x56316226f790 2024-05-20T20:17:31.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.568+0000 7fe7c2a00640 1 -- 172.21.15.28:0/3400236963 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe7c8058050 con 0x56316226f790 2024-05-20T20:17:31.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 10 client.27730 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:31.568 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563162304ab0 con 0x56316226f790 2024-05-20T20:17:31.569 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7c2a00640 1 -- 172.21.15.28:0/3400236963 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe7d00a98f0 con 0x56316226f790 2024-05-20T20:17:31.569 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7c2a00640 1 -- 172.21.15.28:0/3400236963 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe7c8032540 con 0x56316226f790 2024-05-20T20:17:31.569 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 10 client.27730 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:31.569 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 10 client.27730 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:31.569 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 20 client.27730 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:31.569 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162305250 0x563162307670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:31.569 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 4 client.27730 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:17:31.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.569+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x5631621db2c0 con 0x563162305250 2024-05-20T20:17:31.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.570+0000 7fe7e1000640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162305250 0x563162307670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:31.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.571+0000 7fe7e1000640 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162305250 0x563162307670 crc :-1 s=READY pgs=246 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=419601e938581239 server_cookie=ab9fc2b45cd3215 in_seq=0 out_seq=0 2024-05-20T20:17:31.570 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.571+0000 7fe7c2a00640 10 client.27730 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:31.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.635+0000 7fe7c2a00640 1 -- 172.21.15.28:0/3400236963 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x5631621db2c0 con 0x563162305250 2024-05-20T20:17:31.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.635+0000 7fe7c2a00640 10 client.27730 handle_command_reply: tid=0 2024-05-20T20:17:31.635 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.635+0000 7fe7e67cc740 1 client.27730 shutdown 2024-05-20T20:17:31.642 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 msgr2=0x56316227dbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.642 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x56316227dbe0 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7fe7c8009230 tx=0x7fe7c802a3f0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.642 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 shutdown_connections 2024-05-20T20:17:31.642 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x563162305250 0x563162307670 crc :-1 s=READY pgs=246 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.642 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56316227a050 0x5631622fd060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.642 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56316226f790 0x56316227dbe0 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 --2- 172.21.15.28:0/3400236963 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56316226ee70 0x56316227d6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 >> 172.21.15.28:0/3400236963 conn(0x56316226b380 msgr2=0x56316226a3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:31.643 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.642+0000 7fe7c2a00640 0 client.27730 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:31.645 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.645+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 shutdown_connections 2024-05-20T20:17:31.648 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.648+0000 7fe7e67cc740 1 -- 172.21.15.28:0/3400236963 wait complete. 2024-05-20T20:17:31.648 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.648+0000 7fe7e67cc740 20 client.27730 trim_cache size 0 max 16384 2024-05-20T20:17:31.649 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:17:31.649 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:17:31.649 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:17:31.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.650+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe7c40b29d0 msgr2=0x7fe7c40b4e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.650+0000 7fe7e2400640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe7c40b29d0 0x7fe7c40b4e90 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fe7c8031440 tx=0x7fe7c8005a20 comp rx=0 tx=0).stop 2024-05-20T20:17:31.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.650+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc109b00 msgr2=0x7fe7dc19f6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:31.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.650+0000 7fe7e2400640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc109b00 0x7fe7dc19f6a0 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7fe7d000e9f0 tx=0x7fe7d000edf0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.653 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.651+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 shutdown_connections 2024-05-20T20:17:31.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.651+0000 7fe7e2400640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe7c40b29d0 0x7fe7c40b4e90 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.651+0000 7fe7e2400640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe7dc1a3f10 0x7fe7dc1a43c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.651+0000 7fe7e2400640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7dc10e160 0x7fe7dc19fbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.651+0000 7fe7e2400640 1 --2- 172.21.15.28:0/3255458902 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe7dc109b00 0x7fe7dc19f6a0 secure :-1 s=CLOSED pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7fe7d000e9f0 tx=0x7fe7d000edf0 comp rx=0 tx=0).stop 2024-05-20T20:17:31.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.651+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 >> 172.21.15.28:0/3255458902 conn(0x7fe7dc0784a0 msgr2=0x7fe7dc07a890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:31.654 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.654+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 shutdown_connections 2024-05-20T20:17:31.655 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:31.654+0000 7fe7e2400640 1 -- 172.21.15.28:0/3255458902 wait complete. 2024-05-20T20:17:31.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:31 smithi028 ceph-mon[31178]: pgmap v371: 97 pgs: 97 active+clean; 56 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 236 MiB/s wr, 1.89k op/s 2024-05-20T20:17:31.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:31 smithi154 ceph-mon[36314]: pgmap v371: 97 pgs: 97 active+clean; 56 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 236 MiB/s wr, 1.89k op/s 2024-05-20T20:17:31.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:31 smithi033 ceph-mon[36179]: pgmap v371: 97 pgs: 97 active+clean; 56 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 236 MiB/s wr, 1.89k op/s 2024-05-20T20:17:32.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:32 smithi028 ceph-mon[31178]: pgmap v372: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 212 MiB/s wr, 1.69k op/s 2024-05-20T20:17:32.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:32 smithi154 ceph-mon[36314]: pgmap v372: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 212 MiB/s wr, 1.69k op/s 2024-05-20T20:17:32.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:32 smithi033 ceph-mon[36179]: pgmap v372: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 212 MiB/s wr, 1.69k op/s 2024-05-20T20:17:34.614 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:35.410 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:35 smithi028 ceph-mon[31178]: pgmap v373: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 296 MiB/s wr, 2.13k op/s 2024-05-20T20:17:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:35 smithi154 ceph-mon[36314]: pgmap v373: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 296 MiB/s wr, 2.13k op/s 2024-05-20T20:17:35.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:35 smithi033 ceph-mon[36179]: pgmap v373: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 296 MiB/s wr, 2.13k op/s 2024-05-20T20:17:36.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 -- 172.21.15.28:0/1585287163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7a74105780 msgr2=0x7f7a74105c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:36.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1585287163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7a74105780 0x7f7a74105c00 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7f7a60009990 tx=0x7f7a60009ca0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 -- 172.21.15.28:0/1585287163 shutdown_connections 2024-05-20T20:17:36.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1585287163 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7a74106140 0x7f7a74112650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.270 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1585287163 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a7410b780 0x7f7a7410bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1585287163 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7a74105780 0x7f7a74105c00 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 -- 172.21.15.28:0/1585287163 >> 172.21.15.28:0/1585287163 conn(0x7f7a74101270 msgr2=0x7f7a74103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.212+0000 7f7a78c00700 1 -- 172.21.15.28:0/1585287163 shutdown_connections 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.214+0000 7f7a78c00700 1 -- 172.21.15.28:0/1585287163 wait complete. 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.215+0000 7f7a78c00700 1 Processor -- start 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.215+0000 7f7a78c00700 1 -- start start 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a78c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7a74105780 0x7f7a7419bea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a78c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a74106140 0x7f7a7419c3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a78c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7a7410b780 0x7f7a741a0720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a78c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7a741159b0 con 0x7f7a74105780 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a78c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7a74115830 con 0x7f7a74106140 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a78c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7a74115b30 con 0x7f7a7410b780 2024-05-20T20:17:36.271 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a74106140 0x7f7a7419c3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a74106140 0x7f7a7419c3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:59586/0 (socket says 172.21.15.28:59586) 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a72800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7a7410b780 0x7f7a741a0720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 -- 172.21.15.28:0/1884940638 learned_addr learned my addr 172.21.15.28:0/1884940638 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 -- 172.21.15.28:0/1884940638 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7a7410b780 msgr2=0x7f7a741a0720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7a7410b780 0x7f7a741a0720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 -- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7a74105780 msgr2=0x7f7a7419bea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7a74105780 0x7f7a7419bea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a71400700 1 -- 172.21.15.28:0/1884940638 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7a741a0e60 con 0x7f7a74106140 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.216+0000 7f7a72800700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7a7410b780 0x7f7a741a0720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.217+0000 7f7a71400700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a74106140 0x7f7a7419c3e0 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f7a60005910 tx=0x7f7a6000b670 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.217+0000 7f7a6aa00700 1 -- 172.21.15.28:0/1884940638 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a60027070 con 0x7f7a74106140 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.217+0000 7f7a6aa00700 1 -- 172.21.15.28:0/1884940638 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7a60022470 con 0x7f7a74106140 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.217+0000 7f7a6aa00700 1 -- 172.21.15.28:0/1884940638 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7a60017650 con 0x7f7a74106140 2024-05-20T20:17:36.272 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.218+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7a741a10f0 con 0x7f7a74106140 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.218+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7a741a89d0 con 0x7f7a74106140 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.219+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7a74106ec0 con 0x7f7a74106140 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.222+0000 7f7a6aa00700 1 -- 172.21.15.28:0/1884940638 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7a6002a070 con 0x7f7a74106140 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.225+0000 7f7a6aa00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7a580b2a70 0x7f7a580b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.226+0000 7f7a6aa00700 1 -- 172.21.15.28:0/1884940638 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f7a600e3600 con 0x7f7a74106140 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.226+0000 7f7a71e00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7a580b2a70 0x7f7a580b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.227+0000 7f7a71e00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7a580b2a70 0x7f7a580b4f30 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f7a64009b80 tx=0x7f7a64009580 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:36.273 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.229+0000 7f7a6aa00700 1 -- 172.21.15.28:0/1884940638 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7a60013070 con 0x7f7a74106140 2024-05-20T20:17:36.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.848+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f7a7419d1b0 con 0x7f7a74106140 2024-05-20T20:17:36.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.849+0000 7f7a6aa00700 1 -- 172.21.15.28:0/1884940638 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f7a600ab000 con 0x7f7a74106140 2024-05-20T20:17:36.850 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:17:36.851 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:17:36.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7a580b2a70 msgr2=0x7f7a580b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:36.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7a580b2a70 0x7f7a580b4f30 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f7a64009b80 tx=0x7f7a64009580 comp rx=0 tx=0).stop 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a74106140 msgr2=0x7f7a7419c3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a74106140 0x7f7a7419c3e0 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7f7a60005910 tx=0x7f7a6000b670 comp rx=0 tx=0).stop 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 shutdown_connections 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7a7410b780 0x7f7a741a0720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7a74105780 0x7f7a7419bea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7a580b2a70 0x7f7a580b4f30 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 --2- 172.21.15.28:0/1884940638 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7a74106140 0x7f7a7419c3e0 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:36.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.853+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 >> 172.21.15.28:0/1884940638 conn(0x7f7a74101270 msgr2=0x7f7a74101890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:36.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.854+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 shutdown_connections 2024-05-20T20:17:36.856 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:36.854+0000 7f7a78c00700 1 -- 172.21.15.28:0/1884940638 wait complete. 2024-05-20T20:17:36.860 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:17:37.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:37 smithi028 ceph-mon[31178]: pgmap v374: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 248 MiB/s wr, 1.60k op/s 2024-05-20T20:17:37.305 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:37 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1884940638' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:37.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:37 smithi154 ceph-mon[36314]: pgmap v374: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 248 MiB/s wr, 1.60k op/s 2024-05-20T20:17:37.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:37 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1884940638' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:37.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:37 smithi033 ceph-mon[36179]: pgmap v374: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 248 MiB/s wr, 1.60k op/s 2024-05-20T20:17:37.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:37 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1884940638' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:39.553 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:39 smithi028 ceph-mon[31178]: pgmap v375: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 247 MiB/s wr, 1.53k op/s 2024-05-20T20:17:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:39 smithi154 ceph-mon[36314]: pgmap v375: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 247 MiB/s wr, 1.53k op/s 2024-05-20T20:17:39.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:39 smithi033 ceph-mon[36179]: pgmap v375: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 247 MiB/s wr, 1.53k op/s 2024-05-20T20:17:40.058 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 137.9 --await-for 45.0' 2024-05-20T20:17:40.058 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 137.9 --await-for 45.0 2024-05-20T20:17:40.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:40 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:17:40.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:40 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:17:40.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:40 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:17:41.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:41 smithi028 ceph-mon[31178]: pgmap v376: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 332 MiB/s wr, 1.85k op/s 2024-05-20T20:17:41.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:41 smithi154 ceph-mon[36314]: pgmap v376: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 332 MiB/s wr, 1.85k op/s 2024-05-20T20:17:41.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:41 smithi033 ceph-mon[36179]: pgmap v376: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 332 MiB/s wr, 1.85k op/s 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.609+0000 7fc4ec000700 1 -- 172.21.15.28:0/2437400790 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e406cf50 msgr2=0x7fc4e406d330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.609+0000 7fc4ec000700 1 --2- 172.21.15.28:0/2437400790 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e406cf50 0x7fc4e406d330 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8009990 tx=0x7fc4d8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.609+0000 7fc4ec000700 1 -- 172.21.15.28:0/2437400790 shutdown_connections 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.609+0000 7fc4ec000700 1 --2- 172.21.15.28:0/2437400790 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4e411cd60 0x7fc4e411f150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.609+0000 7fc4ec000700 1 --2- 172.21.15.28:0/2437400790 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e406cf50 0x7fc4e406d330 secure :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8009990 tx=0x7fc4d8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.609+0000 7fc4ec000700 1 --2- 172.21.15.28:0/2437400790 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4e406d870 0x7fc4e411c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.609+0000 7fc4ec000700 1 -- 172.21.15.28:0/2437400790 >> 172.21.15.28:0/2437400790 conn(0x7fc4e4070500 msgr2=0x7fc4e4072920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:41.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.612+0000 7fc4ec000700 1 -- 172.21.15.28:0/2437400790 shutdown_connections 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.612+0000 7fc4ec000700 1 -- 172.21.15.28:0/2437400790 wait complete. 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.613+0000 7fc4ec000700 1 Processor -- start 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.614+0000 7fc4ec000700 1 -- start start 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.615+0000 7fc4ec000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4e406d870 0x7fc4e4118080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.615+0000 7fc4ec000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e411cd60 0x7fc4e4111160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.615+0000 7fc4ec000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4e41116a0 0x7fc4e4111b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.616+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e411cd60 0x7fc4e4111160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.616+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e411cd60 0x7fc4e4111160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60192/0 (socket says 172.21.15.28:60192) 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.616+0000 7fc4e8c00700 1 -- 172.21.15.28:0/1482462300 learned_addr learned my addr 172.21.15.28:0/1482462300 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.616+0000 7fc4e9600700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4e406d870 0x7fc4e4118080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.617+0000 7fc4ea000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4e41116a0 0x7fc4e4111b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.617+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc4e41226c0 con 0x7fc4e41116a0 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.617+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc4e4122540 con 0x7fc4e411cd60 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.617+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc4e4122840 con 0x7fc4e406d870 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.620+0000 7fc4ea000700 1 -- 172.21.15.28:0/1482462300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4e406d870 msgr2=0x7fc4e4118080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.620+0000 7fc4ea000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4e406d870 0x7fc4e4118080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.620+0000 7fc4ea000700 1 -- 172.21.15.28:0/1482462300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e411cd60 msgr2=0x7fc4e4111160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.620+0000 7fc4ea000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e411cd60 0x7fc4e4111160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.620+0000 7fc4ea000700 1 -- 172.21.15.28:0/1482462300 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc4e4115da0 con 0x7fc4e41116a0 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.621+0000 7fc4e9600700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4e406d870 0x7fc4e4118080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.621+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e411cd60 0x7fc4e4111160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.623+0000 7fc4ea000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4e41116a0 0x7fc4e4111b20 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fc4dc00bad0 tx=0x7fc4dc00bde0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.623+0000 7fc4e2000700 1 -- 172.21.15.28:0/1482462300 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4dc019070 con 0x7fc4e41116a0 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.624+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc4e4116090 con 0x7fc4e41116a0 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.625+0000 7fc4e2000700 1 -- 172.21.15.28:0/1482462300 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4dc00cde0 con 0x7fc4e41116a0 2024-05-20T20:17:41.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.625+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc4e41165d0 con 0x7fc4e41116a0 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.627+0000 7fc4e2000700 1 -- 172.21.15.28:0/1482462300 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4dc01d3b0 con 0x7fc4e41116a0 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.638+0000 7fc4e2000700 1 -- 172.21.15.28:0/1482462300 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fc4dc01c070 con 0x7fc4e41116a0 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.640+0000 7fc4e2000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4cc0b2b20 0x7fc4cc0b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.641+0000 7fc4f69b2b80 1 Processor -- start 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.641+0000 7fc4e2000700 1 -- 172.21.15.28:0/1482462300 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fc4dc0e2800 con 0x7fc4e41116a0 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.643+0000 7fc4f69b2b80 1 -- start start 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.643+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6610 0x561a4d0d69f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.643+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d0d0b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.643+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d1140 0x561a4d0d15c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.643+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561a4d02ee40 con 0x561a4d0d6610 2024-05-20T20:17:41.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.643+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561a4d01e9e0 con 0x561a4d0d6f30 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.643+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561a4d022a20 con 0x561a4d0d1140 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.644+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d0d0b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.644+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d0d0b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60208/0 (socket says 172.21.15.28:60208) 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.644+0000 7fc4e8c00700 1 -- 172.21.15.28:0/1480363895 learned_addr learned my addr 172.21.15.28:0/1480363895 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.645+0000 7fc4ea000700 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d1140 0x561a4d0d15c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.646+0000 7fc4e9600700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4cc0b2b20 0x7fc4cc0b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.646+0000 7fc4e8c00700 1 -- 172.21.15.28:0/1480363895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d1140 msgr2=0x561a4d0d15c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.646+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d1140 0x561a4d0d15c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.646+0000 7fc4e8c00700 1 -- 172.21.15.28:0/1480363895 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6610 msgr2=0x561a4d0d69f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.646+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6610 0x561a4d0d69f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.646+0000 7fc4e8c00700 1 -- 172.21.15.28:0/1480363895 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561a4cfe7420 con 0x561a4d0d6f30 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.647+0000 7fc4ea000700 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d1140 0x561a4d0d15c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:17:41.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.648+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d0d0b20 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fc4d0009f20 tx=0x7fc4d000e4b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cc37dd411e6199c1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.649+0000 7fc4d6a00700 1 -- 172.21.15.28:0/1480363895 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d0017070 con 0x561a4d0d6f30 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.649+0000 7fc4d6a00700 1 -- 172.21.15.28:0/1480363895 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4d000ed40 con 0x561a4d0d6f30 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.649+0000 7fc4d6a00700 1 -- 172.21.15.28:0/1480363895 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d0012700 con 0x561a4d0d6f30 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.651+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1480363895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 msgr2=0x561a4d0d0b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.651+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d0d0b20 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7fc4d0009f20 tx=0x7fc4d000e4b0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.653+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1480363895 shutdown_connections 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.653+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d1140 0x561a4d0d15c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.653+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6610 0x561a4d0d69f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.653+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1480363895 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d0d0b20 unknown :-1 s=CLOSED pgs=577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.653+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1480363895 >> 172.21.15.28:0/1480363895 conn(0x561a4cffebe0 msgr2=0x561a4d0cb690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.654+0000 7fc4e9600700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4cc0b2b20 0x7fc4cc0b4fe0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fc4d800b5c0 tx=0x7fc4d801a040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.655+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1480363895 shutdown_connections 2024-05-20T20:17:41.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.655+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1480363895 wait complete. 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.656+0000 7fc4f69b2b80 1 -- start start 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.656+0000 7fc4f69b2b80 1 Processor -- start 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.657+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d1140 0x561a4d0dcc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.657+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d0da010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.657+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6f30 0x561a4d0da550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.657+0000 7fc4e9600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d1140 0x561a4d0dcc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.657+0000 7fc4e9600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d1140 0x561a4d0dcc70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60220/0 (socket says 172.21.15.28:60220) 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.657+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d0da010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.658+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d0da010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37966/0 (socket says 172.21.15.28:37966) 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.658+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x561a4cf24fa0 con 0x561a4d0d6f30 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.658+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x561a4cfbf760 con 0x561a4d0d1140 2024-05-20T20:17:41.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.658+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x561a4d0dd9e0 con 0x561a4d0d6610 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.658+0000 7fc4e9600700 1 -- 172.21.15.28:0/3249764476 learned_addr learned my addr 172.21.15.28:0/3249764476 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4f69b2b80 10 client.0 _ll_register_callbacks cb 0x561a4cfc8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4ea000700 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6f30 0x561a4d0da550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4e8c00700 1 -- 172.21.15.28:0/3249764476 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d1140 msgr2=0x561a4d0dcc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d1140 0x561a4d0dcc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4e8c00700 1 -- 172.21.15.28:0/3249764476 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6f30 msgr2=0x561a4d0da550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6f30 0x561a4d0da550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4e8c00700 1 -- 172.21.15.28:0/3249764476 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561a4cf6e8d0 con 0x561a4d0d6610 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d0da010 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7fc4d0009f20 tx=0x7fc4d0013bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=226da2c30dfb7bfb server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.659+0000 7fc4cb400700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.662+0000 7fc4cb400700 1 -- 172.21.15.28:0/3249764476 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d0022070 con 0x561a4d0d6610 2024-05-20T20:17:41.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.662+0000 7fc4cb400700 1 -- 172.21.15.28:0/3249764476 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fc4d001c860 con 0x561a4d0d6610 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.662+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc4d000ed40 con 0x561a4d0d6610 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.664+0000 7fc4cb400700 1 -- 172.21.15.28:0/3249764476 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc4d002c050 con 0x561a4d0d6610 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.664+0000 7fc4f69b2b80 10 client.27745 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.664+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561a4cfef060 con 0x561a4d0d6610 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.665+0000 7fc4cb400700 1 -- 172.21.15.28:0/3249764476 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4d000ed40 con 0x561a4d0d6610 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.665+0000 7fc4cb400700 1 -- 172.21.15.28:0/3249764476 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc4d0027c70 con 0x561a4d0d6610 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.666+0000 7fc4f69b2b80 10 client.27745 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.666+0000 7fc4f69b2b80 10 client.27745 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.666+0000 7fc4f69b2b80 20 client.27745 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.666+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d10e800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.666+0000 7fc4f69b2b80 4 client.27745 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.666+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x561a4d030d90 con 0x561a4d10c410 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.667+0000 7fc4e9600700 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d10e800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.668+0000 7fc4e9600700 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d10e800 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=32f338d1b062589e server_cookie=141d345142372a58 in_seq=0 out_seq=0 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.668+0000 7fc4cb400700 10 client.27745 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.669+0000 7fc4cb400700 1 -- 172.21.15.28:0/3249764476 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x561a4d030d90 con 0x561a4d10c410 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.669+0000 7fc4cb400700 10 client.27745 handle_command_reply: tid=0 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.669+0000 7fc4f69b2b80 1 client.27745 shutdown 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.669+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 msgr2=0x561a4d0da010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.669+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d0da010 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7fc4d0009f20 tx=0x7fc4d0013bc0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 shutdown_connections 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d6f30 0x561a4d0da550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d1140 0x561a4d0dcc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d10e800 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4cb400700 0 client.27745 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/3249764476 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d0da010 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 >> 172.21.15.28:0/3249764476 conn(0x561a4cffebe0 msgr2=0x561a4d0cad70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:41.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 shutdown_connections 2024-05-20T20:17:41.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/3249764476 wait complete. 2024-05-20T20:17:41.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.670+0000 7fc4f69b2b80 20 client.27745 trim_cache size 0 max 16384 2024-05-20T20:17:41.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.695+0000 7fc4f69b2b80 1 Processor -- start 2024-05-20T20:17:41.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.695+0000 7fc4f69b2b80 1 -- start start 2024-05-20T20:17:41.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.695+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0d7ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.697+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6610 0x561a4d0cb4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.697+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6f30 0x561a4d113e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.697+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc4c400d6e0 con 0x561a4d0d1140 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.697+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc4c400f220 con 0x561a4d0d6610 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.697+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc4c400ddb0 con 0x561a4d0d6f30 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.698+0000 7fc4e9600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0d7ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.698+0000 7fc4e9600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0d7ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42868/0 (socket says 172.21.15.28:42868) 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.698+0000 7fc4e9600700 1 -- 172.21.15.28:0/1362137684 learned_addr learned my addr 172.21.15.28:0/1362137684 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.698+0000 7fc4ea000700 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6f30 0x561a4d113e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.698+0000 7fc4e9600700 1 -- 172.21.15.28:0/1362137684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6f30 msgr2=0x561a4d113e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.698+0000 7fc4e9600700 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6f30 0x561a4d113e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.698+0000 7fc4e9600700 1 -- 172.21.15.28:0/1362137684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6610 msgr2=0x561a4d0cb4f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.700+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6610 0x561a4d0cb4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.700+0000 7fc4e9600700 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6610 0x561a4d0cb4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.700+0000 7fc4e9600700 1 -- 172.21.15.28:0/1362137684 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x561a4d0d0e60 con 0x561a4d0d1140 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.700+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6610 0x561a4d0cb4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:41.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.701+0000 7fc4e9600700 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0d7ae0 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8009990 tx=0x7fc4d801d000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=927e945a1e8351d3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:41.705 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.703+0000 7fc4d6000700 1 -- 172.21.15.28:0/1362137684 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d8005270 con 0x561a4d0d1140 2024-05-20T20:17:41.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.704+0000 7fc4d6000700 1 -- 172.21.15.28:0/1362137684 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4d8034070 con 0x561a4d0d1140 2024-05-20T20:17:41.706 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.704+0000 7fc4d6000700 1 -- 172.21.15.28:0/1362137684 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d800f460 con 0x561a4d0d1140 2024-05-20T20:17:41.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.711+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1362137684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 msgr2=0x561a4d0d7ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.711+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0d7ae0 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8009990 tx=0x7fc4d801d000 comp rx=0 tx=0).stop 2024-05-20T20:17:41.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.712+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1362137684 shutdown_connections 2024-05-20T20:17:41.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.712+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6f30 0x561a4d113e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.712+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0d7ae0 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.712+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/1362137684 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6610 0x561a4d0cb4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.712+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1362137684 >> 172.21.15.28:0/1362137684 conn(0x561a4cffebe0 msgr2=0x561a4d0d73d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:41.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.713+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1362137684 shutdown_connections 2024-05-20T20:17:41.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.713+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/1362137684 wait complete. 2024-05-20T20:17:41.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.713+0000 7fc4f69b2b80 1 -- start start 2024-05-20T20:17:41.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.714+0000 7fc4f69b2b80 1 Processor -- start 2024-05-20T20:17:41.718 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.716+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0e3460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.720 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.716+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d113730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.716+0000 7fc4f69b2b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d113d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.716+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc4c400eb50 con 0x561a4d0d1140 2024-05-20T20:17:41.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.716+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc4c400e480 con 0x561a4d0d6f30 2024-05-20T20:17:41.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.716+0000 7fc4f69b2b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc4c400b6a0 con 0x561a4d0d6610 2024-05-20T20:17:41.722 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.716+0000 7fc4f69b2b80 10 client.0 _ll_register_callbacks cb 0x561a4cfc8c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:41.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.724+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d113730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.724+0000 7fc4e8c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d113730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37992/0 (socket says 172.21.15.28:37992) 2024-05-20T20:17:41.728 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.724+0000 7fc4e8c00700 1 -- 172.21.15.28:0/2178282528 learned_addr learned my addr 172.21.15.28:0/2178282528 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.726+0000 7fc4ea000700 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d113d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.726+0000 7fc4e9600700 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0e3460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.727+0000 7fc4e9600700 1 -- 172.21.15.28:0/2178282528 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 msgr2=0x561a4d113730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.727+0000 7fc4e9600700 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d113730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.727+0000 7fc4e9600700 1 -- 172.21.15.28:0/2178282528 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 msgr2=0x561a4d113d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.727+0000 7fc4e9600700 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d113d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.727+0000 7fc4e9600700 1 -- 172.21.15.28:0/2178282528 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x561a4d175cd0 con 0x561a4d0d1140 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.732+0000 7fc4e9600700 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0e3460 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8009990 tx=0x7fc4d800bce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d18010cd97a16b49 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.732+0000 7fc4cb400700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:17:41.738 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.734+0000 7fc4cb400700 1 -- 172.21.15.28:0/2178282528 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d803d070 con 0x561a4d0d1140 2024-05-20T20:17:41.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.734+0000 7fc4cb400700 1 -- 172.21.15.28:0/2178282528 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fc4d802f380 con 0x561a4d0d1140 2024-05-20T20:17:41.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.735+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fc4d8034070 con 0x561a4d0d1140 2024-05-20T20:17:41.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.737+0000 7fc4cb400700 1 -- 172.21.15.28:0/2178282528 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fc4d802f580 con 0x561a4d0d1140 2024-05-20T20:17:41.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.737+0000 7fc4f69b2b80 10 client.17259 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:41.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.737+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x561a4d176d60 con 0x561a4d0d1140 2024-05-20T20:17:41.739 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.738+0000 7fc4cb400700 1 -- 172.21.15.28:0/2178282528 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fc4d8034070 con 0x561a4d0d1140 2024-05-20T20:17:41.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.738+0000 7fc4cb400700 1 -- 172.21.15.28:0/2178282528 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fc4d800f6a0 con 0x561a4d0d1140 2024-05-20T20:17:41.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.739+0000 7fc4f69b2b80 10 client.17259 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:41.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.739+0000 7fc4f69b2b80 10 client.17259 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:17:41.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.739+0000 7fc4f69b2b80 20 client.17259 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:41.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.739+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d1793d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:41.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.739+0000 7fc4f69b2b80 4 client.17259 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:17:41.740 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.739+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x561a4d02a570 con 0x561a4d10c410 2024-05-20T20:17:41.743 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.742+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d1793d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:41.745 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.744+0000 7fc4e8c00700 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d1793d0 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=8a63e4b29ee9cc98 server_cookie=e1b935a285ff6225 in_seq=0 out_seq=0 2024-05-20T20:17:41.746 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.745+0000 7fc4cb400700 10 client.17259 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:17:41.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.745+0000 7fc4cb400700 1 -- 172.21.15.28:0/2178282528 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x561a4d02a570 con 0x561a4d10c410 2024-05-20T20:17:41.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.745+0000 7fc4cb400700 10 client.17259 handle_command_reply: tid=0 2024-05-20T20:17:41.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.745+0000 7fc4f69b2b80 1 client.17259 shutdown 2024-05-20T20:17:41.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.746+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 msgr2=0x561a4d0e3460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.746+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0e3460 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8009990 tx=0x7fc4d800bce0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.747+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 shutdown_connections 2024-05-20T20:17:41.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.748+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x561a4d0d6f30 0x561a4d113d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.748+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x561a4d0d1140 0x561a4d0e3460 unknown :-1 s=CLOSED pgs=467 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.748+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x561a4d10c410 0x561a4d1793d0 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.748+0000 7fc4f69b2b80 1 --2- 172.21.15.28:0/2178282528 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x561a4d0d6610 0x561a4d113730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.748+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 >> 172.21.15.28:0/2178282528 conn(0x561a4cffebe0 msgr2=0x561a4d0d8df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:41.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.749+0000 7fc4cb400700 0 client.17259 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:17:41.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.750+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 shutdown_connections 2024-05-20T20:17:41.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.751+0000 7fc4f69b2b80 1 -- 172.21.15.28:0/2178282528 wait complete. 2024-05-20T20:17:41.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.751+0000 7fc4f69b2b80 20 client.17259 trim_cache size 0 max 16384 2024-05-20T20:17:41.753 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:17:41.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.754+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4cc0b2b20 msgr2=0x7fc4cc0b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.754+0000 7fc4ec000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4cc0b2b20 0x7fc4cc0b4fe0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fc4d800b5c0 tx=0x7fc4d801a040 comp rx=0 tx=0).stop 2024-05-20T20:17:41.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.754+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4e41116a0 msgr2=0x7fc4e4111b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:41.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.754+0000 7fc4ec000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4e41116a0 0x7fc4e4111b20 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7fc4dc00bad0 tx=0x7fc4dc00bde0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.756+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 shutdown_connections 2024-05-20T20:17:41.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.756+0000 7fc4ec000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc4e41116a0 0x7fc4e4111b20 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.756+0000 7fc4ec000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc4e406d870 0x7fc4e4118080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.756+0000 7fc4ec000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fc4cc0b2b20 0x7fc4cc0b4fe0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.756+0000 7fc4ec000700 1 --2- 172.21.15.28:0/1482462300 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc4e411cd60 0x7fc4e4111160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:41.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.756+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 >> 172.21.15.28:0/1482462300 conn(0x7fc4e4070500 msgr2=0x7fc4e411b0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:41.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.757+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 shutdown_connections 2024-05-20T20:17:41.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:41.758+0000 7fc4ec000700 1 -- 172.21.15.28:0/1482462300 wait complete. 2024-05-20T20:17:42.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:17:42.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:17:42.557 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:42 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:17:42.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:17:42.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:17:42.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:42 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:17:42.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:17:42.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:17:42.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:42 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:17:43.309 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:43 smithi028 ceph-mon[31178]: pgmap v377: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 243 MiB/s wr, 1.20k op/s 2024-05-20T20:17:43.526 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:43.561 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:17:43.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:43 smithi154 ceph-mon[36314]: pgmap v377: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 243 MiB/s wr, 1.20k op/s 2024-05-20T20:17:43.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:43 smithi033 ceph-mon[36179]: pgmap v377: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 243 MiB/s wr, 1.20k op/s 2024-05-20T20:17:45.011 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 -- 172.21.15.28:0/1150228777 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8108880 msgr2=0x7f17e8108c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 --2- 172.21.15.28:0/1150228777 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8108880 0x7f17e8108c60 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f17e0009be0 tx=0x7f17e002e860 comp rx=0 tx=0).stop 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 -- 172.21.15.28:0/1150228777 shutdown_connections 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 --2- 172.21.15.28:0/1150228777 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e8103230 0x7f17e810f740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 --2- 172.21.15.28:0/1150228777 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e8102870 0x7f17e8102cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 --2- 172.21.15.28:0/1150228777 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8108880 0x7f17e8108c60 unknown :-1 s=CLOSED pgs=578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 -- 172.21.15.28:0/1150228777 >> 172.21.15.28:0/1150228777 conn(0x7f17e80fe3a0 msgr2=0x7f17e81007c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 -- 172.21.15.28:0/1150228777 shutdown_connections 2024-05-20T20:17:45.012 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.010+0000 7f17e7200640 1 -- 172.21.15.28:0/1150228777 wait complete. 2024-05-20T20:17:45.013 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.012+0000 7f17e7200640 1 Processor -- start 2024-05-20T20:17:45.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.013+0000 7f17e7200640 1 -- start start 2024-05-20T20:17:45.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.013+0000 7f17e7200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8102870 0x7f17e8196310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.013+0000 7f17e7200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e8103230 0x7f17e8196850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.013+0000 7f17e5e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8102870 0x7f17e8196310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.014+0000 7f17e5e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8102870 0x7f17e8196310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:60258/0 (socket says 172.21.15.28:60258) 2024-05-20T20:17:45.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.014+0000 7f17e7200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e8108880 0x7f17e8196d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.014+0000 7f17e7200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f17e8110980 con 0x7f17e8108880 2024-05-20T20:17:45.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.014+0000 7f17e7200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f17e8110800 con 0x7f17e8102870 2024-05-20T20:17:45.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.014+0000 7f17e7200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f17e8110b00 con 0x7f17e8103230 2024-05-20T20:17:45.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.014+0000 7f17e5400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e8103230 0x7f17e8196850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.014+0000 7f17e5e00640 1 -- 172.21.15.28:0/3407188275 learned_addr learned my addr 172.21.15.28:0/3407188275 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:45.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.015+0000 7f17e6800640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e8108880 0x7f17e8196d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.015+0000 7f17e5e00640 1 -- 172.21.15.28:0/3407188275 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e8103230 msgr2=0x7f17e8196850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.015+0000 7f17e5e00640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e8103230 0x7f17e8196850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.015+0000 7f17e5e00640 1 -- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e8108880 msgr2=0x7f17e8196d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.015+0000 7f17e5e00640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e8108880 0x7f17e8196d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.015+0000 7f17e5e00640 1 -- 172.21.15.28:0/3407188275 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f17e81b41a0 con 0x7f17e8102870 2024-05-20T20:17:45.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.015+0000 7f17e6800640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e8108880 0x7f17e8196d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:45.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.016+0000 7f17e5e00640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8102870 0x7f17e8196310 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f17e0006010 tx=0x7f17e0034c20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:45.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.016+0000 7f17dea00640 1 -- 172.21.15.28:0/3407188275 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17e0042030 con 0x7f17e8102870 2024-05-20T20:17:45.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.017+0000 7f17dea00640 1 -- 172.21.15.28:0/3407188275 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f17e0003e20 con 0x7f17e8102870 2024-05-20T20:17:45.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.017+0000 7f17dea00640 1 -- 172.21.15.28:0/3407188275 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17e0004280 con 0x7f17e8102870 2024-05-20T20:17:45.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.017+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f17e81b4430 con 0x7f17e8102870 2024-05-20T20:17:45.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.017+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f17e81b48d0 con 0x7f17e8102870 2024-05-20T20:17:45.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.021+0000 7f17efd98740 1 Processor -- start 2024-05-20T20:17:45.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.022+0000 7f17efd98740 1 -- start start 2024-05-20T20:17:45.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.022+0000 7f17efd98740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697d9230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.022+0000 7f17efd98740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 0x55a8697e3af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.022+0000 7f17efd98740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697e6420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.022+0000 7f17efd98740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a869749b90 con 0x55a8697d8e50 2024-05-20T20:17:45.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.022+0000 7f17efd98740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a869739730 con 0x55a8697d9770 2024-05-20T20:17:45.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.022+0000 7f17efd98740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a86973d770 con 0x55a8697e4030 2024-05-20T20:17:45.024 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.023+0000 7f17e6800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697d9230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.023+0000 7f17e6800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697d9230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:42904/0 (socket says 172.21.15.28:42904) 2024-05-20T20:17:45.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.023+0000 7f17e6800640 1 -- 172.21.15.28:0/3986252772 learned_addr learned my addr 172.21.15.28:0/3986252772 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:45.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.024+0000 7f17e6800640 1 -- 172.21.15.28:0/3986252772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 msgr2=0x55a8697e6420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:45.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.024+0000 7f17e5400640 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697e6420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.024+0000 7f17e6800640 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697e6420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.024+0000 7f17e6800640 1 -- 172.21.15.28:0/3986252772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 msgr2=0x55a8697e3af0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:17:45.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.024+0000 7f17e6800640 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 0x55a8697e3af0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.024+0000 7f17e6800640 1 -- 172.21.15.28:0/3986252772 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a8696d28b0 con 0x55a8697d8e50 2024-05-20T20:17:45.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.024+0000 7f17e5400640 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697e6420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:45.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.025+0000 7f17e6800640 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697d9230 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f17d4009b80 tx=0x7f17d400f730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9733eca3c71cba0c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:45.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.026+0000 7f17d3400640 1 -- 172.21.15.28:0/3986252772 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4011440 con 0x55a8697d8e50 2024-05-20T20:17:45.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.026+0000 7f17d3400640 1 -- 172.21.15.28:0/3986252772 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f17d4011a90 con 0x55a8697d8e50 2024-05-20T20:17:45.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.026+0000 7f17d3400640 1 -- 172.21.15.28:0/3986252772 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4011440 con 0x55a8697d8e50 2024-05-20T20:17:45.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.026+0000 7f17dea00640 1 -- 172.21.15.28:0/3407188275 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f17e0045070 con 0x7f17e8102870 2024-05-20T20:17:45.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.027+0000 7f17efd98740 1 -- 172.21.15.28:0/3986252772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 msgr2=0x55a8697d9230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.027+0000 7f17efd98740 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697d9230 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7f17d4009b80 tx=0x7f17d400f730 comp rx=0 tx=0).stop 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.028+0000 7f17efd98740 1 -- 172.21.15.28:0/3986252772 shutdown_connections 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.028+0000 7f17efd98740 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 0x55a8697e3af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.028+0000 7f17efd98740 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697e6420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.028+0000 7f17efd98740 1 --2- 172.21.15.28:0/3986252772 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697d9230 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.028+0000 7f17efd98740 1 -- 172.21.15.28:0/3986252772 >> 172.21.15.28:0/3986252772 conn(0x55a8697d5360 msgr2=0x55a8697d5740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.028+0000 7f17efd98740 1 -- 172.21.15.28:0/3986252772 shutdown_connections 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.028+0000 7f17efd98740 1 -- 172.21.15.28:0/3986252772 wait complete. 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.029+0000 7f17efd98740 1 -- start start 2024-05-20T20:17:45.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.029+0000 7f17efd98740 1 Processor -- start 2024-05-20T20:17:45.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.030+0000 7f17efd98740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d8e50 0x55a8697e39c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.031+0000 7f17dea00640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17cc0b2a80 0x7f17cc0b4f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.031+0000 7f17dea00640 1 -- 172.21.15.28:0/3407188275 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f17e0103200 con 0x7f17e8102870 2024-05-20T20:17:45.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.031+0000 7f17efd98740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d9770 0x55a8697e0cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.031+0000 7f17efd98740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697e4030 0x55a8697e1230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.031+0000 7f17efd98740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a86941f790 con 0x55a8697d9770 2024-05-20T20:17:45.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.031+0000 7f17efd98740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a8696e1bc0 con 0x55a8697e4030 2024-05-20T20:17:45.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.031+0000 7f17efd98740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a8697e6c60 con 0x55a8697d8e50 2024-05-20T20:17:45.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.032+0000 7f17efd98740 10 client.0 _ll_register_callbacks cb 0x55a8696bf400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:45.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.032+0000 7f17e6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d8e50 0x55a8697e39c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.033 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.032+0000 7f17e6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d8e50 0x55a8697e39c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38038/0 (socket says 172.21.15.28:38038) 2024-05-20T20:17:45.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.032+0000 7f17e6800640 1 -- 172.21.15.28:0/199848598 learned_addr learned my addr 172.21.15.28:0/199848598 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:45.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.032+0000 7f17e5e00640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17cc0b2a80 0x7f17cc0b4f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.034 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.032+0000 7f17e5400640 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d9770 0x55a8697e0cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17e6800640 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697e4030 0x55a8697e1230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17e6800640 1 -- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697e4030 msgr2=0x55a8697e1230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17e6800640 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697e4030 0x55a8697e1230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17e6800640 1 -- 172.21.15.28:0/199848598 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d9770 msgr2=0x55a8697e0cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17e6800640 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d9770 0x55a8697e0cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17e6800640 1 -- 172.21.15.28:0/199848598 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a86944e210 con 0x55a8697d8e50 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17e6800640 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d8e50 0x55a8697e39c0 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f17d4009b50 tx=0x7f17d40104e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=af0b12aa43c323e1 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:45.035 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.033+0000 7f17c7e00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:45.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.034+0000 7f17e5e00640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17cc0b2a80 0x7f17cc0b4f40 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x55a8697da3f0 tx=0x7f17e0005e30 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:45.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.034+0000 7f17c7e00640 1 -- 172.21.15.28:0/199848598 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4004220 con 0x55a8697d8e50 2024-05-20T20:17:45.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.035+0000 7f17c7e00640 1 -- 172.21.15.28:0/199848598 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f17d40043c0 con 0x55a8697d8e50 2024-05-20T20:17:45.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.035+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f17d4011a90 con 0x55a8697d8e50 2024-05-20T20:17:45.036 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.035+0000 7f17c7e00640 1 -- 172.21.15.28:0/199848598 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f17d40118e0 con 0x55a8697d8e50 2024-05-20T20:17:45.039 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.038+0000 7f17efd98740 10 client.27760 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:45.040 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.038+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a86951a620 con 0x55a8697d8e50 2024-05-20T20:17:45.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17c7e00640 1 -- 172.21.15.28:0/199848598 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f17d4011a90 con 0x55a8697d8e50 2024-05-20T20:17:45.041 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17c7e00640 1 -- 172.21.15.28:0/199848598 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f17d40038c0 con 0x55a8697d8e50 2024-05-20T20:17:45.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17efd98740 10 client.27760 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:45.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17efd98740 10 client.27760 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:45.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17efd98740 20 client.27760 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:45.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17efd98740 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a8697f38f0 0x55a8697f5ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17efd98740 4 client.27760 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:45.042 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.039+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a86974bae0 con 0x55a8697f38f0 2024-05-20T20:17:45.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.040+0000 7f17e5400640 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a8697f38f0 0x55a8697f5ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.041+0000 7f17e5400640 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a8697f38f0 0x55a8697f5ce0 crc :-1 s=READY pgs=247 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2b86b3b202e4b766 server_cookie=28475c61a8e7454c in_seq=0 out_seq=0 2024-05-20T20:17:45.043 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.041+0000 7f17c7e00640 10 client.27760 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:45.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.045+0000 7f17c7e00640 1 -- 172.21.15.28:0/199848598 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a86974bae0 con 0x55a8697f38f0 2024-05-20T20:17:45.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.045+0000 7f17c7e00640 10 client.27760 handle_command_reply: tid=0 2024-05-20T20:17:45.046 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.045+0000 7f17efd98740 1 client.27760 shutdown 2024-05-20T20:17:45.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.046+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d8e50 msgr2=0x55a8697e39c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.047 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.046+0000 7f17efd98740 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d8e50 0x55a8697e39c0 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f17d4009b50 tx=0x7f17d40104e0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 shutdown_connections 2024-05-20T20:17:45.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17efd98740 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a8697f38f0 0x55a8697f5ce0 crc :-1 s=READY pgs=247 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.048 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17efd98740 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697e4030 0x55a8697e1230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17efd98740 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d9770 0x55a8697e0cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17efd98740 1 --2- 172.21.15.28:0/199848598 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d8e50 0x55a8697e39c0 unknown :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 >> 172.21.15.28:0/199848598 conn(0x55a8697d5360 msgr2=0x55a8697db0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:45.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17c7e00640 0 client.27760 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:45.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.047+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 shutdown_connections 2024-05-20T20:17:45.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.048+0000 7f17efd98740 1 -- 172.21.15.28:0/199848598 wait complete. 2024-05-20T20:17:45.049 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.048+0000 7f17efd98740 20 client.27760 trim_cache size 0 max 16384 2024-05-20T20:17:45.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.103+0000 7f17efd98740 1 Processor -- start 2024-05-20T20:17:45.113 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.112+0000 7f17efd98740 1 -- start start 2024-05-20T20:17:45.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.113+0000 7f17efd98740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697e34f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.114 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.113+0000 7f17efd98740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 0x55a8697e3a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.113+0000 7f17efd98740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697fb3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.113+0000 7f17efd98740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f17c800e8a0 con 0x55a8697d8e50 2024-05-20T20:17:45.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.113+0000 7f17efd98740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f17c80103e0 con 0x55a8697d9770 2024-05-20T20:17:45.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.113+0000 7f17efd98740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f17c800ef70 con 0x55a8697e4030 2024-05-20T20:17:45.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.114+0000 7f17e6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697fb3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.114+0000 7f17e6800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697fb3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38048/0 (socket says 172.21.15.28:38048) 2024-05-20T20:17:45.115 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.114+0000 7f17e6800640 1 -- 172.21.15.28:0/1510843156 learned_addr learned my addr 172.21.15.28:0/1510843156 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:45.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.114+0000 7f17e5400640 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 0x55a8697e3a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.115+0000 7f17e6800640 1 -- 172.21.15.28:0/1510843156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 msgr2=0x55a8697e3a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:45.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.115+0000 7f17e6800640 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 0x55a8697e3a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.115+0000 7f17e6800640 1 -- 172.21.15.28:0/1510843156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 msgr2=0x55a8697e34f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:45.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.115+0000 7f17e6800640 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697e34f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.115+0000 7f17e6800640 1 -- 172.21.15.28:0/1510843156 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a8697df220 con 0x55a8697e4030 2024-05-20T20:17:45.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.115+0000 7f17e6800640 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697fb3d0 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f17d4009990 tx=0x7f17d4000bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=711f4ea8384e70d7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:45.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.116+0000 7f17d2a00640 1 -- 172.21.15.28:0/1510843156 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4037070 con 0x55a8697e4030 2024-05-20T20:17:45.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.116+0000 7f17d2a00640 1 -- 172.21.15.28:0/1510843156 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f17d4003aa0 con 0x55a8697e4030 2024-05-20T20:17:45.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.116+0000 7f17d2a00640 1 -- 172.21.15.28:0/1510843156 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4037070 con 0x55a8697e4030 2024-05-20T20:17:45.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.117+0000 7f17efd98740 1 -- 172.21.15.28:0/1510843156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 msgr2=0x55a8697fb3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.117+0000 7f17efd98740 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697fb3d0 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f17d4009990 tx=0x7f17d4000bc0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.118+0000 7f17efd98740 1 -- 172.21.15.28:0/1510843156 shutdown_connections 2024-05-20T20:17:45.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.118+0000 7f17efd98740 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d9770 0x55a8697e3a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.118+0000 7f17efd98740 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697e4030 0x55a8697fb3d0 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.118+0000 7f17efd98740 1 --2- 172.21.15.28:0/1510843156 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697d8e50 0x55a8697e34f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.118+0000 7f17efd98740 1 -- 172.21.15.28:0/1510843156 >> 172.21.15.28:0/1510843156 conn(0x55a8697d5360 msgr2=0x55a8697e0c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:45.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.119+0000 7f17efd98740 1 -- 172.21.15.28:0/1510843156 shutdown_connections 2024-05-20T20:17:45.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.119+0000 7f17efd98740 1 -- 172.21.15.28:0/1510843156 wait complete. 2024-05-20T20:17:45.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.120+0000 7f17efd98740 1 -- start start 2024-05-20T20:17:45.121 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.120+0000 7f17efd98740 1 Processor -- start 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.121+0000 7f17efd98740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d8e50 0x55a8697ec5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.121+0000 7f17efd98740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d9770 0x55a8697e8eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.121+0000 7f17efd98740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697e4030 0x55a8697e93f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.121+0000 7f17efd98740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f17c800fd10 con 0x55a8697e4030 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.121+0000 7f17efd98740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f17c800f640 con 0x55a8697d8e50 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.121+0000 7f17efd98740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f17c800c8a0 con 0x55a8697d9770 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.121+0000 7f17efd98740 10 client.0 _ll_register_callbacks cb 0x7f17d4015070 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:45.122 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.122+0000 7f17e5400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d9770 0x55a8697e8eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.122+0000 7f17e5400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d9770 0x55a8697e8eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:38050/0 (socket says 172.21.15.28:38050) 2024-05-20T20:17:45.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.122+0000 7f17e5400640 1 -- 172.21.15.28:0/733281960 learned_addr learned my addr 172.21.15.28:0/733281960 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:45.123 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.122+0000 7f17e6800640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697e4030 0x55a8697e93f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.124 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.123+0000 7f17e6800640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d8e50 0x55a8697ec5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17e6800640 1 -- 172.21.15.28:0/733281960 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d9770 msgr2=0x55a8697e8eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:45.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17e6800640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d9770 0x55a8697e8eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.128 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17e6800640 1 -- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d8e50 msgr2=0x55a8697ec5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:45.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17e6800640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d8e50 0x55a8697ec5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17e6800640 1 -- 172.21.15.28:0/733281960 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a8697e9c00 con 0x55a8697e4030 2024-05-20T20:17:45.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17e5400640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d9770 0x55a8697e8eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:45.129 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17e6800640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697e4030 0x55a8697e93f0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f17d4003e30 tx=0x7f17d40045d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1a52582dbeed63f3 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:45.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.128+0000 7f17c7e00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:17:45.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.129+0000 7f17c7e00640 1 -- 172.21.15.28:0/733281960 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17d4037070 con 0x55a8697e4030 2024-05-20T20:17:45.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.129+0000 7f17c7e00640 1 -- 172.21.15.28:0/733281960 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f17d40118e0 con 0x55a8697e4030 2024-05-20T20:17:45.130 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.129+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f17d4003aa0 con 0x55a8697e4030 2024-05-20T20:17:45.131 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.130+0000 7f17c7e00640 1 -- 172.21.15.28:0/733281960 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f17d400fc30 con 0x55a8697e4030 2024-05-20T20:17:45.131 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.130+0000 7f17efd98740 10 client.17271 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:45.131 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.130+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a86985d280 con 0x55a8697e4030 2024-05-20T20:17:45.131 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.130+0000 7f17c7e00640 1 -- 172.21.15.28:0/733281960 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f17d4003aa0 con 0x55a8697e4030 2024-05-20T20:17:45.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.131+0000 7f17c7e00640 1 -- 172.21.15.28:0/733281960 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f17d402ab00 con 0x55a8697e4030 2024-05-20T20:17:45.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.132+0000 7f17efd98740 10 client.17271 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:45.132 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.132+0000 7f17efd98740 10 client.17271 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:45.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.132+0000 7f17efd98740 20 client.17271 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:45.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.132+0000 7f17efd98740 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a86985d940 0x55a86985fd60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:45.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.132+0000 7f17efd98740 4 client.17271 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:17:45.133 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.132+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a8697452c0 con 0x55a86985d940 2024-05-20T20:17:45.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.132+0000 7f17e5400640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a86985d940 0x55a86985fd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:45.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.133+0000 7f17e5400640 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a86985d940 0x55a86985fd60 crc :-1 s=READY pgs=248 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c9fc9dd6a7cdc32e server_cookie=e59761a2d4bc1e1c in_seq=0 out_seq=0 2024-05-20T20:17:45.134 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.133+0000 7f17c7e00640 10 client.17271 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:45.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.191+0000 7f17c7e00640 1 -- 172.21.15.28:0/733281960 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55a8697452c0 con 0x55a86985d940 2024-05-20T20:17:45.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.191+0000 7f17c7e00640 10 client.17271 handle_command_reply: tid=0 2024-05-20T20:17:45.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.191+0000 7f17efd98740 1 client.17271 shutdown 2024-05-20T20:17:45.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.191+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697e4030 msgr2=0x55a8697e93f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.192 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.191+0000 7f17efd98740 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697e4030 0x55a8697e93f0 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f17d4003e30 tx=0x7f17d40045d0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 shutdown_connections 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a86985d940 0x55a86985fd60 crc :-1 s=READY pgs=248 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a8697e4030 0x55a8697e93f0 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a8697d9770 0x55a8697e8eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 --2- 172.21.15.28:0/733281960 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a8697d8e50 0x55a8697ec5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17c7e00640 0 client.17271 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 >> 172.21.15.28:0/733281960 conn(0x55a8697d5360 msgr2=0x55a8697d3c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 shutdown_connections 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 1 -- 172.21.15.28:0/733281960 wait complete. 2024-05-20T20:17:45.193 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.192+0000 7f17efd98740 20 client.17271 trim_cache size 0 max 16384 2024-05-20T20:17:45.198 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:17:45.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:17:45.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:17:45.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.195+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17cc0b2a80 msgr2=0x7f17cc0b4f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.195+0000 7f17e7200640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17cc0b2a80 0x7f17cc0b4f40 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x55a8697da3f0 tx=0x7f17e0005e30 comp rx=0 tx=0).stop 2024-05-20T20:17:45.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.195+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8102870 msgr2=0x7f17e8196310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:45.199 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.195+0000 7f17e7200640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8102870 0x7f17e8196310 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f17e0006010 tx=0x7f17e0034c20 comp rx=0 tx=0).stop 2024-05-20T20:17:45.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.196+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 shutdown_connections 2024-05-20T20:17:45.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.196+0000 7f17e7200640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f17cc0b2a80 0x7f17cc0b4f40 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.196+0000 7f17e7200640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f17e8108880 0x7f17e8196d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.200 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.196+0000 7f17e7200640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f17e8103230 0x7f17e8196850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.197+0000 7f17e7200640 1 --2- 172.21.15.28:0/3407188275 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f17e8102870 0x7f17e8196310 unknown :-1 s=CLOSED pgs=579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:45.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.197+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 >> 172.21.15.28:0/3407188275 conn(0x7f17e80fe3a0 msgr2=0x7f17e80fec50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:45.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.197+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 shutdown_connections 2024-05-20T20:17:45.201 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:45.198+0000 7f17e7200640 1 -- 172.21.15.28:0/3407188275 wait complete. 2024-05-20T20:17:45.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:45 smithi028 ceph-mon[31178]: pgmap v378: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 334 MiB/s wr, 1.54k op/s 2024-05-20T20:17:45.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:45 smithi154 ceph-mon[36314]: pgmap v378: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 334 MiB/s wr, 1.54k op/s 2024-05-20T20:17:45.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:45 smithi033 ceph-mon[36179]: pgmap v378: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 334 MiB/s wr, 1.54k op/s 2024-05-20T20:17:45.919 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:47.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:47 smithi028 ceph-mon[31178]: pgmap v379: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 257 MiB/s wr, 1.08k op/s 2024-05-20T20:17:47.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:47 smithi154 ceph-mon[36314]: pgmap v379: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 257 MiB/s wr, 1.08k op/s 2024-05-20T20:17:47.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:47 smithi033 ceph-mon[36179]: pgmap v379: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 257 MiB/s wr, 1.08k op/s 2024-05-20T20:17:48.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.593+0000 7fee41a00700 1 -- 172.21.15.28:0/3954670848 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 msgr2=0x7fee3c105cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.593+0000 7fee41a00700 1 --2- 172.21.15.28:0/3954670848 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 0x7fee3c105cc0 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7fee2c009990 tx=0x7fee2c009ca0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.601 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.599+0000 7fee41a00700 1 -- 172.21.15.28:0/3954670848 shutdown_connections 2024-05-20T20:17:48.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.599+0000 7fee41a00700 1 --2- 172.21.15.28:0/3954670848 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee3c1078b0 0x7fee3c112540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.599+0000 7fee41a00700 1 --2- 172.21.15.28:0/3954670848 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 0x7fee3c105cc0 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.599+0000 7fee41a00700 1 --2- 172.21.15.28:0/3954670848 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee3c106200 0x7fee3c107180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.599+0000 7fee41a00700 1 -- 172.21.15.28:0/3954670848 >> 172.21.15.28:0/3954670848 conn(0x7fee3c101050 msgr2=0x7fee3c103470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:48.602 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.601+0000 7fee41a00700 1 -- 172.21.15.28:0/3954670848 shutdown_connections 2024-05-20T20:17:48.603 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.601+0000 7fee41a00700 1 -- 172.21.15.28:0/3954670848 wait complete. 2024-05-20T20:17:48.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.603+0000 7fee41a00700 1 Processor -- start 2024-05-20T20:17:48.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.604+0000 7fee41a00700 1 -- start start 2024-05-20T20:17:48.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.605+0000 7fee41a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 0x7fee3c1123d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.606 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.605+0000 7fee41a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee3c106200 0x7fee3c10f520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.607 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.606+0000 7fee41a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee3c1078b0 0x7fee3c10fa60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.607+0000 7fee41000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee3c1078b0 0x7fee3c10fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.607+0000 7fee41000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee3c1078b0 0x7fee3c10fa60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60362/0 (socket says 172.21.15.28:60362) 2024-05-20T20:17:48.608 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.607+0000 7fee41000700 1 -- 172.21.15.28:0/24759000 learned_addr learned my addr 172.21.15.28:0/24759000 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:48.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.608+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fee3c1159d0 con 0x7fee3c1078b0 2024-05-20T20:17:48.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.608+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fee3c115850 con 0x7fee3c106200 2024-05-20T20:17:48.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.607+0000 7fee3be00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 0x7fee3c1123d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.609 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.608+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fee3c115b50 con 0x7fee3c1058e0 2024-05-20T20:17:48.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.608+0000 7fee3b400700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee3c106200 0x7fee3c10f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.609+0000 7fee3be00700 1 -- 172.21.15.28:0/24759000 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee3c106200 msgr2=0x7fee3c10f520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.609+0000 7fee3be00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee3c106200 0x7fee3c10f520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.609+0000 7fee3be00700 1 -- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee3c1078b0 msgr2=0x7fee3c10fa60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.610 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.609+0000 7fee3be00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee3c1078b0 0x7fee3c10fa60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.609+0000 7fee3be00700 1 -- 172.21.15.28:0/24759000 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fee3c110330 con 0x7fee3c1058e0 2024-05-20T20:17:48.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.610+0000 7fee3b400700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee3c106200 0x7fee3c10f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:17:48.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.611+0000 7fee3be00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 0x7fee3c1123d0 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7fee2c0098e0 tx=0x7fee2c0057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:48.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.613+0000 7fee38c00700 1 -- 172.21.15.28:0/24759000 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee2c027070 con 0x7fee3c1058e0 2024-05-20T20:17:48.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.613+0000 7fee38c00700 1 -- 172.21.15.28:0/24759000 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fee2c004000 con 0x7fee3c1058e0 2024-05-20T20:17:48.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.613+0000 7fee38c00700 1 -- 172.21.15.28:0/24759000 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee2c022ce0 con 0x7fee3c1058e0 2024-05-20T20:17:48.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.613+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fee3c1a1fb0 con 0x7fee3c1058e0 2024-05-20T20:17:48.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.614+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fee3c1a2490 con 0x7fee3c1058e0 2024-05-20T20:17:48.628 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.626+0000 7fee38c00700 1 -- 172.21.15.28:0/24759000 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fee2c02a070 con 0x7fee3c1058e0 2024-05-20T20:17:48.639 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.635+0000 7fee4d9ffb80 1 Processor -- start 2024-05-20T20:17:48.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.635+0000 7fee4d9ffb80 1 -- start start 2024-05-20T20:17:48.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.636+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229494a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.636+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 0x56522948eb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.636+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 0x56522948f5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.640 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.636+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5652293ecfd0 con 0x565229494640 2024-05-20T20:17:48.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.636+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5652293dcb70 con 0x565229494f60 2024-05-20T20:17:48.641 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.636+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5652293e0bb0 con 0x56522948f170 2024-05-20T20:17:48.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.639+0000 7fee41000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229494a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.642 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.639+0000 7fee41000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229494a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60366/0 (socket says 172.21.15.28:60366) 2024-05-20T20:17:48.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.639+0000 7fee41000700 1 -- 172.21.15.28:0/1730800689 learned_addr learned my addr 172.21.15.28:0/1730800689 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:48.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.640+0000 7fee38c00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fee200b2b20 0x7fee200b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.640+0000 7fee38c00700 1 -- 172.21.15.28:0/24759000 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fee2c003bb0 con 0x7fee3c1058e0 2024-05-20T20:17:48.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.640+0000 7fee3b400700 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 0x56522948f5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.641+0000 7fee41000700 1 -- 172.21.15.28:0/1730800689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 msgr2=0x56522948f5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:48.643 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.641+0000 7fee41000700 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 0x56522948f5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.641+0000 7fee41000700 1 -- 172.21.15.28:0/1730800689 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 msgr2=0x56522948eb50 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:17:48.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.641+0000 7fee41000700 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 0x56522948eb50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.641+0000 7fee41000700 1 -- 172.21.15.28:0/1730800689 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5652293a55b0 con 0x565229494640 2024-05-20T20:17:48.644 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.642+0000 7fee41000700 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229494a20 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fee3400e270 tx=0x7fee34009710 comp rx=0 tx=0).ready entity=mon.0 client_cookie=15e2a061cb94da6e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:48.647 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.643+0000 7fee41000700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fee200b2b20 0x7fee200b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.650+0000 7fee31600700 1 -- 172.21.15.28:0/1730800689 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee34018020 con 0x565229494640 2024-05-20T20:17:48.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.650+0000 7fee31600700 1 -- 172.21.15.28:0/1730800689 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fee34014bb0 con 0x565229494640 2024-05-20T20:17:48.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.650+0000 7fee31600700 1 -- 172.21.15.28:0/1730800689 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee3401ee20 con 0x565229494640 2024-05-20T20:17:48.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.651+0000 7fee41000700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fee200b2b20 0x7fee200b4fe0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fee34019000 tx=0x7fee34013950 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:48.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.656+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1730800689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 msgr2=0x565229494a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.656+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229494a20 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7fee3400e270 tx=0x7fee34009710 comp rx=0 tx=0).stop 2024-05-20T20:17:48.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.659+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1730800689 shutdown_connections 2024-05-20T20:17:48.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.659+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 0x56522948f5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.659+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229494a20 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.659+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1730800689 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 0x56522948eb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.660 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.659+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1730800689 >> 172.21.15.28:0/1730800689 conn(0x5652293bcd70 msgr2=0x5652294896c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:48.664 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.662+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1730800689 shutdown_connections 2024-05-20T20:17:48.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.669+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1730800689 wait complete. 2024-05-20T20:17:48.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.670+0000 7fee4d9ffb80 1 -- start start 2024-05-20T20:17:48.673 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.671+0000 7fee4d9ffb80 1 Processor -- start 2024-05-20T20:17:48.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.674+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 0x56522949acd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.674+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229497ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.674+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 0x565229498530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.674+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5652292e3130 con 0x565229494640 2024-05-20T20:17:48.678 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.674+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56522937d8f0 con 0x565229494f60 2024-05-20T20:17:48.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.674+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56522949ba10 con 0x56522948f170 2024-05-20T20:17:48.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.674+0000 7fee4d9ffb80 10 client.0 _ll_register_callbacks cb 0x565229386e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:48.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.675+0000 7fee3b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229497ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.675+0000 7fee3b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229497ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60368/0 (socket says 172.21.15.28:60368) 2024-05-20T20:17:48.679 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.675+0000 7fee3b400700 1 -- 172.21.15.28:0/890214 learned_addr learned my addr 172.21.15.28:0/890214 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:48.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.676+0000 7fee41000700 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 0x565229498530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.677+0000 7fee3b400700 1 -- 172.21.15.28:0/890214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 msgr2=0x56522949acd0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:17:48.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.677+0000 7fee3b400700 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 0x56522949acd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.677+0000 7fee3b400700 1 -- 172.21.15.28:0/890214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 msgr2=0x565229498530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:48.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.677+0000 7fee3b400700 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 0x565229498530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.678+0000 7fee3b400700 1 -- 172.21.15.28:0/890214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56522932ca60 con 0x565229494640 2024-05-20T20:17:48.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.680+0000 7fee3b400700 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229497ff0 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7fee2800e1f0 tx=0x7fee2800e500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7236d88594a8a47d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:48.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.684+0000 7fee26000700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:17:48.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.685+0000 7fee26000700 1 -- 172.21.15.28:0/890214 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee28017020 con 0x565229494640 2024-05-20T20:17:48.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.685+0000 7fee26000700 1 -- 172.21.15.28:0/890214 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fee2800ee30 con 0x565229494640 2024-05-20T20:17:48.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.686+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fee34014bb0 con 0x565229494640 2024-05-20T20:17:48.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.686+0000 7fee26000700 1 -- 172.21.15.28:0/890214 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fee2800f040 con 0x565229494640 2024-05-20T20:17:48.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.686+0000 7fee4d9ffb80 10 client.17280 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:48.689 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.686+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5652293ad1f0 con 0x565229494640 2024-05-20T20:17:48.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.687+0000 7fee26000700 1 -- 172.21.15.28:0/890214 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fee34014bb0 con 0x565229494640 2024-05-20T20:17:48.690 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.687+0000 7fee26000700 1 -- 172.21.15.28:0/890214 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fee28012050 con 0x565229494640 2024-05-20T20:17:48.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.689+0000 7fee4d9ffb80 10 client.17280 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:48.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.689+0000 7fee4d9ffb80 10 client.17280 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:17:48.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.689+0000 7fee4d9ffb80 20 client.17280 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:48.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.689+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5652294a85f0 0x5652294aa9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.689+0000 7fee4d9ffb80 4 client.17280 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:48.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.689+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5652293eef20 con 0x5652294a85f0 2024-05-20T20:17:48.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.689+0000 7fee3be00700 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5652294a85f0 0x5652294aa9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.690+0000 7fee3be00700 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5652294a85f0 0x5652294aa9e0 crc :-1 s=READY pgs=249 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=599784c5eebf15bb server_cookie=cf81a516d2f7e2fd in_seq=0 out_seq=0 2024-05-20T20:17:48.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.690+0000 7fee26000700 10 client.17280 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:48.693 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.691+0000 7fee26000700 1 -- 172.21.15.28:0/890214 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5652293eef20 con 0x5652294a85f0 2024-05-20T20:17:48.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.691+0000 7fee26000700 10 client.17280 handle_command_reply: tid=0 2024-05-20T20:17:48.696 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.694+0000 7fee4d9ffb80 1 client.17280 shutdown 2024-05-20T20:17:48.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.694+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 msgr2=0x565229497ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.694+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229497ff0 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7fee2800e1f0 tx=0x7fee2800e500 comp rx=0 tx=0).stop 2024-05-20T20:17:48.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.695+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 shutdown_connections 2024-05-20T20:17:48.698 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.695+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5652294a85f0 0x5652294aa9e0 crc :-1 s=READY pgs=249 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.695+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x565229494f60 0x565229498530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.695+0000 7fee26000700 0 client.17280 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:48.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.695+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56522948f170 0x56522949acd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.695+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/890214 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x565229497ff0 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.699 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.695+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 >> 172.21.15.28:0/890214 conn(0x5652293bcd70 msgr2=0x56522948db40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:48.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.696+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 shutdown_connections 2024-05-20T20:17:48.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.696+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/890214 wait complete. 2024-05-20T20:17:48.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.696+0000 7fee4d9ffb80 20 client.17280 trim_cache size 0 max 16384 2024-05-20T20:17:48.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 Processor -- start 2024-05-20T20:17:48.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 -- start start 2024-05-20T20:17:48.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x56522948eb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494640 0x565229495ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494f60 0x5652294b02f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fee1c00d750 con 0x565229494f60 2024-05-20T20:17:48.748 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fee1c00f290 con 0x56522948f170 2024-05-20T20:17:48.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.746+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fee1c00de20 con 0x565229494640 2024-05-20T20:17:48.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x56522948eb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x56522948eb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58992/0 (socket says 172.21.15.28:58992) 2024-05-20T20:17:48.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 -- 172.21.15.28:0/1938448878 learned_addr learned my addr 172.21.15.28:0/1938448878 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:48.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3b400700 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494640 0x565229495ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 -- 172.21.15.28:0/1938448878 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494640 msgr2=0x565229495ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:48.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494640 0x565229495ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 -- 172.21.15.28:0/1938448878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494f60 msgr2=0x5652294b02f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:48.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494f60 0x5652294b02f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 -- 172.21.15.28:0/1938448878 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56522949a890 con 0x56522948f170 2024-05-20T20:17:48.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.747+0000 7fee3be00700 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x56522948eb50 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7fee2c009990 tx=0x7fee2c0cd040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ebbae3b1c68d69be server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:48.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.748+0000 7fee30c00700 1 -- 172.21.15.28:0/1938448878 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee2c0aa7d0 con 0x56522948f170 2024-05-20T20:17:48.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.748+0000 7fee30c00700 1 -- 172.21.15.28:0/1938448878 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fee2c0c3020 con 0x56522948f170 2024-05-20T20:17:48.753 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.748+0000 7fee30c00700 1 -- 172.21.15.28:0/1938448878 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee2c0af070 con 0x56522948f170 2024-05-20T20:17:48.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.749+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1938448878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 msgr2=0x56522948eb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.749+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x56522948eb50 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7fee2c009990 tx=0x7fee2c0cd040 comp rx=0 tx=0).stop 2024-05-20T20:17:48.754 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.752+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1938448878 shutdown_connections 2024-05-20T20:17:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.752+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494f60 0x5652294b02f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.752+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x56522948eb50 unknown :-1 s=CLOSED pgs=580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.752+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1938448878 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494640 0x565229495ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.752+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1938448878 >> 172.21.15.28:0/1938448878 conn(0x5652293bcd70 msgr2=0x5652294953a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:48.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.752+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1938448878 shutdown_connections 2024-05-20T20:17:48.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.752+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1938448878 wait complete. 2024-05-20T20:17:48.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.753+0000 7fee4d9ffb80 1 -- start start 2024-05-20T20:17:48.756 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.753+0000 7fee4d9ffb80 1 Processor -- start 2024-05-20T20:17:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.754+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x5652294a15a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.754+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x56522949dd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.754+0000 7fee4d9ffb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494f60 0x56522949e440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.754+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fee1c00ebc0 con 0x565229494640 2024-05-20T20:17:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.754+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fee1c00e4f0 con 0x56522948f170 2024-05-20T20:17:48.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.754+0000 7fee4d9ffb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fee1c00b710 con 0x565229494f60 2024-05-20T20:17:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.754+0000 7fee4d9ffb80 10 client.0 _ll_register_callbacks cb 0x565229386e00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494f60 0x56522949e440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494f60 0x56522949e440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41406/0 (socket says 172.21.15.28:41406) 2024-05-20T20:17:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 -- 172.21.15.28:0/1722011124 learned_addr learned my addr 172.21.15.28:0/1722011124 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:48.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee3be00700 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x5652294a15a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 -- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 msgr2=0x5652294a15a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x5652294a15a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 -- 172.21.15.28:0/1722011124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 msgr2=0x56522949dd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x56522949dd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.755+0000 7fee41000700 1 -- 172.21.15.28:0/1722011124 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5652295131d0 con 0x565229494f60 2024-05-20T20:17:48.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.756+0000 7fee26000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:48.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.756+0000 7fee41000700 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494f60 0x56522949e440 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7fee34009450 tx=0x7fee34013000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=391f675994f76a43 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:48.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.757+0000 7fee26000700 1 -- 172.21.15.28:0/1722011124 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fee34038060 con 0x565229494f60 2024-05-20T20:17:48.761 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.757+0000 7fee26000700 1 -- 172.21.15.28:0/1722011124 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fee34030070 con 0x565229494f60 2024-05-20T20:17:48.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.757+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fee2c0c3020 con 0x565229494f60 2024-05-20T20:17:48.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.758+0000 7fee26000700 1 -- 172.21.15.28:0/1722011124 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fee34038060 con 0x565229494f60 2024-05-20T20:17:48.762 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.758+0000 7fee4d9ffb80 10 client.27778 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:48.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.758+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5652295143a0 con 0x565229494f60 2024-05-20T20:17:48.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.758+0000 7fee26000700 1 -- 172.21.15.28:0/1722011124 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fee2c0c3020 con 0x565229494f60 2024-05-20T20:17:48.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.758+0000 7fee26000700 1 -- 172.21.15.28:0/1722011124 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fee34014510 con 0x565229494f60 2024-05-20T20:17:48.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.759+0000 7fee4d9ffb80 10 client.27778 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:48.763 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.759+0000 7fee4d9ffb80 10 client.27778 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:17:48.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.759+0000 7fee4d9ffb80 20 client.27778 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:48.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.759+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x565229514830 0x565229516c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:48.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.760+0000 7fee4d9ffb80 4 client.27778 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 137.9, "await_for": 45.0}] 2024-05-20T20:17:48.764 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.760+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 137.9, "await_for": 45.0}) -- 0x5652293e8700 con 0x565229514830 2024-05-20T20:17:48.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.761+0000 7fee3be00700 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x565229514830 0x565229516c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:48.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.762+0000 7fee3be00700 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x565229514830 0x565229516c20 crc :-1 s=READY pgs=250 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4229be51fc76442b server_cookie=63a2c41f176a3c68 in_seq=0 out_seq=0 2024-05-20T20:17:48.765 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.762+0000 7fee26000700 10 client.27778 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:48.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.871+0000 7fee26000700 1 -- 172.21.15.28:0/1722011124 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x5652293e8700 con 0x565229514830 2024-05-20T20:17:48.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.871+0000 7fee26000700 10 client.27778 handle_command_reply: tid=0 2024-05-20T20:17:48.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.871+0000 7fee4d9ffb80 1 client.27778 shutdown 2024-05-20T20:17:48.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494f60 msgr2=0x56522949e440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494f60 0x56522949e440 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7fee34009450 tx=0x7fee34013000 comp rx=0 tx=0).stop 2024-05-20T20:17:48.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 shutdown_connections 2024-05-20T20:17:48.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x565229514830 0x565229516c20 crc :-1 s=READY pgs=250 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x565229494f60 0x56522949e440 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56522948f170 0x5652294a15a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 --2- 172.21.15.28:0/1722011124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x565229494640 0x56522949dd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 >> 172.21.15.28:0/1722011124 conn(0x5652293bcd70 msgr2=0x565229495340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:48.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.872+0000 7fee26000700 0 client.27778 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:48.876 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:17:48.876 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:17:48.876 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:17:48.876 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 30, 2024-05-20T20:17:48.877 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:17:48.877 INFO:teuthology.orchestra.run.smithi028.stdout: "6c5436e5": { 2024-05-20T20:17:48.877 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 30, 2024-05-20T20:17:48.877 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:17:48.877 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:17:48.877 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:17:48.877 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:17:48.878 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:17:48.878 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:17:48.878 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 137.9, 2024-05-20T20:17:48.878 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:17:48.878 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:17:48.878 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:17:48.878 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:17:48.879 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:17:48.879 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:17:48.881 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:17:48.881 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:17:48.881 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:17:48.881 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:17:48.881 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:17:48.881 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:17:48.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.873+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 shutdown_connections 2024-05-20T20:17:48.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.873+0000 7fee4d9ffb80 1 -- 172.21.15.28:0/1722011124 wait complete. 2024-05-20T20:17:48.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.873+0000 7fee4d9ffb80 20 client.27778 trim_cache size 0 max 16384 2024-05-20T20:17:48.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.874+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fee200b2b20 msgr2=0x7fee200b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.874+0000 7fee41a00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fee200b2b20 0x7fee200b4fe0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fee34019000 tx=0x7fee34013950 comp rx=0 tx=0).stop 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.874+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 msgr2=0x7fee3c1123d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.874+0000 7fee41a00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 0x7fee3c1123d0 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7fee2c0098e0 tx=0x7fee2c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.876+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 shutdown_connections 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.876+0000 7fee41a00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fee3c1078b0 0x7fee3c10fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.876+0000 7fee41a00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fee3c1058e0 0x7fee3c1123d0 secure :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7fee2c0098e0 tx=0x7fee2c0057c0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.876+0000 7fee41a00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fee200b2b20 0x7fee200b4fe0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.876+0000 7fee41a00700 1 --2- 172.21.15.28:0/24759000 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fee3c106200 0x7fee3c10f520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.876+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 >> 172.21.15.28:0/24759000 conn(0x7fee3c101050 msgr2=0x7fee3c102a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.878+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 shutdown_connections 2024-05-20T20:17:48.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:48.879+0000 7fee41a00700 1 -- 172.21.15.28:0/24759000 wait complete. 2024-05-20T20:17:49.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:49 smithi154 ceph-mon[36314]: pgmap v380: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 251 MiB/s wr, 1.04k op/s 2024-05-20T20:17:49.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:49 smithi033 ceph-mon[36179]: pgmap v380: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 251 MiB/s wr, 1.04k op/s 2024-05-20T20:17:49.641 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:49 smithi028 ceph-mon[31178]: pgmap v380: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 251 MiB/s wr, 1.04k op/s 2024-05-20T20:17:51.181 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set '6c5436e5', quiesce took 21.8 seconds. Will release after: -3.900000000000002 2024-05-20T20:17:51.181 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:17:51.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:51 smithi028 ceph-mon[31178]: pgmap v381: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 340 MiB/s wr, 1.45k op/s 2024-05-20T20:17:51.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:51 smithi154 ceph-mon[36314]: pgmap v381: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 340 MiB/s wr, 1.45k op/s 2024-05-20T20:17:51.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:51 smithi033 ceph-mon[36179]: pgmap v381: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 340 MiB/s wr, 1.45k op/s 2024-05-20T20:17:52.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.047+0000 7fdda2a00700 1 -- 172.21.15.28:0/43150548 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c115490 msgr2=0x7fdd9c117880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.047+0000 7fdda2a00700 1 --2- 172.21.15.28:0/43150548 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c115490 0x7fdd9c117880 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7fdd9400b0a0 tx=0x7fdd9400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.048+0000 7fdda2a00700 1 -- 172.21.15.28:0/43150548 shutdown_connections 2024-05-20T20:17:52.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.048+0000 7fdda2a00700 1 --2- 172.21.15.28:0/43150548 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c115490 0x7fdd9c117880 unknown :-1 s=CLOSED pgs=581 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.048+0000 7fdda2a00700 1 --2- 172.21.15.28:0/43150548 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd9c10bee0 0x7fdd9c10c2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.048+0000 7fdda2a00700 1 --2- 172.21.15.28:0/43150548 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd9c10c800 0x7fdd9c114f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.048+0000 7fdda2a00700 1 -- 172.21.15.28:0/43150548 >> 172.21.15.28:0/43150548 conn(0x7fdd9c06e6b0 msgr2=0x7fdd9c06eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:52.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.051+0000 7fdda2a00700 1 -- 172.21.15.28:0/43150548 shutdown_connections 2024-05-20T20:17:52.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.052+0000 7fdda2a00700 1 -- 172.21.15.28:0/43150548 wait complete. 2024-05-20T20:17:52.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.064+0000 7fdda2a00700 1 Processor -- start 2024-05-20T20:17:52.065 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.064+0000 7fdda2a00700 1 -- start start 2024-05-20T20:17:52.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.065+0000 7fdda2a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c10bee0 0x7fdd9c115940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.065+0000 7fdda2a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd9c10c800 0x7fdd9c115e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.065+0000 7fdda2a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd9c115490 0x7fdd9c1163c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.066+0000 7fdda2a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c11af40 con 0x7fdd9c10c800 2024-05-20T20:17:52.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.066+0000 7fdda2a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c11adc0 con 0x7fdd9c10bee0 2024-05-20T20:17:52.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.066+0000 7fdda2a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c11b0c0 con 0x7fdd9c115490 2024-05-20T20:17:52.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.066+0000 7fdd9be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c10bee0 0x7fdd9c115940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.067 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.066+0000 7fdd9be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c10bee0 0x7fdd9c115940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:59028/0 (socket says 172.21.15.28:59028) 2024-05-20T20:17:52.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.066+0000 7fdd9be00700 1 -- 172.21.15.28:0/4179874281 learned_addr learned my addr 172.21.15.28:0/4179874281 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:52.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.067+0000 7fdd9be00700 1 -- 172.21.15.28:0/4179874281 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd9c115490 msgr2=0x7fdd9c1163c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:17:52.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.067+0000 7fdd9be00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd9c115490 0x7fdd9c1163c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.067+0000 7fdd9be00700 1 -- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd9c10c800 msgr2=0x7fdd9c115e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.067+0000 7fdd9be00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd9c10c800 0x7fdd9c115e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.067+0000 7fdd9be00700 1 -- 172.21.15.28:0/4179874281 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdd9c1b4e30 con 0x7fdd9c10bee0 2024-05-20T20:17:52.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.068+0000 7fdd9be00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c10bee0 0x7fdd9c115940 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7fdd9000c9b0 tx=0x7fdd9000cd70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:52.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.068+0000 7fdd98c00700 1 -- 172.21.15.28:0/4179874281 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd90007a80 con 0x7fdd9c10bee0 2024-05-20T20:17:52.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.068+0000 7fdd98c00700 1 -- 172.21.15.28:0/4179874281 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd90007c20 con 0x7fdd9c10bee0 2024-05-20T20:17:52.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.068+0000 7fdd98c00700 1 -- 172.21.15.28:0/4179874281 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd90018730 con 0x7fdd9c10bee0 2024-05-20T20:17:52.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.068+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdd9c1b50c0 con 0x7fdd9c10bee0 2024-05-20T20:17:52.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.069+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdd9c1b55b0 con 0x7fdd9c10bee0 2024-05-20T20:17:52.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.074+0000 7fdd98c00700 1 -- 172.21.15.28:0/4179874281 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fdd9000f990 con 0x7fdd9c10bee0 2024-05-20T20:17:52.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.076+0000 7fddad2c3b80 1 Processor -- start 2024-05-20T20:17:52.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.076+0000 7fddad2c3b80 1 -- start start 2024-05-20T20:17:52.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fdd98c00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd840b2b20 0x7fdd840b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a38242eb0 0x556a38243290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fdd98c00700 1 -- 172.21.15.28:0/4179874281 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fdd900e2690 con 0x7fdd9c10bee0 2024-05-20T20:17:52.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a382437d0 0x556a3823d260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a3823dce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556a3819be40 con 0x556a382437d0 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556a3818b9e0 con 0x556a3823d880 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556a3818fa20 con 0x556a38242eb0 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fdda0a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd840b2b20 0x7fdd840b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.077+0000 7fdda0a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a3823dce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdda0a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a3823dce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:59036/0 (socket says 172.21.15.28:59036) 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdda0a00700 1 -- 172.21.15.28:0/1392739882 learned_addr learned my addr 172.21.15.28:0/1392739882 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdd9be00700 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a38242eb0 0x556a38243290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdd9b400700 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a382437d0 0x556a3823d260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdda0a00700 1 -- 172.21.15.28:0/1392739882 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a38242eb0 msgr2=0x556a38243290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdda0a00700 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a38242eb0 0x556a38243290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdda0a00700 1 -- 172.21.15.28:0/1392739882 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a382437d0 msgr2=0x556a3823d260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdda0a00700 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a382437d0 0x556a3823d260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.078+0000 7fdda0a00700 1 -- 172.21.15.28:0/1392739882 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556a38154420 con 0x556a3823d880 2024-05-20T20:17:52.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.079+0000 7fdd9b400700 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a382437d0 0x556a3823d260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:52.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.079+0000 7fdda0a00700 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a3823dce0 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7fdd94009530 tx=0x7fdd94009560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3f18a4dbd97e899 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:52.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.080+0000 7fdda0a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd840b2b20 0x7fdd840b4fe0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fdd94028000 tx=0x7fdd94008b30 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:52.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.080+0000 7fdd8d600700 1 -- 172.21.15.28:0/1392739882 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd9401f920 con 0x556a3823d880 2024-05-20T20:17:52.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.081+0000 7fdd8d600700 1 -- 172.21.15.28:0/1392739882 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd9401fc80 con 0x556a3823d880 2024-05-20T20:17:52.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.081+0000 7fdd8d600700 1 -- 172.21.15.28:0/1392739882 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd940087a0 con 0x556a3823d880 2024-05-20T20:17:52.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1392739882 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 msgr2=0x556a3823dce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a3823dce0 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7fdd94009530 tx=0x7fdd94009560 comp rx=0 tx=0).stop 2024-05-20T20:17:52.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1392739882 shutdown_connections 2024-05-20T20:17:52.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a3823dce0 unknown :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a38242eb0 0x556a38243290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1392739882 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a382437d0 0x556a3823d260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1392739882 >> 172.21.15.28:0/1392739882 conn(0x556a3816bbe0 msgr2=0x556a38238010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:52.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.083+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1392739882 shutdown_connections 2024-05-20T20:17:52.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.084+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1392739882 wait complete. 2024-05-20T20:17:52.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.085+0000 7fddad2c3b80 1 -- start start 2024-05-20T20:17:52.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.087+0000 7fddad2c3b80 1 Processor -- start 2024-05-20T20:17:52.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.088+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a38249560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.088+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a38242eb0 0x556a38246950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.088+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a38246e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.088+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x556a38091fa0 con 0x556a38242eb0 2024-05-20T20:17:52.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.088+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x556a3812c760 con 0x556a3823d880 2024-05-20T20:17:52.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.088+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x556a3824a240 con 0x556a382437d0 2024-05-20T20:17:52.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.088+0000 7fddad2c3b80 10 client.0 _ll_register_callbacks cb 0x556a38135c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.089+0000 7fdda0a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a38246e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a38242eb0 0x556a38246950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a38242eb0 0x556a38246950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60406/0 (socket says 172.21.15.28:60406) 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 -- 172.21.15.28:0/2159652574 learned_addr learned my addr 172.21.15.28:0/2159652574 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 -- 172.21.15.28:0/2159652574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 msgr2=0x556a38246e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a38246e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 -- 172.21.15.28:0/2159652574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 msgr2=0x556a38249560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a38249560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.090+0000 7fdd9b400700 1 -- 172.21.15.28:0/2159652574 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556a380db8d0 con 0x556a38242eb0 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.091+0000 7fdd9b400700 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a38242eb0 0x556a38246950 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7fdd8800c200 tx=0x7fdd8800c5c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a94f85b222697650 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:52.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.091+0000 7fdd82000700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.091+0000 7fdd82000700 1 -- 172.21.15.28:0/2159652574 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd88015070 con 0x556a38242eb0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.091+0000 7fdd82000700 1 -- 172.21.15.28:0/2159652574 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fdd880115f0 con 0x556a38242eb0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.092+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdd9401fc80 con 0x556a38242eb0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.093+0000 7fdd82000700 1 -- 172.21.15.28:0/2159652574 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fdd8801f050 con 0x556a38242eb0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.093+0000 7fddad2c3b80 10 client.17283 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.093+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556a3815c060 con 0x556a38242eb0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.094+0000 7fdd82000700 1 -- 172.21.15.28:0/2159652574 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd9401fc80 con 0x556a38242eb0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.094+0000 7fdd82000700 1 -- 172.21.15.28:0/2159652574 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fdd8801aa20 con 0x556a38242eb0 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.096+0000 7fddad2c3b80 10 client.17283 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.096+0000 7fddad2c3b80 10 client.17283 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.096+0000 7fddad2c3b80 20 client.17283 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:52.100 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.096+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a38256e00 0x556a38259210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.096+0000 7fddad2c3b80 4 client.17283 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:52.101 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.096+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x556a3819dd90 con 0x556a38256e00 2024-05-20T20:17:52.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.097+0000 7fdd9be00700 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a38256e00 0x556a38259210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.102 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.098+0000 7fdd82000700 10 client.17283 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.098+0000 7fdd9be00700 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a38256e00 0x556a38259210 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=6786af0d5883f331 server_cookie=8042bb22390eccef in_seq=0 out_seq=0 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.099+0000 7fdd82000700 1 -- 172.21.15.28:0/2159652574 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x556a3819dd90 con 0x556a38256e00 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.099+0000 7fdd82000700 10 client.17283 handle_command_reply: tid=0 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.099+0000 7fddad2c3b80 1 client.17283 shutdown 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a38242eb0 msgr2=0x556a38246950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a38242eb0 0x556a38246950 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7fdd8800c200 tx=0x7fdd8800c5c0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 shutdown_connections 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a38256e00 0x556a38259210 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a38246e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a3823d880 0x556a38249560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2159652574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a38242eb0 0x556a38246950 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 >> 172.21.15.28:0/2159652574 conn(0x556a3816bbe0 msgr2=0x556a3823c250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:52.103 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.100+0000 7fdd82000700 0 client.17283 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:17:52.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.102+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 shutdown_connections 2024-05-20T20:17:52.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.102+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2159652574 wait complete. 2024-05-20T20:17:52.104 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.102+0000 7fddad2c3b80 20 client.17283 trim_cache size 0 max 16384 2024-05-20T20:17:52.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.131+0000 7fddad2c3b80 1 Processor -- start 2024-05-20T20:17:52.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.131+0000 7fddad2c3b80 1 -- start start 2024-05-20T20:17:52.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.132+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3823d880 0x556a38253330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.132+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 0x556a3823d260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.132+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a382409c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.132+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c1bcee0 con 0x556a3823d880 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.132+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c1bea20 con 0x556a38242eb0 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.132+0000 7fddad2c3b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c1bd5b0 con 0x556a382437d0 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.133+0000 7fdd9be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3823d880 0x556a38253330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.133+0000 7fdd9be00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3823d880 0x556a38253330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:60410/0 (socket says 172.21.15.28:60410) 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.134+0000 7fdd9be00700 1 -- 172.21.15.28:0/1444232164 learned_addr learned my addr 172.21.15.28:0/1444232164 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.134+0000 7fdd9be00700 1 -- 172.21.15.28:0/1444232164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 msgr2=0x556a382409c0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.134+0000 7fdd9be00700 1 --2- 172.21.15.28:0/1444232164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a382409c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.135+0000 7fdd9be00700 1 -- 172.21.15.28:0/1444232164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 msgr2=0x556a3823d260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:52.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.135+0000 7fdd9be00700 1 --2- 172.21.15.28:0/1444232164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 0x556a3823d260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.135+0000 7fdd9be00700 1 -- 172.21.15.28:0/1444232164 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x556a38253950 con 0x556a3823d880 2024-05-20T20:17:52.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.135+0000 7fdd9be00700 1 --2- 172.21.15.28:0/1444232164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3823d880 0x556a38253330 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7fdd900074a0 tx=0x7fdd90005ed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f553f8a36285578b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:52.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.136+0000 7fdd8cc00700 1 -- 172.21.15.28:0/1444232164 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd900b2760 con 0x556a3823d880 2024-05-20T20:17:52.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.136+0000 7fdd8cc00700 1 -- 172.21.15.28:0/1444232164 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd90007dd0 con 0x556a3823d880 2024-05-20T20:17:52.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.136+0000 7fdd8cc00700 1 -- 172.21.15.28:0/1444232164 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd900a9dc0 con 0x556a3823d880 2024-05-20T20:17:52.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1444232164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3823d880 msgr2=0x556a38253330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1444232164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3823d880 0x556a38253330 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7fdd900074a0 tx=0x7fdd90005ed0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1444232164 shutdown_connections 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1444232164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a382409c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1444232164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3823d880 0x556a38253330 secure :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7fdd900074a0 tx=0x7fdd90005ed0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/1444232164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 0x556a3823d260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1444232164 >> 172.21.15.28:0/1444232164 conn(0x556a3816bbe0 msgr2=0x556a38245240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1444232164 shutdown_connections 2024-05-20T20:17:52.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.137+0000 7fddad2c3b80 1 -- 172.21.15.28:0/1444232164 wait complete. 2024-05-20T20:17:52.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.138+0000 7fddad2c3b80 1 -- start start 2024-05-20T20:17:52.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.138+0000 7fddad2c3b80 1 Processor -- start 2024-05-20T20:17:52.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.140+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 0x556a382bd130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.140+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a3824f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.140+0000 7fddad2c3b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3824fa60 0x556a382bbc50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.140+0000 7fdd9be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 0x556a382bd130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.141+0000 7fdd9b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a3824f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.141+0000 7fdd9b400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a3824f520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:41458/0 (socket says 172.21.15.28:41458) 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.141+0000 7fdd9b400700 1 -- 172.21.15.28:0/2297846462 learned_addr learned my addr 172.21.15.28:0/2297846462 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.141+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c1be350 con 0x556a3824fa60 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.141+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c1bdc80 con 0x556a38242eb0 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.141+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fdd9c1baea0 con 0x556a382437d0 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.141+0000 7fddad2c3b80 10 client.0 _ll_register_callbacks cb 0x556a38135c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.142+0000 7fdd9b400700 1 -- 172.21.15.28:0/2297846462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 msgr2=0x556a382bd130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.142+0000 7fdd9b400700 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 0x556a382bd130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.142+0000 7fdd9b400700 1 -- 172.21.15.28:0/2297846462 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3824fa60 msgr2=0x556a382bbc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.142+0000 7fdd9b400700 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3824fa60 0x556a382bbc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.143+0000 7fdd9b400700 1 -- 172.21.15.28:0/2297846462 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x556a3824ff60 con 0x556a382437d0 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.143+0000 7fdd9b400700 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a3824f520 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7fdd8800a120 tx=0x7fdd88019ad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ad53e8890a421195 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.143+0000 7fdd82000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:52.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.144+0000 7fdd82000700 1 -- 172.21.15.28:0/2297846462 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdd8802a070 con 0x556a382437d0 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.144+0000 7fdd82000700 1 -- 172.21.15.28:0/2297846462 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7fdd88004220 con 0x556a382437d0 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.144+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fdd90007dd0 con 0x556a382437d0 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.145+0000 7fdd82000700 1 -- 172.21.15.28:0/2297846462 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fdd88033050 con 0x556a382437d0 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.145+0000 7fddad2c3b80 10 client.27784 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.145+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x556a382d2e50 con 0x556a382437d0 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fdd82000700 1 -- 172.21.15.28:0/2297846462 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fdd90007dd0 con 0x556a382437d0 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fdd82000700 1 -- 172.21.15.28:0/2297846462 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fdd88003cc0 con 0x556a382437d0 2024-05-20T20:17:52.148 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fddad2c3b80 10 client.27784 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:52.151 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fddad2c3b80 10 client.27784 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fddad2c3b80 20 client.27784 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a382d3550 0x556a382d5970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fddad2c3b80 4 client.27784 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.146+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x556a38197570 con 0x556a382d3550 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.147+0000 7fdd9be00700 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a382d3550 0x556a382d5970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.147+0000 7fdd9be00700 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a382d3550 0x556a382d5970 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=b107afaa1b1f87eb server_cookie=8ae0e7233b4c131d in_seq=0 out_seq=0 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.147+0000 7fdd82000700 10 client.27784 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.148+0000 7fdd82000700 1 -- 172.21.15.28:0/2297846462 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x556a38197570 con 0x556a382d3550 2024-05-20T20:17:52.152 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.148+0000 7fdd82000700 10 client.27784 handle_command_reply: tid=0 2024-05-20T20:17:52.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.148+0000 7fddad2c3b80 1 client.27784 shutdown 2024-05-20T20:17:52.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.148+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 msgr2=0x556a3824f520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.153 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.148+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a3824f520 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7fdd8800a120 tx=0x7fdd88019ad0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.152+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 shutdown_connections 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.152+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x556a382d3550 0x556a382d5970 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.152+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x556a3824fa60 0x556a382bbc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.152+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x556a38242eb0 0x556a382bd130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.152+0000 7fddad2c3b80 1 --2- 172.21.15.28:0/2297846462 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x556a382437d0 0x556a3824f520 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.152+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 >> 172.21.15.28:0/2297846462 conn(0x556a3816bbe0 msgr2=0x556a382409c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.152+0000 7fdd82000700 0 client.27784 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:17:52.156 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.155+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 shutdown_connections 2024-05-20T20:17:52.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.155+0000 7fddad2c3b80 1 -- 172.21.15.28:0/2297846462 wait complete. 2024-05-20T20:17:52.157 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.155+0000 7fddad2c3b80 20 client.27784 trim_cache size 0 max 16384 2024-05-20T20:17:52.157 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:17:52.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.156+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd840b2b20 msgr2=0x7fdd840b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.158 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.156+0000 7fdda2a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd840b2b20 0x7fdd840b4fe0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fdd94028000 tx=0x7fdd94008b30 comp rx=0 tx=0).stop 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.156+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c10bee0 msgr2=0x7fdd9c115940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.156+0000 7fdda2a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c10bee0 0x7fdd9c115940 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7fdd9000c9b0 tx=0x7fdd9000cd70 comp rx=0 tx=0).stop 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.157+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 shutdown_connections 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.157+0000 7fdda2a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdd9c115490 0x7fdd9c1163c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.157+0000 7fdda2a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fdd840b2b20 0x7fdd840b4fe0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.157+0000 7fdda2a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fdd9c10bee0 0x7fdd9c115940 unknown :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.157+0000 7fdda2a00700 1 --2- 172.21.15.28:0/4179874281 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fdd9c10c800 0x7fdd9c115e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.157+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 >> 172.21.15.28:0/4179874281 conn(0x7fdd9c06e6b0 msgr2=0x7fdd9c0732b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:52.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.159+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 shutdown_connections 2024-05-20T20:17:52.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:52.161+0000 7fdda2a00700 1 -- 172.21.15.28:0/4179874281 wait complete. 2024-05-20T20:17:52.832 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:53.537 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:53 smithi028 ceph-mon[31178]: pgmap v382: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 247 MiB/s wr, 1.06k op/s 2024-05-20T20:17:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:53 smithi154 ceph-mon[36314]: pgmap v382: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 247 MiB/s wr, 1.06k op/s 2024-05-20T20:17:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:53 smithi033 ceph-mon[36179]: pgmap v382: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 247 MiB/s wr, 1.06k op/s 2024-05-20T20:17:54.072 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:17:55.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:55 smithi154 ceph-mon[36314]: pgmap v383: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 350 MiB/s wr, 1.64k op/s 2024-05-20T20:17:55.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:55 smithi033 ceph-mon[36179]: pgmap v383: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 350 MiB/s wr, 1.64k op/s 2024-05-20T20:17:55.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:55 smithi028 ceph-mon[31178]: pgmap v383: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 350 MiB/s wr, 1.64k op/s 2024-05-20T20:17:55.970 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:17:57.556 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:57 smithi028 ceph-mon[31178]: pgmap v384: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 273 MiB/s wr, 1.37k op/s 2024-05-20T20:17:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:57 smithi154 ceph-mon[36314]: pgmap v384: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 273 MiB/s wr, 1.37k op/s 2024-05-20T20:17:57.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:57 smithi033 ceph-mon[36179]: pgmap v384: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 273 MiB/s wr, 1.37k op/s 2024-05-20T20:17:58.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 -- 172.21.15.28:0/3776330920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd810f130 msgr2=0x7effd8111520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 --2- 172.21.15.28:0/3776330920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd810f130 0x7effd8111520 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7effd0009ca0 tx=0x7effd002a970 comp rx=0 tx=0).stop 2024-05-20T20:17:58.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 -- 172.21.15.28:0/3776330920 shutdown_connections 2024-05-20T20:17:58.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 --2- 172.21.15.28:0/3776330920 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd810f130 0x7effd8111520 unknown :-1 s=CLOSED pgs=585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 --2- 172.21.15.28:0/3776330920 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 0x7effd810ebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 --2- 172.21.15.28:0/3776330920 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd8074e00 0x7effd80751e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 -- 172.21.15.28:0/3776330920 >> 172.21.15.28:0/3776330920 conn(0x7effd80fe3a0 msgr2=0x7effd81007c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.894 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 -- 172.21.15.28:0/3776330920 shutdown_connections 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.821+0000 7effdee00640 1 -- 172.21.15.28:0/3776330920 wait complete. 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.823+0000 7effdee00640 1 Processor -- start 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.823+0000 7effdee00640 1 -- start start 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.824+0000 7effdee00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd8074e00 0x7effd8109d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.824+0000 7effdee00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 0x7effd810a270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.824+0000 7effdee00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd810f130 0x7effd8104e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.824+0000 7effdee00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7effd8103a80 con 0x7effd810f130 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.824+0000 7effdee00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7effd8103900 con 0x7effd8074e00 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.824+0000 7effdee00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7effd8103c00 con 0x7effd8075720 2024-05-20T20:17:58.895 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.825+0000 7effdd000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 0x7effd810a270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.825+0000 7effde400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd810f130 0x7effd8104e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.825+0000 7effdd000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 0x7effd810a270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43010/0 (socket says 172.21.15.28:43010) 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.825+0000 7effdd000640 1 -- 172.21.15.28:0/3123305338 learned_addr learned my addr 172.21.15.28:0/3123305338 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.825+0000 7effdd000640 1 -- 172.21.15.28:0/3123305338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd8074e00 msgr2=0x7effd8109d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effdda00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd8074e00 0x7effd8109d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effdd000640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd8074e00 0x7effd8109d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effdd000640 1 -- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd810f130 msgr2=0x7effd8104e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effdd000640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd810f130 0x7effd8104e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effdd000640 1 -- 172.21.15.28:0/3123305338 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7effd8105750 con 0x7effd8075720 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effde400640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd810f130 0x7effd8104e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effdda00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd8074e00 0x7effd8109d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:58.896 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.826+0000 7effdd000640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 0x7effd810a270 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7effcc00d950 tx=0x7effcc00de10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.827+0000 7effd6000640 1 -- 172.21.15.28:0/3123305338 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effcc017030 con 0x7effd8075720 2024-05-20T20:17:58.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.827+0000 7effd6000640 1 -- 172.21.15.28:0/3123305338 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7effcc004020 con 0x7effd8075720 2024-05-20T20:17:58.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.827+0000 7effd6000640 1 -- 172.21.15.28:0/3123305338 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effcc0047a0 con 0x7effd8075720 2024-05-20T20:17:58.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.828+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7effd8105a40 con 0x7effd8075720 2024-05-20T20:17:58.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.828+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7effd810dc10 con 0x7effd8075720 2024-05-20T20:17:58.899 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.836+0000 7effe3281740 1 Processor -- start 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.837+0000 7effd6000640 1 -- 172.21.15.28:0/3123305338 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7effcc009a30 con 0x7effd8075720 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.840+0000 7effd6000640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7effc00b2b20 0x7effc00b4fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.840+0000 7effd6000640 1 -- 172.21.15.28:0/3123305338 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7effcc0e1620 con 0x7effd8075720 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effe3281740 1 -- start start 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effde400640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7effc00b2b20 0x7effc00b4fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effe3281740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b30230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effe3281740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b3aaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effe3281740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b3d420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effe3281740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eb86aa0b90 con 0x55eb86b3b030 2024-05-20T20:17:58.900 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effe3281740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eb86a90730 con 0x55eb86b30770 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.842+0000 7effe3281740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eb86a94770 con 0x55eb86b2fe50 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.843+0000 7effde400640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7effc00b2b20 0x7effc00b4fe0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7effd0004af0 tx=0x7effd0005d80 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.843+0000 7effde400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b3d420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.843+0000 7effde400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b3d420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48476/0 (socket says 172.21.15.28:48476) 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.843+0000 7effde400640 1 -- 172.21.15.28:0/3598502492 learned_addr learned my addr 172.21.15.28:0/3598502492 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.844+0000 7effde400640 1 -- 172.21.15.28:0/3598502492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 msgr2=0x55eb86b30230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.844+0000 7effde400640 1 --2- 172.21.15.28:0/3598502492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b30230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.844+0000 7effde400640 1 -- 172.21.15.28:0/3598502492 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 msgr2=0x55eb86b3aaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.844+0000 7effde400640 1 --2- 172.21.15.28:0/3598502492 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b3aaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.902 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.844+0000 7effde400640 1 -- 172.21.15.28:0/3598502492 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55eb86a298b0 con 0x55eb86b3b030 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.049+0000 7f4676e00700 1 -- 172.21.15.28:0/2925709769 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006bbe0 msgr2=0x7f46701103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.049+0000 7f4676e00700 1 --2- 172.21.15.28:0/2925709769 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006bbe0 0x7f46701103d0 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f466800b0a0 tx=0x7f466800b3b0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.049+0000 7f4676e00700 1 -- 172.21.15.28:0/2925709769 shutdown_connections 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.049+0000 7f4676e00700 1 --2- 172.21.15.28:0/2925709769 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006bbe0 0x7f46701103d0 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.049+0000 7f4676e00700 1 --2- 172.21.15.28:0/2925709769 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f46701095b0 0x7f4670109990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.049+0000 7f4676e00700 1 --2- 172.21.15.28:0/2925709769 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f467006b220 0x7f467006b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.049+0000 7f4676e00700 1 -- 172.21.15.28:0/2925709769 >> 172.21.15.28:0/2925709769 conn(0x7f46700793f0 msgr2=0x7f4670079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.903 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.050+0000 7f4676e00700 1 -- 172.21.15.28:0/2925709769 shutdown_connections 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.050+0000 7f4676e00700 1 -- 172.21.15.28:0/2925709769 wait complete. 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.052+0000 7f4676e00700 1 Processor -- start 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.052+0000 7f4676e00700 1 -- start start 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.052+0000 7f4676e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006b220 0x7f467019f850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.052+0000 7f4676e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f467006bbe0 0x7f467019fd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f4676e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f46701095b0 0x7f4670199a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f4676e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4670113710 con 0x7f467006bbe0 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f4676e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4670113590 con 0x7f467006b220 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f4676e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4670113890 con 0x7f46701095b0 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f466f400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f467006bbe0 0x7f467019fd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.904 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f466fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006b220 0x7f467019f850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f4674e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f46701095b0 0x7f4670199a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f466fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006b220 0x7f467019f850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58018/0 (socket says 172.21.15.28:58018) 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f4674e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f46701095b0 0x7f4670199a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43006/0 (socket says 172.21.15.28:43006) 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f466fe00700 1 -- 172.21.15.28:0/2979816678 learned_addr learned my addr 172.21.15.28:0/2979816678 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.053+0000 7f4674e00700 1 -- 172.21.15.28:0/2979816678 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006b220 msgr2=0x7f467019f850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f4674e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006b220 0x7f467019f850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f4674e00700 1 -- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f467006bbe0 msgr2=0x7f467019fd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f4674e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f467006bbe0 0x7f467019fd90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f4674e00700 1 -- 172.21.15.28:0/2979816678 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f467019a170 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f4674e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f46701095b0 0x7f4670199a10 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f466800b070 tx=0x7f4668004c80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f466cc00700 1 -- 172.21.15.28:0/2979816678 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4668021070 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f466cc00700 1 -- 172.21.15.28:0/2979816678 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4668007e30 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f467019a400 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.054+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4670102440 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.055+0000 7f466cc00700 1 -- 172.21.15.28:0/2979816678 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f46680046c0 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.056+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4650005330 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.059+0000 7f466cc00700 1 -- 172.21.15.28:0/2979816678 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4668007980 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.061+0000 7f466cc00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f46540b2a70 0x7f46540b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.062+0000 7f466cc00700 1 -- 172.21.15.28:0/2979816678 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f46680e3340 con 0x7f46701095b0 2024-05-20T20:17:58.906 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.062+0000 7f466fe00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f46540b2a70 0x7f46540b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.063+0000 7f466fe00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f46540b2a70 0x7f46540b4f30 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f467019ab50 tx=0x7f4660009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.065+0000 7f466cc00700 1 -- 172.21.15.28:0/2979816678 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f46680aada0 con 0x7f46701095b0 2024-05-20T20:17:58.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.480+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f46500058f0 con 0x7f46701095b0 2024-05-20T20:17:58.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.481+0000 7f466cc00700 1 -- 172.21.15.28:0/2979816678 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f46680aa6a0 con 0x7f46701095b0 2024-05-20T20:17:58.907 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:17:58.908 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:17:58.908 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f46540b2a70 msgr2=0x7f46540b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f46540b2a70 0x7f46540b4f30 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f467019ab50 tx=0x7f4660009260 comp rx=0 tx=0).stop 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f46701095b0 msgr2=0x7f4670199a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f46701095b0 0x7f4670199a10 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f466800b070 tx=0x7f4668004c80 comp rx=0 tx=0).stop 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 shutdown_connections 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f46701095b0 0x7f4670199a10 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f46540b2a70 0x7f46540b4f30 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f467006b220 0x7f467019f850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 --2- 172.21.15.28:0/2979816678 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f467006bbe0 0x7f467019fd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 >> 172.21.15.28:0/2979816678 conn(0x7f46700793f0 msgr2=0x7f46701010c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 shutdown_connections 2024-05-20T20:17:58.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:17:58.484+0000 7f4676e00700 1 -- 172.21.15.28:0/2979816678 wait complete. 2024-05-20T20:17:58.910 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:17:58.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.845+0000 7effde400640 1 --2- 172.21.15.28:0/3598502492 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b3d420 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7effd0035000 tx=0x7effd003f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=353e9134e864d23c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.845+0000 7effc6a00640 1 -- 172.21.15.28:0/3598502492 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effd0034040 con 0x55eb86b3b030 2024-05-20T20:17:58.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.845+0000 7effc6a00640 1 -- 172.21.15.28:0/3598502492 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7effd00478d0 con 0x55eb86b3b030 2024-05-20T20:17:58.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.845+0000 7effc6a00640 1 -- 172.21.15.28:0/3598502492 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effd0039840 con 0x55eb86b3b030 2024-05-20T20:17:58.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.860+0000 7effe3281740 1 -- 172.21.15.28:0/3598502492 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 msgr2=0x55eb86b3d420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.860+0000 7effe3281740 1 --2- 172.21.15.28:0/3598502492 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b3d420 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7effd0035000 tx=0x7effd003f000 comp rx=0 tx=0).stop 2024-05-20T20:17:58.914 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.861+0000 7effe3281740 1 -- 172.21.15.28:0/3598502492 shutdown_connections 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.861+0000 7effe3281740 1 --2- 172.21.15.28:0/3598502492 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b3aaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.861+0000 7effe3281740 1 --2- 172.21.15.28:0/3598502492 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b3d420 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.861+0000 7effe3281740 1 --2- 172.21.15.28:0/3598502492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b30230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.861+0000 7effe3281740 1 -- 172.21.15.28:0/3598502492 >> 172.21.15.28:0/3598502492 conn(0x55eb86b2c360 msgr2=0x55eb86b2c740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.861+0000 7effe3281740 1 -- 172.21.15.28:0/3598502492 shutdown_connections 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.861+0000 7effe3281740 1 -- 172.21.15.28:0/3598502492 wait complete. 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.862+0000 7effe3281740 1 -- start start 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.862+0000 7effe3281740 1 Processor -- start 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.864+0000 7effe3281740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b37e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.915 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.864+0000 7effe3281740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b38350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.864+0000 7effe3281740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b38890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effdda00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b37e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effdda00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b37e10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43032/0 (socket says 172.21.15.28:43032) 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effdd000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b38350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effdda00640 1 -- 172.21.15.28:0/3860326969 learned_addr learned my addr 172.21.15.28:0/3860326969 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55eb86776790 con 0x55eb86b3b030 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55eb86a38bc0 con 0x55eb86b30770 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55eb86b4ec10 con 0x55eb86b2fe50 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.865+0000 7effe3281740 10 client.0 _ll_register_callbacks cb 0x55eb86a16400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.866+0000 7effde400640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b38890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.867+0000 7effdda00640 1 -- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 msgr2=0x55eb86b38350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.867+0000 7effdda00640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b38350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.916 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.867+0000 7effdda00640 1 -- 172.21.15.28:0/3860326969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 msgr2=0x55eb86b38890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.867+0000 7effdda00640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b38890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.867+0000 7effdda00640 1 -- 172.21.15.28:0/3860326969 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55eb867a5210 con 0x55eb86b2fe50 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.868+0000 7effde400640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b38890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.868+0000 7effdd000640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b38350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.869+0000 7effbb400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.869+0000 7effdda00640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b37e10 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7effc800e3c0 tx=0x7effc800e880 comp rx=0 tx=0).ready entity=mon.1 client_cookie=41bfaef617f1bf26 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.878+0000 7effbb400640 1 -- 172.21.15.28:0/3860326969 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effc8016070 con 0x55eb86b2fe50 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.879+0000 7effbb400640 1 -- 172.21.15.28:0/3860326969 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7effc80108a0 con 0x55eb86b2fe50 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.879+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7effd00478d0 con 0x55eb86b2fe50 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.880+0000 7effbb400640 1 -- 172.21.15.28:0/3860326969 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7effc8020050 con 0x55eb86b2fe50 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.880+0000 7effe3281740 10 client.27799 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:58.917 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.880+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55eb86871620 con 0x55eb86b2fe50 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.881+0000 7effbb400640 1 -- 172.21.15.28:0/3860326969 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7effd00478d0 con 0x55eb86b2fe50 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.881+0000 7effbb400640 1 -- 172.21.15.28:0/3860326969 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7effc800f040 con 0x55eb86b2fe50 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.882+0000 7effe3281740 10 client.27799 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.882+0000 7effe3281740 10 client.27799 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.882+0000 7effe3281740 20 client.27799 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.882+0000 7effe3281740 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86b5bae0 0x55eb86b5ded0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.882+0000 7effe3281740 4 client.27799 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.882+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55eb86aa2ae0 con 0x55eb86b5bae0 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.883+0000 7effdd000640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86b5bae0 0x55eb86b5ded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.884+0000 7effdd000640 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86b5bae0 0x55eb86b5ded0 crc :-1 s=READY pgs=251 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3e4c1e03f55d7c5b server_cookie=1ffa4ebe54a6a105 in_seq=0 out_seq=0 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.884+0000 7effbb400640 10 client.27799 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:58.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.885+0000 7effbb400640 1 -- 172.21.15.28:0/3860326969 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55eb86aa2ae0 con 0x55eb86b5bae0 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.885+0000 7effbb400640 10 client.27799 handle_command_reply: tid=0 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.887+0000 7effe3281740 1 client.27799 shutdown 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.891+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 msgr2=0x55eb86b37e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.891+0000 7effe3281740 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b37e10 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7effc800e3c0 tx=0x7effc800e880 comp rx=0 tx=0).stop 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.895+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 shutdown_connections 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.895+0000 7effe3281740 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86b5bae0 0x55eb86b5ded0 crc :-1 s=READY pgs=251 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.895+0000 7effe3281740 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b30770 0x55eb86b38350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.895+0000 7effe3281740 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b3b030 0x55eb86b38890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.895+0000 7effe3281740 1 --2- 172.21.15.28:0/3860326969 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b37e10 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.895+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 >> 172.21.15.28:0/3860326969 conn(0x55eb86b2c360 msgr2=0x55eb86b32090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.895+0000 7effbb400640 0 client.27799 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.898+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 shutdown_connections 2024-05-20T20:17:58.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.910+0000 7effe3281740 1 -- 172.21.15.28:0/3860326969 wait complete. 2024-05-20T20:17:58.920 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.910+0000 7effe3281740 20 client.27799 trim_cache size 0 max 16384 2024-05-20T20:17:58.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 Processor -- start 2024-05-20T20:17:58.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 -- start start 2024-05-20T20:17:58.951 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b57c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b30770 0x55eb86b386f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b63790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7effd8073970 con 0x55eb86b30770 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7effd81b67e0 con 0x55eb86b3b030 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effe3281740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7effd8074040 con 0x55eb86b2fe50 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.951+0000 7effdd000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b30770 0x55eb86b386f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdd000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b30770 0x55eb86b386f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48488/0 (socket says 172.21.15.28:48488) 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdda00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b57c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.952 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdd000640 1 -- 172.21.15.28:0/735896586 learned_addr learned my addr 172.21.15.28:0/735896586 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:58.955 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdda00640 1 -- 172.21.15.28:0/735896586 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 msgr2=0x55eb86b63790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdda00640 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b63790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdda00640 1 -- 172.21.15.28:0/735896586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b30770 msgr2=0x55eb86b386f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdda00640 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b30770 0x55eb86b386f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdda00640 1 -- 172.21.15.28:0/735896586 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55eb86b581b0 con 0x55eb86b2fe50 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdd000640 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b30770 0x55eb86b386f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.952+0000 7effdda00640 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b57c70 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7effc801b4e0 tx=0x7effc80058f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e0f5555c919cbb20 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.953+0000 7effc6000640 1 -- 172.21.15.28:0/735896586 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effc8016070 con 0x55eb86b2fe50 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.953+0000 7effc6000640 1 -- 172.21.15.28:0/735896586 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7effc801b760 con 0x55eb86b2fe50 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.953+0000 7effc6000640 1 -- 172.21.15.28:0/735896586 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effc80109d0 con 0x55eb86b2fe50 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 -- 172.21.15.28:0/735896586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 msgr2=0x55eb86b57c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.956 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b57c70 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7effc801b4e0 tx=0x7effc80058f0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 -- 172.21.15.28:0/735896586 shutdown_connections 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b63790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b30770 0x55eb86b386f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 --2- 172.21.15.28:0/735896586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b2fe50 0x55eb86b57c70 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 -- 172.21.15.28:0/735896586 >> 172.21.15.28:0/735896586 conn(0x55eb86b2c360 msgr2=0x55eb86b39ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 -- 172.21.15.28:0/735896586 shutdown_connections 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.954+0000 7effe3281740 1 -- 172.21.15.28:0/735896586 wait complete. 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 -- start start 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 Processor -- start 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b2fe50 0x55eb86b53f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b30770 0x55eb86b54460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b51450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7effd81b6110 con 0x55eb86b2fe50 2024-05-20T20:17:58.957 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7effd8074710 con 0x55eb86b3b030 2024-05-20T20:17:58.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7effd8071970 con 0x55eb86b30770 2024-05-20T20:17:58.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.955+0000 7effe3281740 10 client.0 _ll_register_callbacks cb 0x7effc8009360 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:17:58.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effde400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b51450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effdda00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b2fe50 0x55eb86b53f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effde400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b51450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58076/0 (socket says 172.21.15.28:58076) 2024-05-20T20:17:58.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effde400640 1 -- 172.21.15.28:0/2316560044 learned_addr learned my addr 172.21.15.28:0/2316560044 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:17:58.958 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effdd000640 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b30770 0x55eb86b54460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effdda00640 1 -- 172.21.15.28:0/2316560044 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b30770 msgr2=0x55eb86b54460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effdda00640 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b30770 0x55eb86b54460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effdda00640 1 -- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 msgr2=0x55eb86b51450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effdda00640 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b51450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effdda00640 1 -- 172.21.15.28:0/2316560044 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55eb86b51c80 con 0x55eb86b2fe50 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.956+0000 7effde400640 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b51450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.957+0000 7effdda00640 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b2fe50 0x55eb86b53f20 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7effc8011eb0 tx=0x7effc8004490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b8cc8710143f055f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.957+0000 7effbb400640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.957+0000 7effbb400640 1 -- 172.21.15.28:0/2316560044 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7effc8032060 con 0x55eb86b2fe50 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.957+0000 7effbb400640 1 -- 172.21.15.28:0/2316560044 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7effc801bb10 con 0x55eb86b2fe50 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.957+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7effc801b760 con 0x55eb86b2fe50 2024-05-20T20:17:58.960 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effbb400640 1 -- 172.21.15.28:0/2316560044 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7effc8016070 con 0x55eb86b2fe50 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 10 client.17310 fetch_fsmap learned FSMap version 29 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55eb86bc61a0 con 0x55eb86b2fe50 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effbb400640 1 -- 172.21.15.28:0/2316560044 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7effc801b760 con 0x55eb86b2fe50 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effbb400640 1 -- 172.21.15.28:0/2316560044 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7effc8010ba0 con 0x55eb86b2fe50 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 10 client.17310 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 10 client.17310 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 20 client.17310 populate_metadata read hostname 'smithi028' 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86bc6800 0x55eb86bc8bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 4 client.17310 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.958+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55eb86a9c2c0 con 0x55eb86bc6800 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.959+0000 7effdd000640 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86bc6800 0x55eb86bc8bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.959+0000 7effdd000640 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86bc6800 0x55eb86bc8bf0 crc :-1 s=READY pgs=252 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=de7f45f5719a4e2e server_cookie=4ac7b1ff4dd04403 in_seq=0 out_seq=0 2024-05-20T20:17:58.961 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.959+0000 7effbb400640 10 client.17310 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:58.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.983+0000 7effbb400640 1 -- 172.21.15.28:0/2316560044 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55eb86a9c2c0 con 0x55eb86bc6800 2024-05-20T20:17:58.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.983+0000 7effbb400640 10 client.17310 handle_command_reply: tid=0 2024-05-20T20:17:58.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 client.17310 shutdown 2024-05-20T20:17:58.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b2fe50 msgr2=0x55eb86b53f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b2fe50 0x55eb86b53f20 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7effc8011eb0 tx=0x7effc8004490 comp rx=0 tx=0).stop 2024-05-20T20:17:58.984 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 shutdown_connections 2024-05-20T20:17:58.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55eb86bc6800 0x55eb86bc8bf0 crc :-1 s=READY pgs=252 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55eb86b3b030 0x55eb86b51450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55eb86b30770 0x55eb86b54460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 --2- 172.21.15.28:0/2316560044 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55eb86b2fe50 0x55eb86b53f20 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 >> 172.21.15.28:0/2316560044 conn(0x55eb86b2c360 msgr2=0x55eb86b39ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.985 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.984+0000 7effbb400640 0 client.17310 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:17:58.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.985+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 shutdown_connections 2024-05-20T20:17:58.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.985+0000 7effe3281740 1 -- 172.21.15.28:0/2316560044 wait complete. 2024-05-20T20:17:58.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.985+0000 7effe3281740 20 client.17310 trim_cache size 0 max 16384 2024-05-20T20:17:58.986 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:17:58.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:17:58.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:17:58.987 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7effc00b2b20 msgr2=0x7effc00b4fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7effc00b2b20 0x7effc00b4fe0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7effd0004af0 tx=0x7effd0005d80 comp rx=0 tx=0).stop 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 msgr2=0x7effd810a270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 0x7effd810a270 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7effcc00d950 tx=0x7effcc00de10 comp rx=0 tx=0).stop 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 shutdown_connections 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7effc00b2b20 0x7effc00b4fe0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7effd810f130 0x7effd8104e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7effd8075720 0x7effd810a270 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 --2- 172.21.15.28:0/3123305338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7effd8074e00 0x7effd8109d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.986+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 >> 172.21.15.28:0/3123305338 conn(0x7effd80fe3a0 msgr2=0x7effd80ff1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.987+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 shutdown_connections 2024-05-20T20:17:58.988 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:17:58.987+0000 7effdee00640 1 -- 172.21.15.28:0/3123305338 wait complete. 2024-05-20T20:17:59.561 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:59 smithi028 ceph-mon[31178]: pgmap v385: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 267 MiB/s wr, 1.38k op/s 2024-05-20T20:17:59.561 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:17:59 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2979816678' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:59.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:59 smithi154 ceph-mon[36314]: pgmap v385: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 267 MiB/s wr, 1.38k op/s 2024-05-20T20:17:59.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:17:59 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2979816678' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:59.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:59 smithi033 ceph-mon[36179]: pgmap v385: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 267 MiB/s wr, 1.38k op/s 2024-05-20T20:17:59.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:17:59 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2979816678' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:17:59.812 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id 6c5436e5 --release --await' 2024-05-20T20:17:59.812 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id 6c5436e5 --release --await 2024-05-20T20:18:00.704 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:01.426 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:01 smithi028 ceph-mon[31178]: pgmap v386: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 352 MiB/s wr, 2.01k op/s 2024-05-20T20:18:01.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:01 smithi154 ceph-mon[36314]: pgmap v386: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 352 MiB/s wr, 2.01k op/s 2024-05-20T20:18:01.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:01 smithi033 ceph-mon[36179]: pgmap v386: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 352 MiB/s wr, 2.01k op/s 2024-05-20T20:18:02.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.345+0000 7f830d000700 1 -- 172.21.15.28:0/1838885775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8308076090 msgr2=0x7f83081141d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.345+0000 7f830d000700 1 --2- 172.21.15.28:0/1838885775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8308076090 0x7f83081141d0 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f82f80098f0 tx=0x7f82f8009c00 comp rx=0 tx=0).stop 2024-05-20T20:18:02.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.346+0000 7f830d000700 1 -- 172.21.15.28:0/1838885775 shutdown_connections 2024-05-20T20:18:02.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.346+0000 7f830d000700 1 --2- 172.21.15.28:0/1838885775 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308114710 0x7f8308116b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.346 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.346+0000 7f830d000700 1 --2- 172.21.15.28:0/1838885775 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8308075770 0x7f8308075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.346+0000 7f830d000700 1 --2- 172.21.15.28:0/1838885775 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8308076090 0x7f83081141d0 unknown :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.346+0000 7f830d000700 1 -- 172.21.15.28:0/1838885775 >> 172.21.15.28:0/1838885775 conn(0x7f830807a6a0 msgr2=0x7f830807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:02.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.346+0000 7f830d000700 1 -- 172.21.15.28:0/1838885775 shutdown_connections 2024-05-20T20:18:02.347 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.346+0000 7f830d000700 1 -- 172.21.15.28:0/1838885775 wait complete. 2024-05-20T20:18:02.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.349+0000 7f830d000700 1 Processor -- start 2024-05-20T20:18:02.349 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.349+0000 7f830d000700 1 -- start start 2024-05-20T20:18:02.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.350+0000 7f830d000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8308075770 0x7f83081a0230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.350 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f830d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308076090 0x7f83081a0770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f830d000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8308114710 0x7f83081a4b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f830d000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8308119d20 con 0x7f8308075770 2024-05-20T20:18:02.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f830d000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8308119ba0 con 0x7f8308114710 2024-05-20T20:18:02.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f830d000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8308119ea0 con 0x7f8308076090 2024-05-20T20:18:02.351 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f8305400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308076090 0x7f83081a0770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f8305400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308076090 0x7f83081a0770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43064/0 (socket says 172.21.15.28:43064) 2024-05-20T20:18:02.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f8305400700 1 -- 172.21.15.28:0/2597383891 learned_addr learned my addr 172.21.15.28:0/2597383891 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:02.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.351+0000 7f8306800700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8308114710 0x7f83081a4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.352+0000 7f8305400700 1 -- 172.21.15.28:0/2597383891 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8308114710 msgr2=0x7f83081a4b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.352+0000 7f8305400700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8308114710 0x7f83081a4b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.352+0000 7f8305400700 1 -- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8308075770 msgr2=0x7f83081a0230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.352+0000 7f8305400700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8308075770 0x7f83081a0230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.352+0000 7f8305400700 1 -- 172.21.15.28:0/2597383891 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f83081a5240 con 0x7f8308076090 2024-05-20T20:18:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.352+0000 7f8305400700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308076090 0x7f83081a0770 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f82f8009230 tx=0x7f82f8005040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.352+0000 7f82fea00700 1 -- 172.21.15.28:0/2597383891 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82f8027070 con 0x7f8308076090 2024-05-20T20:18:02.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.353+0000 7f82fea00700 1 -- 172.21.15.28:0/2597383891 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f82f800b560 con 0x7f8308076090 2024-05-20T20:18:02.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.353+0000 7f82fea00700 1 -- 172.21.15.28:0/2597383891 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82f8004b00 con 0x7f8308076090 2024-05-20T20:18:02.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.353+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f83081a54d0 con 0x7f8308076090 2024-05-20T20:18:02.354 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.353+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f83081acdb0 con 0x7f8308076090 2024-05-20T20:18:02.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.359+0000 7f82fea00700 1 -- 172.21.15.28:0/2597383891 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f82f802a070 con 0x7f8308076090 2024-05-20T20:18:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 Processor -- start 2024-05-20T20:18:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 -- start start 2024-05-20T20:18:02.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5599f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e553b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e5545c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562e2e4b1e40 con 0x562e2e559f30 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562e2e4a19e0 con 0x562e2e554140 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.360+0000 7f83179efb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562e2e4a5a20 con 0x562e2e559610 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5599f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8305400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e5545c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5599f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43066/0 (socket says 172.21.15.28:43066) 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 -- 172.21.15.28:0/127442905 learned_addr learned my addr 172.21.15.28:0/127442905 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:02.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8305e00700 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e553b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 -- 172.21.15.28:0/127442905 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 msgr2=0x562e2e5545c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e5545c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 -- 172.21.15.28:0/127442905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 msgr2=0x562e2e553b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e553b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.361+0000 7f8306800700 1 -- 172.21.15.28:0/127442905 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562e2e46a420 con 0x562e2e559610 2024-05-20T20:18:02.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.362+0000 7f8306800700 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5599f0 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f8300009cf0 tx=0x7f8300007500 comp rx=0 tx=0).ready entity=mon.1 client_cookie=adea337c1e0e156d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.362+0000 7f82f7400700 1 -- 172.21.15.28:0/127442905 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8300007d90 con 0x562e2e559610 2024-05-20T20:18:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.362+0000 7f82f7400700 1 -- 172.21.15.28:0/127442905 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8300004bb0 con 0x562e2e559610 2024-05-20T20:18:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.362+0000 7f82f7400700 1 -- 172.21.15.28:0/127442905 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8300014670 con 0x562e2e559610 2024-05-20T20:18:02.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.362+0000 7f82fea00700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82ec0b2a70 0x7f82ec0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f82fea00700 1 -- 172.21.15.28:0/2597383891 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f82f80e30e0 con 0x7f8308076090 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f8305e00700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82ec0b2a70 0x7f82ec0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 -- 172.21.15.28:0/127442905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 msgr2=0x562e2e5599f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5599f0 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f8300009cf0 tx=0x7f8300007500 comp rx=0 tx=0).stop 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 -- 172.21.15.28:0/127442905 shutdown_connections 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e5545c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5599f0 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 --2- 172.21.15.28:0/127442905 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e553b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 -- 172.21.15.28:0/127442905 >> 172.21.15.28:0/127442905 conn(0x562e2e481be0 msgr2=0x562e2e54e690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.363+0000 7f83179efb80 1 -- 172.21.15.28:0/127442905 shutdown_connections 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.364+0000 7f8305e00700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82ec0b2a70 0x7f82ec0b4f30 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f82f00095a0 tx=0x7f82f0009510 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:02.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.364+0000 7f83179efb80 1 -- 172.21.15.28:0/127442905 wait complete. 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.364+0000 7f83179efb80 1 -- start start 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 1 Processor -- start 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e554140 0x562e2e55fc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559610 0x562e2e55d000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e559f30 0x562e2e55d540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x562e2e3a7fa0 con 0x562e2e559610 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x562e2e442760 con 0x562e2e559f30 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x562e2e5609e0 con 0x562e2e554140 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.365+0000 7f83179efb80 10 client.0 _ll_register_callbacks cb 0x562e2e44bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e559f30 0x562e2e55d540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e559f30 0x562e2e55d540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58112/0 (socket says 172.21.15.28:58112) 2024-05-20T20:18:02.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 -- 172.21.15.28:0/1156185213 learned_addr learned my addr 172.21.15.28:0/1156185213 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8305400700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559610 0x562e2e55d000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8305e00700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e554140 0x562e2e55fc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 -- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e554140 msgr2=0x562e2e55fc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e554140 0x562e2e55fc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 -- 172.21.15.28:0/1156185213 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559610 msgr2=0x562e2e55d000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559610 0x562e2e55d000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 -- 172.21.15.28:0/1156185213 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562e2e3f18d0 con 0x562e2e559f30 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8305e00700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e554140 0x562e2e55fc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f8306800700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e559f30 0x562e2e55d540 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7f8300004b80 tx=0x7f8300005510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d1bf2627f1cae733 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:02.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f82ebe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:02.369 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.366+0000 7f82ebe00700 1 -- 172.21.15.28:0/1156185213 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8300022070 con 0x562e2e559f30 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.367+0000 7f82ebe00700 1 -- 172.21.15.28:0/1156185213 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f8300005970 con 0x562e2e559f30 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.367+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f8300004bb0 con 0x562e2e559f30 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.367+0000 7f82ebe00700 1 -- 172.21.15.28:0/1156185213 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f830002c050 con 0x562e2e559f30 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.367+0000 7f83179efb80 10 client.27875 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.367+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562e2e472060 con 0x562e2e559f30 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.367+0000 7f82ebe00700 1 -- 172.21.15.28:0/1156185213 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8300004bb0 con 0x562e2e559f30 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.367+0000 7f82ebe00700 1 -- 172.21.15.28:0/1156185213 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f830001e420 con 0x562e2e559f30 2024-05-20T20:18:02.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.368+0000 7f83179efb80 10 client.27875 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.368+0000 7f83179efb80 10 client.27875 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.368+0000 7f83179efb80 20 client.27875 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.368+0000 7f83179efb80 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e56d4b0 0x562e2e56f8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.368+0000 7f83179efb80 4 client.27875 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.368+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x562e2e4b3d90 con 0x562e2e56d4b0 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.368+0000 7f8305e00700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e56d4b0 0x562e2e56f8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.369+0000 7f8305e00700 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e56d4b0 0x562e2e56f8a0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=bc29069b624049a5 server_cookie=a777d5d43b120440 in_seq=0 out_seq=0 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.369+0000 7f82ebe00700 10 client.27875 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.369+0000 7f82ebe00700 1 -- 172.21.15.28:0/1156185213 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x562e2e4b3d90 con 0x562e2e56d4b0 2024-05-20T20:18:02.371 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.369+0000 7f82ebe00700 10 client.27875 handle_command_reply: tid=0 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.369+0000 7f83179efb80 1 client.27875 shutdown 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e559f30 msgr2=0x562e2e55d540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e559f30 0x562e2e55d540 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7f8300004b80 tx=0x7f8300005510 comp rx=0 tx=0).stop 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 shutdown_connections 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e559f30 0x562e2e55d540 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e554140 0x562e2e55fc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e56d4b0 0x562e2e56f8a0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 --2- 172.21.15.28:0/1156185213 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559610 0x562e2e55d000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 >> 172.21.15.28:0/1156185213 conn(0x562e2e481be0 msgr2=0x562e2e54dd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:02.372 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f82ebe00700 0 client.27875 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:18:02.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.370+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 shutdown_connections 2024-05-20T20:18:02.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.371+0000 7f83179efb80 1 -- 172.21.15.28:0/1156185213 wait complete. 2024-05-20T20:18:02.373 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.371+0000 7f83179efb80 20 client.27875 trim_cache size 0 max 16384 2024-05-20T20:18:02.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.404+0000 7f83179efb80 1 Processor -- start 2024-05-20T20:18:02.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.404+0000 7f83179efb80 1 -- start start 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f83179efb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e553b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f83179efb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e55aba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f83179efb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e55c020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f83179efb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f83081b4320 con 0x562e2e559f30 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f83179efb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f83081b5e60 con 0x562e2e554140 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f83179efb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f83081b49f0 con 0x562e2e559610 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f8305400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e55aba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f8305e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e553b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f8305400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e55aba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43082/0 (socket says 172.21.15.28:43082) 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f8305400700 1 -- 172.21.15.28:0/3356793829 learned_addr learned my addr 172.21.15.28:0/3356793829 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.405+0000 7f8306800700 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e55c020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.406+0000 7f8305400700 1 -- 172.21.15.28:0/3356793829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 msgr2=0x562e2e553b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.406+0000 7f8305400700 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e553b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.406+0000 7f8305400700 1 -- 172.21.15.28:0/3356793829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 msgr2=0x562e2e55c020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.406+0000 7f8305400700 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e55c020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.406+0000 7f8305400700 1 -- 172.21.15.28:0/3356793829 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x562e2e55b550 con 0x562e2e559610 2024-05-20T20:18:02.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.406+0000 7f8305400700 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e55aba0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f82f80aa3d0 tx=0x7f82f80e6c30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b3f6b46143a0ce79 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:02.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.407+0000 7f82f6a00700 1 -- 172.21.15.28:0/3356793829 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82f80aa210 con 0x562e2e559610 2024-05-20T20:18:02.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.407+0000 7f82f6a00700 1 -- 172.21.15.28:0/3356793829 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f82f80a9f50 con 0x562e2e559610 2024-05-20T20:18:02.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.407+0000 7f82f6a00700 1 -- 172.21.15.28:0/3356793829 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82f80a9bb0 con 0x562e2e559610 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 -- 172.21.15.28:0/3356793829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 msgr2=0x562e2e55aba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e55aba0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f82f80aa3d0 tx=0x7f82f80e6c30 comp rx=0 tx=0).stop 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 -- 172.21.15.28:0/3356793829 shutdown_connections 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e55c020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e553b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 --2- 172.21.15.28:0/3356793829 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e55aba0 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 -- 172.21.15.28:0/3356793829 >> 172.21.15.28:0/3356793829 conn(0x562e2e481be0 msgr2=0x562e2e55b910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:02.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.408+0000 7f83179efb80 1 -- 172.21.15.28:0/3356793829 shutdown_connections 2024-05-20T20:18:02.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.409+0000 7f83179efb80 1 -- 172.21.15.28:0/3356793829 wait complete. 2024-05-20T20:18:02.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.409+0000 7f83179efb80 1 -- start start 2024-05-20T20:18:02.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.409+0000 7f83179efb80 1 Processor -- start 2024-05-20T20:18:02.409 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.410+0000 7f83179efb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e565c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.410+0000 7f83179efb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5661b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.410 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.410+0000 7f83179efb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e562c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5661b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5661b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43088/0 (socket says 172.21.15.28:43088) 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f83081b5790 con 0x562e2e559f30 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 -- 172.21.15.28:0/1367046024 learned_addr learned my addr 172.21.15.28:0/1367046024 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f83081b50c0 con 0x562e2e554140 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f83081b22e0 con 0x562e2e559610 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305e00700 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e565c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f83179efb80 10 client.0 _ll_register_callbacks cb 0x562e2e44bc70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:02.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8306800700 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e562c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 -- 172.21.15.28:0/1367046024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 msgr2=0x562e2e565c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e565c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 -- 172.21.15.28:0/1367046024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 msgr2=0x562e2e562c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:02.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e562c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.411+0000 7f8305400700 1 -- 172.21.15.28:0/1367046024 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x562e2e566830 con 0x562e2e559610 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.412+0000 7f8305400700 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5661b0 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f82f80e08d0 tx=0x7f82f80aaa00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=aa23f8e716c6e37c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.412+0000 7f82ebe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.412+0000 7f82ebe00700 1 -- 172.21.15.28:0/1367046024 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82f80a9d90 con 0x562e2e559610 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.412+0000 7f82ebe00700 1 -- 172.21.15.28:0/1367046024 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f82f80b2f90 con 0x562e2e559610 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.412+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f82f80a9f50 con 0x562e2e559610 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.413+0000 7f82ebe00700 1 -- 172.21.15.28:0/1367046024 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f82f80a9bb0 con 0x562e2e559610 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.413+0000 7f83179efb80 10 client.27820 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:02.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.413+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x562e2e5d8360 con 0x562e2e559610 2024-05-20T20:18:02.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f82ebe00700 1 -- 172.21.15.28:0/1367046024 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f82f80a9f50 con 0x562e2e559610 2024-05-20T20:18:02.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f82ebe00700 1 -- 172.21.15.28:0/1367046024 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f82f80a4b60 con 0x562e2e559610 2024-05-20T20:18:02.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f83179efb80 10 client.27820 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:02.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f83179efb80 10 client.27820 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:18:02.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f83179efb80 20 client.27820 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:02.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f83179efb80 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e5d8990 0x562e2e5dadb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:02.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f83179efb80 4 client.27820 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:18:02.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.415+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x562e2e4ad570 con 0x562e2e5d8990 2024-05-20T20:18:02.416 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.416+0000 7f8305e00700 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e5d8990 0x562e2e5dadb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:02.418 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.419+0000 7f82ebe00700 10 client.27820 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:18:02.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.419+0000 7f8305e00700 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e5d8990 0x562e2e5dadb0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f2a10c900bff1ca3 server_cookie=d43393ecbb9e73aa in_seq=0 out_seq=0 2024-05-20T20:18:02.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.419+0000 7f82ebe00700 1 -- 172.21.15.28:0/1367046024 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x562e2e4ad570 con 0x562e2e5d8990 2024-05-20T20:18:02.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.419+0000 7f82ebe00700 10 client.27820 handle_command_reply: tid=0 2024-05-20T20:18:02.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.420+0000 7f83179efb80 1 client.27820 shutdown 2024-05-20T20:18:02.421 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 msgr2=0x562e2e5661b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5661b0 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f82f80e08d0 tx=0x7f82f80aaa00 comp rx=0 tx=0).stop 2024-05-20T20:18:02.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 shutdown_connections 2024-05-20T20:18:02.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x562e2e5d8990 0x562e2e5dadb0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x562e2e559f30 0x562e2e562c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.422 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x562e2e554140 0x562e2e565c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 --2- 172.21.15.28:0/1367046024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x562e2e559610 0x562e2e5661b0 secure :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f82f80e08d0 tx=0x7f82f80aaa00 comp rx=0 tx=0).stop 2024-05-20T20:18:02.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 >> 172.21.15.28:0/1367046024 conn(0x562e2e481be0 msgr2=0x562e2e55c1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:02.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.421+0000 7f82ebe00700 0 client.27820 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:18:02.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.422+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 shutdown_connections 2024-05-20T20:18:02.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.422+0000 7f83179efb80 1 -- 172.21.15.28:0/1367046024 wait complete. 2024-05-20T20:18:02.423 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.424+0000 7f83179efb80 20 client.27820 trim_cache size 0 max 16384 2024-05-20T20:18:02.423 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:18:02.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82ec0b2a70 msgr2=0x7f82ec0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82ec0b2a70 0x7f82ec0b4f30 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f82f00095a0 tx=0x7f82f0009510 comp rx=0 tx=0).stop 2024-05-20T20:18:02.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308076090 msgr2=0x7f83081a0770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:02.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308076090 0x7f83081a0770 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f82f8009230 tx=0x7f82f8005040 comp rx=0 tx=0).stop 2024-05-20T20:18:02.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 shutdown_connections 2024-05-20T20:18:02.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f82ec0b2a70 0x7f82ec0b4f30 secure :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f82f00095a0 tx=0x7f82f0009510 comp rx=0 tx=0).stop 2024-05-20T20:18:02.426 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8308114710 0x7f83081a4b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8308075770 0x7f83081a0230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 --2- 172.21.15.28:0/2597383891 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8308076090 0x7f83081a0770 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:02.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.426+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 >> 172.21.15.28:0/2597383891 conn(0x7f830807a6a0 msgr2=0x7f83081101b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:02.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.427+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 shutdown_connections 2024-05-20T20:18:02.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:02.427+0000 7f830d000700 1 -- 172.21.15.28:0/2597383891 wait complete. 2024-05-20T20:18:03.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:03 smithi028 ceph-mon[31178]: pgmap v387: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 255 MiB/s wr, 1.60k op/s 2024-05-20T20:18:03.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:03 smithi154 ceph-mon[36314]: pgmap v387: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 255 MiB/s wr, 1.60k op/s 2024-05-20T20:18:03.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:03 smithi033 ceph-mon[36179]: pgmap v387: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 255 MiB/s wr, 1.60k op/s 2024-05-20T20:18:04.602 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:18:05.082 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:05.430 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:05 smithi028 ceph-mon[31178]: pgmap v388: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 354 MiB/s wr, 2.45k op/s 2024-05-20T20:18:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 -- 172.21.15.28:0/3257524659 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 msgr2=0x7f4a70110440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 --2- 172.21.15.28:0/3257524659 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 0x7f4a70110440 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f4a6800b0d0 tx=0x7f4a6800b3e0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 -- 172.21.15.28:0/3257524659 shutdown_connections 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 --2- 172.21.15.28:0/3257524659 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 0x7f4a70110440 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 --2- 172.21.15.28:0/3257524659 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a70109670 0x7f4a70109a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 --2- 172.21.15.28:0/3257524659 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a701035f0 0x7f4a70103a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 -- 172.21.15.28:0/3257524659 >> 172.21.15.28:0/3257524659 conn(0x7f4a7007a690 msgr2=0x7f4a7007aaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 -- 172.21.15.28:0/3257524659 shutdown_connections 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.777+0000 7f4a75000700 1 -- 172.21.15.28:0/3257524659 wait complete. 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.778+0000 7f4a75000700 1 Processor -- start 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a75000700 1 -- start start 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a75000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a701035f0 0x7f4a7019bda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a75000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 0x7f4a7019c2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a75000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a70109670 0x7f4a701a0670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a75000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4a701137b0 con 0x7f4a70103fb0 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a75000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4a70113630 con 0x7f4a70109670 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a75000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4a70113930 con 0x7f4a701035f0 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.779+0000 7f4a6de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a701035f0 0x7f4a7019bda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a701035f0 0x7f4a7019bda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43108/0 (socket says 172.21.15.28:43108) 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6e800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a70109670 0x7f4a701a0670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 -- 172.21.15.28:0/547919252 learned_addr learned my addr 172.21.15.28:0/547919252 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6d400700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 0x7f4a7019c2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 -- 172.21.15.28:0/547919252 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a70109670 msgr2=0x7f4a701a0670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a70109670 0x7f4a701a0670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 -- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 msgr2=0x7f4a7019c2e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 0x7f4a7019c2e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 -- 172.21.15.28:0/547919252 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4a701a0db0 con 0x7f4a701035f0 2024-05-20T20:18:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6d400700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 0x7f4a7019c2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a6de00700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a701035f0 0x7f4a7019bda0 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f4a6000e9c0 tx=0x7f4a6000ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.780+0000 7f4a66a00700 1 -- 172.21.15.28:0/547919252 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a6000cbd0 con 0x7f4a701035f0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.781+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4a701a10a0 con 0x7f4a701035f0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.781+0000 7f4a66a00700 1 -- 172.21.15.28:0/547919252 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a6000cd70 con 0x7f4a701035f0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.781+0000 7f4a66a00700 1 -- 172.21.15.28:0/547919252 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a60005610 con 0x7f4a701035f0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.781+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4a701a8920 con 0x7f4a701035f0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.784+0000 7f4a66a00700 1 -- 172.21.15.28:0/547919252 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4a60004750 con 0x7f4a701035f0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.785+0000 7f4a7f9a1b80 1 Processor -- start 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a7f9a1b80 1 -- start start 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a66a00700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4a540b29f0 0x7f4a540b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a66a00700 1 -- 172.21.15.28:0/547919252 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f4a600e29b0 con 0x7f4a701035f0 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba53ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba54530 0x5627bba4dfd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba4ea50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5627bb9abf40 con 0x5627bba4e5f0 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5627bb99bae0 con 0x5627bba53c10 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5627bb99fb20 con 0x5627bba54530 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a6e800700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4a540b29f0 0x7f4a540b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a6d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba54530 0x5627bba4dfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a6d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba54530 0x5627bba4dfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43110/0 (socket says 172.21.15.28:43110) 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a6d400700 1 -- 172.21.15.28:0/1781025124 learned_addr learned my addr 172.21.15.28:0/1781025124 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.786+0000 7f4a6de00700 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba53ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6e800700 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba4ea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6de00700 1 -- 172.21.15.28:0/1781025124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba54530 msgr2=0x5627bba4dfd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6de00700 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba54530 0x5627bba4dfd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6de00700 1 -- 172.21.15.28:0/1781025124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 msgr2=0x5627bba4ea50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6de00700 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba4ea50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6de00700 1 -- 172.21.15.28:0/1781025124 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5627bb964520 con 0x5627bba53c10 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6d400700 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba54530 0x5627bba4dfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6de00700 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba53ff0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f4a600e0090 tx=0x7f4a600a9b20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fbf5f464312be723 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a6e800700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4a540b29f0 0x7f4a540b4eb0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f4a68006260 tx=0x7f4a68009440 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a5b400700 1 -- 172.21.15.28:0/1781025124 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a600a9730 con 0x5627bba53c10 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a5b400700 1 -- 172.21.15.28:0/1781025124 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a600aa320 con 0x5627bba53c10 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.787+0000 7f4a5b400700 1 -- 172.21.15.28:0/1781025124 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a600a9e80 con 0x5627bba53c10 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.788+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/1781025124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 msgr2=0x5627bba53ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.788+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba53ff0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f4a600e0090 tx=0x7f4a600a9b20 comp rx=0 tx=0).stop 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.788+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/1781025124 shutdown_connections 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.788+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba4ea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.788+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba53ff0 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.788+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/1781025124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba54530 0x5627bba4dfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.788+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/1781025124 >> 172.21.15.28:0/1781025124 conn(0x5627bb97bce0 msgr2=0x5627bba48f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.789+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/1781025124 shutdown_connections 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.789+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/1781025124 wait complete. 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.789+0000 7f4a7f9a1b80 1 -- start start 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.789+0000 7f4a7f9a1b80 1 Processor -- start 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba4e5f0 0x5627bba5a2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba53c10 0x5627bba57650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba57b90 0x5627bba58010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5627bb8a20a0 con 0x5627bba53c10 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5627bb93c860 con 0x5627bba57b90 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5627bba5afa0 con 0x5627bba4e5f0 2024-05-20T20:18:05.824 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a7f9a1b80 10 client.0 _ll_register_callbacks cb 0x5627bb945d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a6d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba53c10 0x5627bba57650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a6de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba4e5f0 0x5627bba5a2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a6d400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba53c10 0x5627bba57650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:48586/0 (socket says 172.21.15.28:48586) 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a6d400700 1 -- 172.21.15.28:0/2461571093 learned_addr learned my addr 172.21.15.28:0/2461571093 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a6de00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba4e5f0 0x5627bba5a2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43116/0 (socket says 172.21.15.28:43116) 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.790+0000 7f4a6e800700 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba57b90 0x5627bba58010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a6de00700 1 -- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba57b90 msgr2=0x5627bba58010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a6de00700 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba57b90 0x5627bba58010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a6de00700 1 -- 172.21.15.28:0/2461571093 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba53c10 msgr2=0x5627bba57650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a6de00700 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba53c10 0x5627bba57650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a6de00700 1 -- 172.21.15.28:0/2461571093 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627bb8eb9d0 con 0x5627bba4e5f0 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a6de00700 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba4e5f0 0x5627bba5a2b0 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f4a600a98d0 tx=0x7f4a600aa110 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4d7e55148c0f6b83 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:05.825 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a4fe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2461571093 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a600a9730 con 0x5627bba4e5f0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2461571093 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f4a600c42b0 con 0x5627bba4e5f0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4a600aa320 con 0x5627bba4e5f0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2461571093 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4a600adfd0 con 0x5627bba4e5f0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.791+0000 7f4a7f9a1b80 10 client.27832 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5627bb96c160 con 0x5627bba4e5f0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2461571093 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a600aa320 con 0x5627bba4e5f0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2461571093 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4a600a5700 con 0x5627bba4e5f0 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a7f9a1b80 10 client.27832 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a7f9a1b80 10 client.27832 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a7f9a1b80 20 client.27832 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bba67e00 0x5627bba6a1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a7f9a1b80 4 client.27832 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.792+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5627bb9ade90 con 0x5627bba67e00 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.793+0000 7f4a6d400700 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bba67e00 0x5627bba6a1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.793+0000 7f4a6d400700 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bba67e00 0x5627bba6a1f0 crc :-1 s=READY pgs=253 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8324e7c64b47a34c server_cookie=2616a2d497de794b in_seq=0 out_seq=0 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.793+0000 7f4a4fe00700 10 client.27832 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.794+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2461571093 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5627bb9ade90 con 0x5627bba67e00 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.794+0000 7f4a4fe00700 10 client.27832 handle_command_reply: tid=0 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.794+0000 7f4a7f9a1b80 1 client.27832 shutdown 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba4e5f0 msgr2=0x5627bba5a2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba4e5f0 0x5627bba5a2b0 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f4a600a98d0 tx=0x7f4a600aa110 comp rx=0 tx=0).stop 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 shutdown_connections 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bba67e00 0x5627bba6a1f0 crc :-1 s=READY pgs=253 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba57b90 0x5627bba58010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba4e5f0 0x5627bba5a2b0 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2461571093 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba53c10 0x5627bba57650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 >> 172.21.15.28:0/2461571093 conn(0x5627bb97bce0 msgr2=0x5627bba48430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a4fe00700 0 client.27832 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 shutdown_connections 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2461571093 wait complete. 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.795+0000 7f4a7f9a1b80 20 client.27832 trim_cache size 0 max 16384 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.817+0000 7f4a7f9a1b80 1 Processor -- start 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a7f9a1b80 1 -- start start 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba4e5f0 0x5627bba4dfd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba53c10 0x5627bba51630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba57b90 0x5627bba51b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4a701b0020 con 0x5627bba57b90 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4a701b1b60 con 0x5627bba4e5f0 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4a701b06f0 con 0x5627bba53c10 2024-05-20T20:18:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a6e800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba57b90 0x5627bba51b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a6de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba4e5f0 0x5627bba4dfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a6d400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba53c10 0x5627bba51630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a6de00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba4e5f0 0x5627bba4dfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58186/0 (socket says 172.21.15.28:58186) 2024-05-20T20:18:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.818+0000 7f4a6de00700 1 -- 172.21.15.28:0/3313584126 learned_addr learned my addr 172.21.15.28:0/3313584126 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.819+0000 7f4a6de00700 1 -- 172.21.15.28:0/3313584126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba53c10 msgr2=0x5627bba51630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.819+0000 7f4a6de00700 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba53c10 0x5627bba51630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.819+0000 7f4a6de00700 1 -- 172.21.15.28:0/3313584126 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba57b90 msgr2=0x5627bba51b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.819+0000 7f4a6de00700 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba57b90 0x5627bba51b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.819+0000 7f4a6de00700 1 -- 172.21.15.28:0/3313584126 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5627bba5a890 con 0x5627bba4e5f0 2024-05-20T20:18:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.819+0000 7f4a6d400700 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba53c10 0x5627bba51630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.819+0000 7f4a6de00700 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba4e5f0 0x5627bba4dfd0 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7f4a600a6e90 tx=0x7f4a600050b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=64a66b8e41267088 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a5aa00700 1 -- 172.21.15.28:0/3313584126 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a6008b070 con 0x5627bba4e5f0 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a5aa00700 1 -- 172.21.15.28:0/3313584126 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a600aa320 con 0x5627bba4e5f0 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a5aa00700 1 -- 172.21.15.28:0/3313584126 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a600a7ff0 con 0x5627bba4e5f0 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/3313584126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba4e5f0 msgr2=0x5627bba4dfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba4e5f0 0x5627bba4dfd0 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x7f4a600a6e90 tx=0x7f4a600050b0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/3313584126 shutdown_connections 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba57b90 0x5627bba51b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba4e5f0 0x5627bba4dfd0 unknown :-1 s=CLOSED pgs=588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/3313584126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba53c10 0x5627bba51630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.820+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/3313584126 >> 172.21.15.28:0/3313584126 conn(0x5627bb97bce0 msgr2=0x5627bba55f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.821+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/3313584126 shutdown_connections 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.821+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/3313584126 wait complete. 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.821+0000 7f4a7f9a1b80 1 -- start start 2024-05-20T20:18:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.821+0000 7f4a7f9a1b80 1 Processor -- start 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba6d850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba6bc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a7f9a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba57b90 0x5627bba6c140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4a701b1490 con 0x5627bba4e5f0 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4a701b0dc0 con 0x5627bba53c10 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a7f9a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4a701adfe0 con 0x5627bba57b90 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a7f9a1b80 10 client.0 _ll_register_callbacks cb 0x5627bb945d70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a6de00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba6d850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a6d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba6bc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a6d400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba6bc00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:58194/0 (socket says 172.21.15.28:58194) 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a6d400700 1 -- 172.21.15.28:0/2317808552 learned_addr learned my addr 172.21.15.28:0/2317808552 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:05.834 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.822+0000 7f4a6e800700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba57b90 0x5627bba6c140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6de00700 1 -- 172.21.15.28:0/2317808552 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba57b90 msgr2=0x5627bba6c140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6de00700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba57b90 0x5627bba6c140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6de00700 1 -- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 msgr2=0x5627bba6bc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6de00700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba6bc00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6de00700 1 -- 172.21.15.28:0/2317808552 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5627bba6c980 con 0x5627bba4e5f0 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6e800700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba57b90 0x5627bba6c140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6d400700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba6bc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a6de00700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba6d850 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f4a60009fd0 tx=0x7f4a600a5b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=51e24d1078db12dc server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.823+0000 7f4a4fe00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2317808552 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a6008b020 con 0x5627bba4e5f0 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2317808552 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(93..93 src has 1..93) ==== 8542+0+0 (secure 0 0 0) 0x7f4a600c42b0 con 0x5627bba4e5f0 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4a600aa320 con 0x5627bba4e5f0 2024-05-20T20:18:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2317808552 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4a600b2a80 con 0x5627bba4e5f0 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 10 client.17343 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5627bba6c980 con 0x5627bba4e5f0 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2317808552 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4a600aa320 con 0x5627bba4e5f0 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2317808552 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4a6008e020 con 0x5627bba4e5f0 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 10 client.17343 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 10 client.17343 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 20 client.17343 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bbad37e0 0x5627bbad5bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 4 client.17343 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "6c5436e5", "release": true, "await": true}] 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.824+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "6c5436e5", "release": true, "await": true}) -- 0x5627bb9a7670 con 0x5627bbad37e0 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.825+0000 7f4a6d400700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bbad37e0 0x5627bbad5bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.825+0000 7f4a6d400700 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bbad37e0 0x5627bbad5bd0 crc :-1 s=READY pgs=254 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f6a58b078202d5cd server_cookie=bb0da0ac8b2b846c in_seq=0 out_seq=0 2024-05-20T20:18:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.825+0000 7f4a4fe00700 10 client.17343 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.833+0000 7f4a4fe00700 1 -- 172.21.15.28:0/2317808552 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x5627bb9a7670 con 0x5627bbad37e0 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.833+0000 7f4a4fe00700 10 client.17343 handle_command_reply: tid=0 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.833+0000 7f4a7f9a1b80 1 client.17343 shutdown 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 msgr2=0x5627bba6d850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba6d850 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f4a60009fd0 tx=0x7f4a600a5b60 comp rx=0 tx=0).stop 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 shutdown_connections 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5627bbad37e0 0x5627bbad5bd0 crc :-1 s=READY pgs=254 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5627bba57b90 0x5627bba6c140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5627bba4e5f0 0x5627bba6d850 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 --2- 172.21.15.28:0/2317808552 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5627bba53c10 0x5627bba6bc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a4fe00700 0 client.17343 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 >> 172.21.15.28:0/2317808552 conn(0x5627bb97bce0 msgr2=0x5627bba51630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.834+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 shutdown_connections 2024-05-20T20:18:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.835+0000 7f4a7f9a1b80 1 -- 172.21.15.28:0/2317808552 wait complete. 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.836+0000 7f4a7f9a1b80 20 client.17343 trim_cache size 0 max 16384 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 32, 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout: "6c5436e5": { 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 32, 2024-05-20T20:18:05.839 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 137.9, 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:18:05.840 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:05.841 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:05.841 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:05.841 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:05.841 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:05.841 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.838+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4a540b29f0 msgr2=0x7f4a540b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.838+0000 7f4a75000700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4a540b29f0 0x7f4a540b4eb0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f4a68006260 tx=0x7f4a68009440 comp rx=0 tx=0).stop 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.838+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a701035f0 msgr2=0x7f4a7019bda0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.838+0000 7f4a75000700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a701035f0 0x7f4a7019bda0 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f4a6000e9c0 tx=0x7f4a6000ecd0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.840+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 shutdown_connections 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.840+0000 7f4a75000700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4a540b29f0 0x7f4a540b4eb0 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.840+0000 7f4a75000700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4a70109670 0x7f4a701a0670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.840+0000 7f4a75000700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a701035f0 0x7f4a7019bda0 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.840+0000 7f4a75000700 1 --2- 172.21.15.28:0/547919252 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4a70103fb0 0x7f4a7019c2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:05.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.840+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 >> 172.21.15.28:0/547919252 conn(0x7f4a7007a690 msgr2=0x7f4a70101fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:05.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.840+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 shutdown_connections 2024-05-20T20:18:05.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:05.841+0000 7f4a75000700 1 -- 172.21.15.28:0/547919252 wait complete. 2024-05-20T20:18:05.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:05 smithi154 ceph-mon[36314]: pgmap v388: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 354 MiB/s wr, 2.45k op/s 2024-05-20T20:18:05.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:05 smithi033 ceph-mon[36179]: pgmap v388: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 354 MiB/s wr, 2.45k op/s 2024-05-20T20:18:07.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:07 smithi028 ceph-mon[31178]: pgmap v389: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 263 MiB/s wr, 2.02k op/s 2024-05-20T20:18:07.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:07 smithi154 ceph-mon[36314]: pgmap v389: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 263 MiB/s wr, 2.02k op/s 2024-05-20T20:18:07.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:07 smithi033 ceph-mon[36179]: pgmap v389: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 263 MiB/s wr, 2.02k op/s 2024-05-20T20:18:08.432 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set '6c5436e5', total seconds elapsed: 39.0 2024-05-20T20:18:08.432 DEBUG:tasks.quiescer.fs.[cephfs]:Going to quiesce for duration: 17.9 2024-05-20T20:18:08.432 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:18:08.671 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:08 smithi028 ceph-mon[31178]: pgmap v390: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 249 MiB/s wr, 1.98k op/s 2024-05-20T20:18:08.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:08 smithi154 ceph-mon[36314]: pgmap v390: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 249 MiB/s wr, 1.98k op/s 2024-05-20T20:18:08.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:08 smithi033 ceph-mon[36179]: pgmap v390: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 249 MiB/s wr, 1.98k op/s 2024-05-20T20:18:09.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:09 smithi028 ceph-mon[31178]: osdmap e94: 12 total, 12 up, 12 in 2024-05-20T20:18:09.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:09 smithi154 ceph-mon[36314]: osdmap e94: 12 total, 12 up, 12 in 2024-05-20T20:18:09.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:09 smithi033 ceph-mon[36179]: osdmap e94: 12 total, 12 up, 12 in 2024-05-20T20:18:10.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:10 smithi028 ceph-mon[31178]: pgmap v392: 97 pgs: 16 active+clean+snaptrim, 23 active+clean+snaptrim_wait, 58 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 197 MiB/s wr, 1.82k op/s 2024-05-20T20:18:10.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:10 smithi154 ceph-mon[36314]: pgmap v392: 97 pgs: 16 active+clean+snaptrim, 23 active+clean+snaptrim_wait, 58 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 197 MiB/s wr, 1.82k op/s 2024-05-20T20:18:10.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:10 smithi033 ceph-mon[36179]: pgmap v392: 97 pgs: 16 active+clean+snaptrim, 23 active+clean+snaptrim_wait, 58 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 197 MiB/s wr, 1.82k op/s 2024-05-20T20:18:11.799 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.346+0000 7f0b33e00640 1 -- 172.21.15.28:0/1874676622 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b240a2a30 msgr2=0x7f0b240a4e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.346+0000 7f0b33e00640 1 --2- 172.21.15.28:0/1874676622 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b240a2a30 0x7f0b240a4e20 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c0681c0 tx=0x7f0b2c0977e0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.347+0000 7f0b33e00640 1 -- 172.21.15.28:0/1874676622 shutdown_connections 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.347+0000 7f0b33e00640 1 --2- 172.21.15.28:0/1874676622 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b240a2a30 0x7f0b240a4e20 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.347+0000 7f0b33e00640 1 --2- 172.21.15.28:0/1874676622 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006e80 0x7f0b240a24f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.347+0000 7f0b33e00640 1 --2- 172.21.15.28:0/1874676622 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b24006560 0x7f0b24006940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.347+0000 7f0b33e00640 1 -- 172.21.15.28:0/1874676622 >> 172.21.15.28:0/1874676622 conn(0x7f0b2400a770 msgr2=0x7f0b2400ab80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.347+0000 7f0b33e00640 1 -- 172.21.15.28:0/1874676622 shutdown_connections 2024-05-20T20:18:12.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.347+0000 7f0b33e00640 1 -- 172.21.15.28:0/1874676622 wait complete. 2024-05-20T20:18:12.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.349+0000 7f0b33e00640 1 Processor -- start 2024-05-20T20:18:12.352 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.350+0000 7f0b33e00640 1 -- start start 2024-05-20T20:18:12.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b33e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006560 0x7f0b2412e6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b33e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b24006e80 0x7f0b2412ec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.354 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b31400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006560 0x7f0b2412e6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b33e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b240a2a30 0x7f0b24132f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b33e00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0b240a81a0 con 0x7f0b240a2a30 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b31400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006560 0x7f0b2412e6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43960/0 (socket says 172.21.15.28:43960) 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0b240a8020 con 0x7f0b24006e80 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b31400640 1 -- 172.21.15.28:0/857558965 learned_addr learned my addr 172.21.15.28:0/857558965 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.351+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0b240a8320 con 0x7f0b24006560 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.352+0000 7f0b30a00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b24006e80 0x7f0b2412ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.352+0000 7f0b31400640 1 -- 172.21.15.28:0/857558965 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b24006e80 msgr2=0x7f0b2412ec00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.352+0000 7f0b31400640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b24006e80 0x7f0b2412ec00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.352+0000 7f0b31400640 1 -- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b240a2a30 msgr2=0x7f0b24132f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:12.355 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.352+0000 7f0b31400640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b240a2a30 0x7f0b24132f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.354+0000 7f0b31400640 1 -- 172.21.15.28:0/857558965 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0b241336b0 con 0x7f0b24006560 2024-05-20T20:18:12.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.354+0000 7f0b31400640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006560 0x7f0b2412e6c0 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f0b2000b4a0 tx=0x7f0b2000b960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:12.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.355+0000 7f0b2a000640 1 -- 172.21.15.28:0/857558965 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b20025070 con 0x7f0b24006560 2024-05-20T20:18:12.356 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.355+0000 7f0b2a000640 1 -- 172.21.15.28:0/857558965 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0b20021ac0 con 0x7f0b24006560 2024-05-20T20:18:12.357 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.357+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0b241339a0 con 0x7f0b24006560 2024-05-20T20:18:12.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.357+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0b24097090 con 0x7f0b24006560 2024-05-20T20:18:12.358 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.357+0000 7f0b2a000640 1 -- 172.21.15.28:0/857558965 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b20025070 con 0x7f0b24006560 2024-05-20T20:18:12.374 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.374+0000 7f0b36dec740 1 Processor -- start 2024-05-20T20:18:12.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.374+0000 7f0b36dec740 1 -- start start 2024-05-20T20:18:12.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.375+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd5f250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.375 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.375+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 0x56111bd69b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.375+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd6c440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.375+0000 7f0b36dec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56111bccfb90 con 0x56111bd5f790 2024-05-20T20:18:12.376 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.375+0000 7f0b36dec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56111bcbf730 con 0x56111bd6a050 2024-05-20T20:18:12.377 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.375+0000 7f0b36dec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56111bcc3770 con 0x56111bd5ee70 2024-05-20T20:18:12.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.376+0000 7f0b31e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd5f250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.379 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.376+0000 7f0b31e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd5f250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43970/0 (socket says 172.21.15.28:43970) 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.376+0000 7f0b31e00640 1 -- 172.21.15.28:0/3525308936 learned_addr learned my addr 172.21.15.28:0/3525308936 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.377+0000 7f0b30a00640 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd6c440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.377+0000 7f0b30a00640 1 -- 172.21.15.28:0/3525308936 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 msgr2=0x56111bd5f250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.377+0000 7f0b30a00640 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd5f250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.378+0000 7f0b30a00640 1 -- 172.21.15.28:0/3525308936 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 msgr2=0x56111bd69b10 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.378+0000 7f0b30a00640 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 0x56111bd69b10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.378+0000 7f0b30a00640 1 -- 172.21.15.28:0/3525308936 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56111bc588b0 con 0x56111bd6a050 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.378+0000 7f0b30a00640 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd6c440 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f0b24096190 tx=0x7f0b1800acc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6b5dddb81198f590 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.378+0000 7f0b1e600640 1 -- 172.21.15.28:0/3525308936 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b18015070 con 0x56111bd6a050 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.379+0000 7f0b1e600640 1 -- 172.21.15.28:0/3525308936 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0b180113f0 con 0x56111bd6a050 2024-05-20T20:18:12.381 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.379+0000 7f0b1e600640 1 -- 172.21.15.28:0/3525308936 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b180185c0 con 0x56111bd6a050 2024-05-20T20:18:12.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.381+0000 7f0b2a000640 1 -- 172.21.15.28:0/857558965 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0b20028070 con 0x7f0b24006560 2024-05-20T20:18:12.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.382+0000 7f0b36dec740 1 -- 172.21.15.28:0/3525308936 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 msgr2=0x56111bd6c440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:12.384 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.382+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd6c440 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f0b24096190 tx=0x7f0b1800acc0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.385 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.385+0000 7f0b2a000640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b140b2db0 0x7f0b140b5270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.386 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.385+0000 7f0b31e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b140b2db0 0x7f0b140b5270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.386+0000 7f0b2a000640 1 -- 172.21.15.28:0/857558965 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 8586+0+0 (secure 0 0 0) 0x7f0b20021410 con 0x7f0b24006560 2024-05-20T20:18:12.388 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.387+0000 7f0b36dec740 1 -- 172.21.15.28:0/3525308936 shutdown_connections 2024-05-20T20:18:12.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.388+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd6c440 unknown :-1 s=CLOSED pgs=590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.388+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 0x56111bd69b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.388+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3525308936 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd5f250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.388+0000 7f0b36dec740 1 -- 172.21.15.28:0/3525308936 >> 172.21.15.28:0/3525308936 conn(0x56111bd5b380 msgr2=0x56111bd5b760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:12.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.388+0000 7f0b36dec740 1 -- 172.21.15.28:0/3525308936 shutdown_connections 2024-05-20T20:18:12.389 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.389+0000 7f0b36dec740 1 -- 172.21.15.28:0/3525308936 wait complete. 2024-05-20T20:18:12.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.389+0000 7f0b36dec740 1 -- start start 2024-05-20T20:18:12.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.389+0000 7f0b36dec740 1 Processor -- start 2024-05-20T20:18:12.390 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.390+0000 7f0b31e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b140b2db0 0x7f0b140b5270 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c051a50 tx=0x7f0b2c079040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.392+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5ee70 0x56111bd69980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.392+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 0x56111bd66c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.392+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd6a050 0x56111bd671b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.392+0000 7f0b36dec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56111b9a5790 con 0x56111bd5f790 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.392+0000 7f0b36dec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56111bc67bc0 con 0x56111bd5ee70 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.392+0000 7f0b36dec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56111bd6cc80 con 0x56111bd6a050 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.392+0000 7f0b36dec740 10 client.0 _ll_register_callbacks cb 0x56111bc45400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:12.395 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.393+0000 7f0b31400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5ee70 0x56111bd69980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.393+0000 7f0b31400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5ee70 0x56111bd69980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:35654/0 (socket says 172.21.15.28:35654) 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.393+0000 7f0b31400640 1 -- 172.21.15.28:0/191619833 learned_addr learned my addr 172.21.15.28:0/191619833 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.393+0000 7f0b30a00640 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 0x56111bd66c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.393+0000 7f0b31e00640 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd6a050 0x56111bd671b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.395+0000 7f0b31400640 1 -- 172.21.15.28:0/191619833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd6a050 msgr2=0x56111bd671b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.395+0000 7f0b31400640 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd6a050 0x56111bd671b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.395+0000 7f0b31400640 1 -- 172.21.15.28:0/191619833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 msgr2=0x56111bd66c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.395+0000 7f0b31400640 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 0x56111bd66c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.396 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.395+0000 7f0b31400640 1 -- 172.21.15.28:0/191619833 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56111b9d4210 con 0x56111bd5ee70 2024-05-20T20:18:12.397 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.397+0000 7f0b31400640 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5ee70 0x56111bd69980 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f0b200b8e90 tx=0x7f0b200b8d60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b71f4d1d73852c0e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:12.397 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.397+0000 7f0b12a00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:12.398 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.397+0000 7f0b12a00640 1 -- 172.21.15.28:0/191619833 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b20025030 con 0x56111bd5ee70 2024-05-20T20:18:12.398 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.397+0000 7f0b12a00640 1 -- 172.21.15.28:0/191619833 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 8586+0+0 (secure 0 0 0) 0x7f0b200efdc0 con 0x56111bd5ee70 2024-05-20T20:18:12.398 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.398+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0b180113f0 con 0x56111bd5ee70 2024-05-20T20:18:12.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.398+0000 7f0b12a00640 1 -- 172.21.15.28:0/191619833 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0b200bd540 con 0x56111bd5ee70 2024-05-20T20:18:12.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.398+0000 7f0b36dec740 10 client.27908 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:12.399 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.398+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56111baa0620 con 0x56111bd5ee70 2024-05-20T20:18:12.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.400+0000 7f0b12a00640 1 -- 172.21.15.28:0/191619833 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0b180113f0 con 0x56111bd5ee70 2024-05-20T20:18:12.400 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.400+0000 7f0b12a00640 1 -- 172.21.15.28:0/191619833 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0b20020050 con 0x56111bd5ee70 2024-05-20T20:18:12.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.403+0000 7f0b36dec740 10 client.27908 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:12.403 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.403+0000 7f0b36dec740 10 client.27908 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:12.404 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.403+0000 7f0b36dec740 20 client.27908 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:12.404 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.403+0000 7f0b36dec740 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bd79880 0x56111bd7bc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.404 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.403+0000 7f0b36dec740 4 client.27908 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:12.404 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.403+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56111bcd1ae0 con 0x56111bd79880 2024-05-20T20:18:12.404 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.404+0000 7f0b30a00640 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bd79880 0x56111bd7bc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.405+0000 7f0b30a00640 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bd79880 0x56111bd7bc70 crc :-1 s=READY pgs=255 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d4fc388ba1db25c8 server_cookie=9205ee303e9c0cd9 in_seq=0 out_seq=0 2024-05-20T20:18:12.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.406+0000 7f0b12a00640 10 client.27908 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:12.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.406+0000 7f0b12a00640 1 -- 172.21.15.28:0/191619833 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56111bcd1ae0 con 0x56111bd79880 2024-05-20T20:18:12.406 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.406+0000 7f0b12a00640 10 client.27908 handle_command_reply: tid=0 2024-05-20T20:18:12.407 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.406+0000 7f0b36dec740 1 client.27908 shutdown 2024-05-20T20:18:12.409 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.408+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5ee70 msgr2=0x56111bd69980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:12.409 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.408+0000 7f0b36dec740 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5ee70 0x56111bd69980 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f0b200b8e90 tx=0x7f0b200b8d60 comp rx=0 tx=0).stop 2024-05-20T20:18:12.409 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 shutdown_connections 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bd79880 0x56111bd7bc70 crc :-1 s=READY pgs=255 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd6a050 0x56111bd671b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5f790 0x56111bd66c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 --2- 172.21.15.28:0/191619833 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5ee70 0x56111bd69980 unknown :-1 s=CLOSED pgs=591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b12a00640 0 client.27908 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 >> 172.21.15.28:0/191619833 conn(0x56111bd5b380 msgr2=0x56111bd61100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 shutdown_connections 2024-05-20T20:18:12.410 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.409+0000 7f0b36dec740 1 -- 172.21.15.28:0/191619833 wait complete. 2024-05-20T20:18:12.411 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.410+0000 7f0b36dec740 20 client.27908 trim_cache size 0 max 16384 2024-05-20T20:18:12.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.458+0000 7f0b36dec740 1 Processor -- start 2024-05-20T20:18:12.458 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.458+0000 7f0b36dec740 1 -- start start 2024-05-20T20:18:12.459 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.459+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd75d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.460 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.459+0000 7f0b31400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd75d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.462 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.459+0000 7f0b31400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd75d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:43990/0 (socket says 172.21.15.28:43990) 2024-05-20T20:18:12.462 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.460+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5f790 0x56111bd814a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.462 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.460+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd6a050 0x56111bd819e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.460+0000 7f0b36dec740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0b2413fa10 con 0x56111bd6a050 2024-05-20T20:18:12.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.460+0000 7f0b36dec740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0b24141550 con 0x56111bd5f790 2024-05-20T20:18:12.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.460+0000 7f0b36dec740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0b241400e0 con 0x56111bd5ee70 2024-05-20T20:18:12.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.462+0000 7f0b31400640 1 -- 172.21.15.28:0/1926976042 learned_addr learned my addr 172.21.15.28:0/1926976042 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:12.463 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.463+0000 7f0b30a00640 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5f790 0x56111bd814a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.463+0000 7f0b31400640 1 -- 172.21.15.28:0/1926976042 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5f790 msgr2=0x56111bd814a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:12.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.463+0000 7f0b31400640 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5f790 0x56111bd814a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.463+0000 7f0b31400640 1 -- 172.21.15.28:0/1926976042 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd6a050 msgr2=0x56111bd819e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:12.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.463+0000 7f0b31400640 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd6a050 0x56111bd819e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.464 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.463+0000 7f0b31400640 1 -- 172.21.15.28:0/1926976042 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56111bd7c230 con 0x56111bd5ee70 2024-05-20T20:18:12.465 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.464+0000 7f0b31400640 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd75d10 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7f0b200b6400 tx=0x7f0b2000bf90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d904707c00db66f0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:12.474 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.473+0000 7f0b1dc00640 1 -- 172.21.15.28:0/1926976042 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b2007d070 con 0x56111bd5ee70 2024-05-20T20:18:12.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.473+0000 7f0b1dc00640 1 -- 172.21.15.28:0/1926976042 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0b20021ac0 con 0x56111bd5ee70 2024-05-20T20:18:12.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.473+0000 7f0b1dc00640 1 -- 172.21.15.28:0/1926976042 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b20025070 con 0x56111bd5ee70 2024-05-20T20:18:12.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.476+0000 7f0b36dec740 1 -- 172.21.15.28:0/1926976042 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 msgr2=0x56111bd75d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:12.476 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.476+0000 7f0b36dec740 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd75d10 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7f0b200b6400 tx=0x7f0b2000bf90 comp rx=0 tx=0).stop 2024-05-20T20:18:12.477 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.477+0000 7f0b36dec740 1 -- 172.21.15.28:0/1926976042 shutdown_connections 2024-05-20T20:18:12.477 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.477+0000 7f0b36dec740 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd5f790 0x56111bd814a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.477+0000 7f0b36dec740 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd6a050 0x56111bd819e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.477+0000 7f0b36dec740 1 --2- 172.21.15.28:0/1926976042 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5ee70 0x56111bd75d10 unknown :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.477+0000 7f0b36dec740 1 -- 172.21.15.28:0/1926976042 >> 172.21.15.28:0/1926976042 conn(0x56111bd5b380 msgr2=0x56111bd66db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:12.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.477+0000 7f0b36dec740 1 -- 172.21.15.28:0/1926976042 shutdown_connections 2024-05-20T20:18:12.478 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.478+0000 7f0b36dec740 1 -- 172.21.15.28:0/1926976042 wait complete. 2024-05-20T20:18:12.482 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.481+0000 7f0b36dec740 1 -- start start 2024-05-20T20:18:12.482 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.481+0000 7f0b36dec740 1 Processor -- start 2024-05-20T20:18:12.483 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.483+0000 7f0b36dec740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5ee70 0x56111bd6fc50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.484 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.483+0000 7f0b31400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5ee70 0x56111bd6fc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.484+0000 7f0b31400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5ee70 0x56111bd6fc50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:50842/0 (socket says 172.21.15.28:50842) 2024-05-20T20:18:12.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.484+0000 7f0b31400640 1 -- 172.21.15.28:0/3219175421 learned_addr learned my addr 172.21.15.28:0/3219175421 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:12.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.484+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5f790 0x56111bd70190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.485 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.484+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd706d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.485+0000 7f0b30a00640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5f790 0x56111bd70190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.485+0000 7f0b31e00640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd706d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.485+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0b24140e80 con 0x56111bd5ee70 2024-05-20T20:18:12.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.485+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0b241407b0 con 0x56111bd6a050 2024-05-20T20:18:12.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.485+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0b2413da30 con 0x56111bd5f790 2024-05-20T20:18:12.486 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.485+0000 7f0b36dec740 10 client.0 _ll_register_callbacks cb 0x7f0b200b8f60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:12.487 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.487+0000 7f0b31e00640 1 -- 172.21.15.28:0/3219175421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5f790 msgr2=0x56111bd70190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:12.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.487+0000 7f0b31e00640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5f790 0x56111bd70190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.487+0000 7f0b31e00640 1 -- 172.21.15.28:0/3219175421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5ee70 msgr2=0x56111bd6fc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:12.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.487+0000 7f0b31e00640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5ee70 0x56111bd6fc50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:12.488 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.488+0000 7f0b31e00640 1 -- 172.21.15.28:0/3219175421 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56111bde3380 con 0x56111bd6a050 2024-05-20T20:18:12.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.488+0000 7f0b31400640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5ee70 0x56111bd6fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:12.489 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.488+0000 7f0b30a00640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5f790 0x56111bd70190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:18:12.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.491+0000 7f0b31e00640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd706d0 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c02e130 tx=0x7f0b2c0689c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1c5ea212a04af13f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:12.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.491+0000 7f0b12a00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:12.492 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.492+0000 7f0b12a00640 1 -- 172.21.15.28:0/3219175421 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0b2c0741b0 con 0x56111bd6a050 2024-05-20T20:18:12.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.492+0000 7f0b12a00640 1 -- 172.21.15.28:0/3219175421 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 8586+0+0 (secure 0 0 0) 0x7f0b2c06e110 con 0x56111bd6a050 2024-05-20T20:18:12.493 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.492+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0b20021ac0 con 0x56111bd6a050 2024-05-20T20:18:12.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.493+0000 7f0b12a00640 1 -- 172.21.15.28:0/3219175421 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0b2c0cd050 con 0x56111bd6a050 2024-05-20T20:18:12.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.493+0000 7f0b36dec740 10 client.27914 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:12.494 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.493+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56111bde4630 con 0x56111bd6a050 2024-05-20T20:18:12.495 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.494+0000 7f0b12a00640 1 -- 172.21.15.28:0/3219175421 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0b20021ac0 con 0x56111bd6a050 2024-05-20T20:18:12.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.495+0000 7f0b12a00640 1 -- 172.21.15.28:0/3219175421 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0b2c06c520 con 0x56111bd6a050 2024-05-20T20:18:12.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.495+0000 7f0b36dec740 10 client.27914 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:12.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.495+0000 7f0b36dec740 10 client.27914 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:12.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.495+0000 7f0b36dec740 20 client.27914 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:12.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.495+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bde4db0 0x56111bde71d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:12.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.495+0000 7f0b36dec740 4 client.27914 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:18:12.496 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.495+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x56111bccb2c0 con 0x56111bde4db0 2024-05-20T20:18:12.502 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.501+0000 7f0b31400640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bde4db0 0x56111bde71d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:12.505 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.505+0000 7f0b31400640 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bde4db0 0x56111bde71d0 crc :-1 s=READY pgs=256 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=81d056842280dedb server_cookie=3893d664817223e2 in_seq=0 out_seq=0 2024-05-20T20:18:12.506 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:12.505+0000 7f0b12a00640 10 client.27914 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:13.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.217+0000 7f0b12a00640 1 -- 172.21.15.28:0/3219175421 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x56111bccb2c0 con 0x56111bde4db0 2024-05-20T20:18:13.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.217+0000 7f0b12a00640 10 client.27914 handle_command_reply: tid=0 2024-05-20T20:18:13.218 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.218+0000 7f0b36dec740 1 client.27914 shutdown 2024-05-20T20:18:13.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.218+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 msgr2=0x56111bd706d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:13.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.218+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd706d0 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c02e130 tx=0x7f0b2c0689c0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.218+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 shutdown_connections 2024-05-20T20:18:13.219 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.218+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56111bde4db0 0x56111bde71d0 crc :-1 s=READY pgs=256 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b12a00640 0 client.27914 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56111bd6a050 0x56111bd706d0 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56111bd5f790 0x56111bd70190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b36dec740 1 --2- 172.21.15.28:0/3219175421 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56111bd5ee70 0x56111bd6fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 >> 172.21.15.28:0/3219175421 conn(0x56111bd5b380 msgr2=0x56111bd74d00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 shutdown_connections 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b36dec740 1 -- 172.21.15.28:0/3219175421 wait complete. 2024-05-20T20:18:13.222 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.221+0000 7f0b36dec740 20 client.27914 trim_cache size 0 max 16384 2024-05-20T20:18:13.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:18:13.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:18:13.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:18:13.223 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.222+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b140b2db0 msgr2=0x7f0b140b5270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.222+0000 7f0b33e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b140b2db0 0x7f0b140b5270 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f0b2c051a50 tx=0x7f0b2c079040 comp rx=0 tx=0).stop 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.222+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006560 msgr2=0x7f0b2412e6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.222+0000 7f0b33e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006560 0x7f0b2412e6c0 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f0b2000b4a0 tx=0x7f0b2000b960 comp rx=0 tx=0).stop 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 shutdown_connections 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0b24006e80 0x7f0b2412ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0b140b2db0 0x7f0b140b5270 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0b240a2a30 0x7f0b24132f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.224 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 --2- 172.21.15.28:0/857558965 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0b24006560 0x7f0b2412e6c0 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:13.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 >> 172.21.15.28:0/857558965 conn(0x7f0b2400a770 msgr2=0x7f0b240a3ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:13.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 shutdown_connections 2024-05-20T20:18:13.225 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:13.223+0000 7f0b33e00640 1 -- 172.21.15.28:0/857558965 wait complete. 2024-05-20T20:18:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:13 smithi154 ceph-mon[36314]: pgmap v393: 97 pgs: 19 active+clean+snaptrim, 29 active+clean+snaptrim_wait, 49 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 204 MiB/s wr, 1.90k op/s 2024-05-20T20:18:13.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:13 smithi033 ceph-mon[36179]: pgmap v393: 97 pgs: 19 active+clean+snaptrim, 29 active+clean+snaptrim_wait, 49 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 204 MiB/s wr, 1.90k op/s 2024-05-20T20:18:13.798 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:13 smithi028 ceph-mon[31178]: pgmap v393: 97 pgs: 19 active+clean+snaptrim, 29 active+clean+snaptrim_wait, 49 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 204 MiB/s wr, 1.90k op/s 2024-05-20T20:18:15.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:15 smithi154 ceph-mon[36314]: pgmap v394: 97 pgs: 17 active+clean+snaptrim, 25 active+clean+snaptrim_wait, 55 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 176 MiB/s wr, 1.65k op/s 2024-05-20T20:18:15.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:15 smithi033 ceph-mon[36179]: pgmap v394: 97 pgs: 17 active+clean+snaptrim, 25 active+clean+snaptrim_wait, 55 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 176 MiB/s wr, 1.65k op/s 2024-05-20T20:18:15.642 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:15 smithi028 ceph-mon[31178]: pgmap v394: 97 pgs: 17 active+clean+snaptrim, 25 active+clean+snaptrim_wait, 55 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 176 MiB/s wr, 1.65k op/s 2024-05-20T20:18:17.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:17 smithi033 ceph-mon[36179]: pgmap v395: 97 pgs: 16 active+clean+snaptrim, 25 active+clean+snaptrim_wait, 56 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 158 MiB/s wr, 1.47k op/s 2024-05-20T20:18:17.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:17 smithi028 ceph-mon[31178]: pgmap v395: 97 pgs: 16 active+clean+snaptrim, 25 active+clean+snaptrim_wait, 56 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 158 MiB/s wr, 1.47k op/s 2024-05-20T20:18:17.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:17 smithi154 ceph-mon[36314]: pgmap v395: 97 pgs: 16 active+clean+snaptrim, 25 active+clean+snaptrim_wait, 56 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 158 MiB/s wr, 1.47k op/s 2024-05-20T20:18:19.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:19 smithi028 ceph-mon[31178]: pgmap v396: 97 pgs: 14 active+clean+snaptrim, 19 active+clean+snaptrim_wait, 64 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 183 MiB/s wr, 1.62k op/s 2024-05-20T20:18:19.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.856+0000 7f1e0b400700 1 -- 172.21.15.28:0/1179170028 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1e040092e0 con 0x7f1e0c06b790 2024-05-20T20:18:19.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.857+0000 7f1e13800700 1 -- 172.21.15.28:0/1179170028 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c06b790 msgr2=0x7f1e0c06bbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:19.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.857+0000 7f1e13800700 1 --2- 172.21.15.28:0/1179170028 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c06b790 0x7f1e0c06bbf0 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f1e04009f90 tx=0x7f1e0400b330 comp rx=0 tx=0).stop 2024-05-20T20:18:19.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.857+0000 7f1e13800700 1 -- 172.21.15.28:0/1179170028 shutdown_connections 2024-05-20T20:18:19.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.857+0000 7f1e13800700 1 --2- 172.21.15.28:0/1179170028 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1e0c105f80 0x7f1e0c106410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.860 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.857+0000 7f1e13800700 1 --2- 172.21.15.28:0/1179170028 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1e0c108470 0x7f1e0c06b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.857+0000 7f1e13800700 1 --2- 172.21.15.28:0/1179170028 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c06b790 0x7f1e0c06bbf0 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.857+0000 7f1e13800700 1 -- 172.21.15.28:0/1179170028 >> 172.21.15.28:0/1179170028 conn(0x7f1e0c0793f0 msgr2=0x7f1e0c079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:19.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.865+0000 7f1e13800700 1 -- 172.21.15.28:0/1179170028 shutdown_connections 2024-05-20T20:18:19.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.865+0000 7f1e13800700 1 -- 172.21.15.28:0/1179170028 wait complete. 2024-05-20T20:18:19.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.870+0000 7f1e13800700 1 Processor -- start 2024-05-20T20:18:19.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.870+0000 7f1e13800700 1 -- start start 2024-05-20T20:18:19.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.872+0000 7f1e13800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1e0c06b790 0x7f1e0c1a0200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.872+0000 7f1e13800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1e0c105f80 0x7f1e0c1a0740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.873+0000 7f1e13800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c108470 0x7f1e0c1a4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.873+0000 7f1e10e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1e0c06b790 0x7f1e0c1a0200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.873+0000 7f1e10e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1e0c06b790 0x7f1e0c1a0200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39216/0 (socket says 172.21.15.28:39216) 2024-05-20T20:18:19.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e13800700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1e0c119d70 con 0x7f1e0c105f80 2024-05-20T20:18:19.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e13800700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1e0c119bf0 con 0x7f1e0c108470 2024-05-20T20:18:19.875 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e13800700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1e0c119ef0 con 0x7f1e0c06b790 2024-05-20T20:18:19.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:19 smithi154 ceph-mon[36314]: pgmap v396: 97 pgs: 14 active+clean+snaptrim, 19 active+clean+snaptrim_wait, 64 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 183 MiB/s wr, 1.62k op/s 2024-05-20T20:18:19.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c108470 0x7f1e0c1a4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c108470 0x7f1e0c1a4ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:47902/0 (socket says 172.21.15.28:47902) 2024-05-20T20:18:19.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 -- 172.21.15.28:0/2424978167 learned_addr learned my addr 172.21.15.28:0/2424978167 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:19.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 -- 172.21.15.28:0/2424978167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1e0c06b790 msgr2=0x7f1e0c1a0200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:19.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:19 smithi033 ceph-mon[36179]: pgmap v396: 97 pgs: 14 active+clean+snaptrim, 19 active+clean+snaptrim_wait, 64 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 183 MiB/s wr, 1.62k op/s 2024-05-20T20:18:19.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1e0c06b790 0x7f1e0c1a0200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 -- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1e0c105f80 msgr2=0x7f1e0c1a0740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:19.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1e0c105f80 0x7f1e0c1a0740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.874+0000 7f1e11800700 1 -- 172.21.15.28:0/2424978167 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1e0c1a5210 con 0x7f1e0c108470 2024-05-20T20:18:19.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.875+0000 7f1e11800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c108470 0x7f1e0c1a4ad0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f1df400e970 tx=0x7f1df400ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:19.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.875+0000 7f1e09600700 1 -- 172.21.15.28:0/2424978167 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1df4004a30 con 0x7f1e0c108470 2024-05-20T20:18:19.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.877+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1e0c1a5500 con 0x7f1e0c108470 2024-05-20T20:18:19.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.877+0000 7f1e09600700 1 -- 172.21.15.28:0/2424978167 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1df400cc70 con 0x7f1e0c108470 2024-05-20T20:18:19.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.877+0000 7f1e09600700 1 -- 172.21.15.28:0/2424978167 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1df4004a30 con 0x7f1e0c108470 2024-05-20T20:18:19.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.877+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1e0c1acd80 con 0x7f1e0c108470 2024-05-20T20:18:19.890 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.888+0000 7f1e09600700 1 -- 172.21.15.28:0/2424978167 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1df400c7c0 con 0x7f1e0c108470 2024-05-20T20:18:19.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.891+0000 7f1e1e0bfb80 1 Processor -- start 2024-05-20T20:18:19.895 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.895+0000 7f1e09600700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1df00b2c70 0x7f1df00b5130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.895+0000 7f1e09600700 1 -- 172.21.15.28:0/2424978167 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(94..94 src has 1..94) ==== 8586+0+0 (secure 0 0 0) 0x7f1df40e2000 con 0x7f1e0c108470 2024-05-20T20:18:19.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.921+0000 7f1e1e0bfb80 1 -- start start 2024-05-20T20:18:19.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.922+0000 7f1e0be00700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1df00b2c70 0x7f1df00b5130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.927+0000 7f1e0be00700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1df00b2c70 0x7f1df00b5130 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f1e0c06bef0 tx=0x7f1e04006000 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:19.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.930+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98610 0x563994f989f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.930+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994f92b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.930+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f93140 0x563994f935c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.931+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563994ef0e40 con 0x563994f98610 2024-05-20T20:18:19.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.931+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563994ee09e0 con 0x563994f98f30 2024-05-20T20:18:19.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.931+0000 7f1e11800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f93140 0x563994f935c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.933+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563994ee4a20 con 0x563994f93140 2024-05-20T20:18:19.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.931+0000 7f1e0be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994f92b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.933+0000 7f1e11800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f93140 0x563994f935c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39226/0 (socket says 172.21.15.28:39226) 2024-05-20T20:18:19.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.933+0000 7f1e11800700 1 -- 172.21.15.28:0/3782915409 learned_addr learned my addr 172.21.15.28:0/3782915409 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:19.935 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.934+0000 7f1e11800700 1 -- 172.21.15.28:0/3782915409 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 msgr2=0x563994f92b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.934+0000 7f1e11800700 1 --2- 172.21.15.28:0/3782915409 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994f92b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.934+0000 7f1e11800700 1 -- 172.21.15.28:0/3782915409 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98610 msgr2=0x563994f989f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.934+0000 7f1e11800700 1 --2- 172.21.15.28:0/3782915409 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98610 0x563994f989f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.934+0000 7f1e11800700 1 -- 172.21.15.28:0/3782915409 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563994ea9420 con 0x563994f93140 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.935+0000 7f1e11800700 1 --2- 172.21.15.28:0/3782915409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f93140 0x563994f935c0 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7f1df40a9230 tx=0x7f1df40a8d20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d877c803ddfd4bb0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.935+0000 7f1e02000700 1 -- 172.21.15.28:0/3782915409 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1df40dfe60 con 0x563994f93140 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.936+0000 7f1e02000700 1 -- 172.21.15.28:0/3782915409 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1df40a99a0 con 0x563994f93140 2024-05-20T20:18:19.936 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.936+0000 7f1e02000700 1 -- 172.21.15.28:0/3782915409 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1df40a5530 con 0x563994f93140 2024-05-20T20:18:19.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.937+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3782915409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f93140 msgr2=0x563994f935c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:19.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.937+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3782915409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f93140 0x563994f935c0 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7f1df40a9230 tx=0x7f1df40a8d20 comp rx=0 tx=0).stop 2024-05-20T20:18:19.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.939+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3782915409 shutdown_connections 2024-05-20T20:18:19.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.939+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3782915409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f93140 0x563994f935c0 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.939+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3782915409 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98610 0x563994f989f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.939+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3782915409 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994f92b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.939+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3782915409 >> 172.21.15.28:0/3782915409 conn(0x563994ec0be0 msgr2=0x563994f8d690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:19.944 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.943+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3782915409 shutdown_connections 2024-05-20T20:18:19.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.944+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3782915409 wait complete. 2024-05-20T20:18:19.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.945+0000 7f1e1e0bfb80 1 -- start start 2024-05-20T20:18:19.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.945+0000 7f1e1e0bfb80 1 Processor -- start 2024-05-20T20:18:19.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.958+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f93140 0x563994f9edd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.959+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f98f30 0x563994f9c0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.960+0000 7f1e10e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f93140 0x563994f9edd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.960+0000 7f1e10e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f93140 0x563994f9edd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:47920/0 (socket says 172.21.15.28:47920) 2024-05-20T20:18:19.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.960+0000 7f1e0be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f98f30 0x563994f9c0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.960+0000 7f1e0be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f98f30 0x563994f9c0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39234/0 (socket says 172.21.15.28:39234) 2024-05-20T20:18:19.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.961+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 0x563994f9ca90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.967 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.964+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x563994de6fa0 con 0x563994f9c610 2024-05-20T20:18:19.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.964+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x563994e81760 con 0x563994f93140 2024-05-20T20:18:19.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.964+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x563994f9f9e0 con 0x563994f98f30 2024-05-20T20:18:19.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.965+0000 7f1e10e00700 1 -- 172.21.15.28:0/3074293803 learned_addr learned my addr 172.21.15.28:0/3074293803 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:19.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.965+0000 7f1e10e00700 1 -- 172.21.15.28:0/3074293803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f98f30 msgr2=0x563994f9c0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:19.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.965+0000 7f1e10e00700 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f98f30 0x563994f9c0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.968 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.966+0000 7f1e10e00700 1 -- 172.21.15.28:0/3074293803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 msgr2=0x563994f9ca90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:19.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.966+0000 7f1e10e00700 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 0x563994f9ca90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.967+0000 7f1e1e0bfb80 10 client.0 _ll_register_callbacks cb 0x563994e8ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:19.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.970+0000 7f1e10e00700 1 -- 172.21.15.28:0/3074293803 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563994e308d0 con 0x563994f93140 2024-05-20T20:18:19.971 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.970+0000 7f1e0be00700 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f98f30 0x563994f9c0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello state changed while learned_addr, mark_down or replacing must be happened just now 2024-05-20T20:18:19.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.970+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1df40a99a0 con 0x563994f93140 2024-05-20T20:18:19.972 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.970+0000 7f1dfaa00700 10 client.27926 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:19.975 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.974+0000 7f1e10e00700 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f93140 0x563994f9edd0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f1dfc00bd00 tx=0x7f1dfc0106b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c794b6434a445593 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:19.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.975+0000 7f1dfaa00700 1 -- 172.21.15.28:0/3074293803 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1dfc004230 con 0x563994f93140 2024-05-20T20:18:19.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.975+0000 7f1dfaa00700 1 -- 172.21.15.28:0/3074293803 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 8586+0+0 (secure 0 0 0) 0x7f1dfc004030 con 0x563994f93140 2024-05-20T20:18:19.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.975+0000 7f1dfaa00700 1 -- 172.21.15.28:0/3074293803 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1dfc004230 con 0x563994f93140 2024-05-20T20:18:19.976 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.975+0000 7f1e1e0bfb80 10 client.27926 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:19.977 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.976+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({fsmap=29}) -- 0x563994eb1060 con 0x563994f93140 2024-05-20T20:18:19.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.977+0000 7f1dfaa00700 1 -- 172.21.15.28:0/3074293803 <== mon.2 v2:172.21.15.33:3300/0 4 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1dfc0043d0 con 0x563994f93140 2024-05-20T20:18:19.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.977+0000 7f1e1e0bfb80 10 client.27926 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:19.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.977+0000 7f1e1e0bfb80 10 client.27926 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:18:19.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.977+0000 7f1e1e0bfb80 20 client.27926 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:19.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.977+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563994fac910 0x563994faedd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:19.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.977+0000 7f1e1e0bfb80 4 client.27926 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:19.978 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.977+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x563994ef2d90 con 0x563994fac910 2024-05-20T20:18:19.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.978+0000 7f1e0be00700 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563994fac910 0x563994faedd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:19.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.981+0000 7f1e0be00700 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563994fac910 0x563994faedd0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=23bc7579c497c993 server_cookie=53372a6948a79af4 in_seq=0 out_seq=0 2024-05-20T20:18:19.988 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.987+0000 7f1dfaa00700 10 client.27926 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:18:19.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.990+0000 7f1dfaa00700 1 -- 172.21.15.28:0/3074293803 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x563994ef2d90 con 0x563994fac910 2024-05-20T20:18:19.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.990+0000 7f1dfaa00700 10 client.27926 handle_command_reply: tid=0 2024-05-20T20:18:19.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.990+0000 7f1e1e0bfb80 1 client.27926 shutdown 2024-05-20T20:18:19.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f93140 msgr2=0x563994f9edd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:19.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f93140 0x563994f9edd0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f1dfc00bd00 tx=0x7f1dfc0106b0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 shutdown_connections 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x563994fac910 0x563994faedd0 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 0x563994f9ca90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f93140 0x563994f9edd0 secure :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f1dfc00bd00 tx=0x7f1dfc0106b0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/3074293803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f98f30 0x563994f9c0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 >> 172.21.15.28:0/3074293803 conn(0x563994ec0be0 msgr2=0x563994f8cd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.991+0000 7f1dfaa00700 0 client.27926 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.992+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 shutdown_connections 2024-05-20T20:18:19.993 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.992+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/3074293803 wait complete. 2024-05-20T20:18:19.994 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:19.992+0000 7f1e1e0bfb80 20 client.27926 trim_cache size 0 max 16384 2024-05-20T20:18:20.052 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.044+0000 7f1e1e0bfb80 1 Processor -- start 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.045+0000 7f1e1e0bfb80 1 -- start start 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.045+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98f30 0x563994f92b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.045+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f9c610 0x563994faed00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.046+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f8d4f0 0x563994f99a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.046+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1de800d720 con 0x563994f98f30 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.046+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1de800f260 con 0x563994f8d4f0 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.046+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1de800ddf0 con 0x563994f9c610 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.046+0000 7f1e0be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f9c610 0x563994faed00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.046+0000 7f1e0be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f9c610 0x563994faed00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39236/0 (socket says 172.21.15.28:39236) 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.046+0000 7f1e0be00700 1 -- 172.21.15.28:0/2903670875 learned_addr learned my addr 172.21.15.28:0/2903670875 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.047+0000 7f1e11800700 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f8d4f0 0x563994f99a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:20.053 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.047+0000 7f1e0be00700 1 -- 172.21.15.28:0/2903670875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f8d4f0 msgr2=0x563994f99a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:20.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.047+0000 7f1e0be00700 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f8d4f0 0x563994f99a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.047+0000 7f1e0be00700 1 -- 172.21.15.28:0/2903670875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98f30 msgr2=0x563994f92b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:20.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.047+0000 7f1e0be00700 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98f30 0x563994f92b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.047+0000 7f1e0be00700 1 -- 172.21.15.28:0/2903670875 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x563994f9a640 con 0x563994f9c610 2024-05-20T20:18:20.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.048+0000 7f1e0be00700 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f9c610 0x563994faed00 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f1e04000f80 tx=0x7f1e0401b000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=aaf4eee8c110c541 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:20.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.048+0000 7f1e01600700 1 -- 172.21.15.28:0/2903670875 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1e04037070 con 0x563994f9c610 2024-05-20T20:18:20.054 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.049+0000 7f1e01600700 1 -- 172.21.15.28:0/2903670875 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1e04021040 con 0x563994f9c610 2024-05-20T20:18:20.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.050+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/2903670875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f9c610 msgr2=0x563994faed00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:20.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.050+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f9c610 0x563994faed00 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f1e04000f80 tx=0x7f1e0401b000 comp rx=0 tx=0).stop 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.054+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/2903670875 shutdown_connections 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.054+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f8d4f0 0x563994f99a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.054+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f98f30 0x563994f92b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.055+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/2903670875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f9c610 0x563994faed00 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.055+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/2903670875 >> 172.21.15.28:0/2903670875 conn(0x563994ec0be0 msgr2=0x563994f99310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.055+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/2903670875 shutdown_connections 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.055+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/2903670875 wait complete. 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.056+0000 7f1e1e0bfb80 1 -- start start 2024-05-20T20:18:20.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.056+0000 7f1e1e0bfb80 1 Processor -- start 2024-05-20T20:18:20.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.059+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f8d4f0 0x563994fb4380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:20.060 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.059+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994fb48c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:20.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.059+0000 7f1e1e0bfb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 0x563994fb4e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:20.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e0be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994fb48c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:20.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1de800eb90 con 0x563994f9c610 2024-05-20T20:18:20.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1de800e4c0 con 0x563994f98f30 2024-05-20T20:18:20.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e0be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994fb48c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:47940/0 (socket says 172.21.15.28:47940) 2024-05-20T20:18:20.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e0be00700 1 -- 172.21.15.28:0/1687692554 learned_addr learned my addr 172.21.15.28:0/1687692554 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:20.061 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e1e0bfb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1de800b680 con 0x563994f8d4f0 2024-05-20T20:18:20.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e1e0bfb80 10 client.0 _ll_register_callbacks cb 0x563994e8ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:20.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.060+0000 7f1e10e00700 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f8d4f0 0x563994fb4380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:20.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.061+0000 7f1e0be00700 1 -- 172.21.15.28:0/1687692554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f8d4f0 msgr2=0x563994fb4380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:20.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.062+0000 7f1e0be00700 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f8d4f0 0x563994fb4380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.062 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.062+0000 7f1e0be00700 1 -- 172.21.15.28:0/1687692554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 msgr2=0x563994fb4e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.062+0000 7f1e0be00700 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 0x563994fb4e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.062+0000 7f1e0be00700 1 -- 172.21.15.28:0/1687692554 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x563995016740 con 0x563994f98f30 2024-05-20T20:18:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.062+0000 7f1e0be00700 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994fb48c0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7f1e04003fd0 tx=0x7f1e040093d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7fc9c9b3b28d355b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:20.063 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.062+0000 7f1dfaa00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:20.066 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.066+0000 7f1dfaa00700 1 -- 172.21.15.28:0/1687692554 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1e04005920 con 0x563994f98f30 2024-05-20T20:18:20.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.067+0000 7f1dfaa00700 1 -- 172.21.15.28:0/1687692554 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(94..94 src has 1..94) ==== 8586+0+0 (secure 0 0 0) 0x7f1e04029660 con 0x563994f98f30 2024-05-20T20:18:20.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.067+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1e04021040 con 0x563994f98f30 2024-05-20T20:18:20.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.067+0000 7f1dfaa00700 1 -- 172.21.15.28:0/1687692554 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1e04029860 con 0x563994f98f30 2024-05-20T20:18:20.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.077+0000 7f1e1e0bfb80 10 client.27932 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:20.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.077+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x563995017a20 con 0x563994f98f30 2024-05-20T20:18:20.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.079+0000 7f1dfaa00700 1 -- 172.21.15.28:0/1687692554 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1e04021040 con 0x563994f98f30 2024-05-20T20:18:20.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.079+0000 7f1dfaa00700 1 -- 172.21.15.28:0/1687692554 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1e04011070 con 0x563994f98f30 2024-05-20T20:18:20.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.080+0000 7f1e1e0bfb80 10 client.27932 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:20.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.080+0000 7f1e1e0bfb80 10 client.27932 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:18:20.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.080+0000 7f1e1e0bfb80 20 client.27932 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:20.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.080+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5639950180b0 0x56399501a4d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:20.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.080+0000 7f1e1e0bfb80 4 client.27932 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:18:20.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.080+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x563994eec570 con 0x5639950180b0 2024-05-20T20:18:20.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.081+0000 7f1e10e00700 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5639950180b0 0x56399501a4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:20.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.082+0000 7f1e10e00700 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5639950180b0 0x56399501a4d0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=60caf036d396fec8 server_cookie=7bfc782b087a84da in_seq=0 out_seq=0 2024-05-20T20:18:20.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.082+0000 7f1dfaa00700 10 client.27932 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:18:20.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.082+0000 7f1dfaa00700 1 -- 172.21.15.28:0/1687692554 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x563994eec570 con 0x5639950180b0 2024-05-20T20:18:20.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.082+0000 7f1dfaa00700 10 client.27932 handle_command_reply: tid=0 2024-05-20T20:18:20.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.082+0000 7f1e1e0bfb80 1 client.27932 shutdown 2024-05-20T20:18:20.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.082+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 msgr2=0x563994fb48c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:20.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.082+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994fb48c0 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7f1e04003fd0 tx=0x7f1e040093d0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 shutdown_connections 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x5639950180b0 0x56399501a4d0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x563994f9c610 0x563994fb4e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x563994f8d4f0 0x563994fb4380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 --2- 172.21.15.28:0/1687692554 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x563994f98f30 0x563994fb48c0 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1dfaa00700 0 client.27932 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 >> 172.21.15.28:0/1687692554 conn(0x563994ec0be0 msgr2=0x563994f8cd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 shutdown_connections 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 1 -- 172.21.15.28:0/1687692554 wait complete. 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.083+0000 7f1e1e0bfb80 20 client.27932 trim_cache size 0 max 16384 2024-05-20T20:18:20.085 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:18:20.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1df00b2c70 msgr2=0x7f1df00b5130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:20.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1df00b2c70 0x7f1df00b5130 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f1e0c06bef0 tx=0x7f1e04006000 comp rx=0 tx=0).stop 2024-05-20T20:18:20.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c108470 msgr2=0x7f1e0c1a4ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:20.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c108470 0x7f1e0c1a4ad0 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f1df400e970 tx=0x7f1df400ec80 comp rx=0 tx=0).stop 2024-05-20T20:18:20.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 shutdown_connections 2024-05-20T20:18:20.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1df00b2c70 0x7f1df00b5130 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1e0c108470 0x7f1e0c1a4ad0 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1e0c06b790 0x7f1e0c1a0200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 --2- 172.21.15.28:0/2424978167 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1e0c105f80 0x7f1e0c1a0740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:20.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.085+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 >> 172.21.15.28:0/2424978167 conn(0x7f1e0c0793f0 msgr2=0x7f1e0c104f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:20.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.087+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 shutdown_connections 2024-05-20T20:18:20.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:20.087+0000 7f1e13800700 1 -- 172.21.15.28:0/2424978167 wait complete. 2024-05-20T20:18:21.677 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:21 smithi154 ceph-mon[36314]: pgmap v397: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 227 MiB/s wr, 1.86k op/s 2024-05-20T20:18:21.808 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:21 smithi028 ceph-mon[31178]: pgmap v397: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 227 MiB/s wr, 1.86k op/s 2024-05-20T20:18:21.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:21 smithi033 ceph-mon[36179]: pgmap v397: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 227 MiB/s wr, 1.86k op/s 2024-05-20T20:18:22.799 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:18:23.692 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:23 smithi028 ceph-mon[31178]: pgmap v398: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 218 MiB/s wr, 1.69k op/s 2024-05-20T20:18:23.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:23 smithi154 ceph-mon[36314]: pgmap v398: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 218 MiB/s wr, 1.69k op/s 2024-05-20T20:18:23.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:23 smithi033 ceph-mon[36179]: pgmap v398: 97 pgs: 97 active+clean; 57 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 218 MiB/s wr, 1.69k op/s 2024-05-20T20:18:25.134 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:25.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:25 smithi028 ceph-mon[31178]: pgmap v399: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 307 MiB/s wr, 2.25k op/s 2024-05-20T20:18:25.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:25 smithi154 ceph-mon[36314]: pgmap v399: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 307 MiB/s wr, 2.25k op/s 2024-05-20T20:18:25.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:25 smithi033 ceph-mon[36179]: pgmap v399: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 307 MiB/s wr, 2.25k op/s 2024-05-20T20:18:26.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.064+0000 7f7c6c400700 1 -- 172.21.15.28:0/1612631363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64108470 msgr2=0x7f7c6406b250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.065+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1612631363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64108470 0x7f7c6406b250 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f7c5c00b0a0 tx=0x7f7c5c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.065+0000 7f7c6c400700 1 -- 172.21.15.28:0/1612631363 shutdown_connections 2024-05-20T20:18:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.065+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1612631363 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 0x7f7c64106410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.065+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1612631363 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64108470 0x7f7c6406b250 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.065+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1612631363 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6406b790 0x7f7c6406bbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.065+0000 7f7c6c400700 1 -- 172.21.15.28:0/1612631363 >> 172.21.15.28:0/1612631363 conn(0x7f7c640793f0 msgr2=0x7f7c64079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:26.069 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.069+0000 7f7c6c400700 1 -- 172.21.15.28:0/1612631363 shutdown_connections 2024-05-20T20:18:26.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.069+0000 7f7c6c400700 1 -- 172.21.15.28:0/1612631363 wait complete. 2024-05-20T20:18:26.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.070+0000 7f7c6c400700 1 Processor -- start 2024-05-20T20:18:26.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.071+0000 7f7c6c400700 1 -- start start 2024-05-20T20:18:26.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.072+0000 7f7c6c400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6406b790 0x7f7c641166a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.073+0000 7f7c6c400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 0x7f7c64116be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.073+0000 7f7c6c400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64108470 0x7f7c6410f7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.074+0000 7f7c6c400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f7c64119d00 con 0x7f7c64105f80 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.074+0000 7f7c6c400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f7c64119b80 con 0x7f7c6406b790 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.074+0000 7f7c6c400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f7c64119e80 con 0x7f7c64108470 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.075+0000 7f7c69000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 0x7f7c64116be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.076+0000 7f7c69000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 0x7f7c64116be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58342/0 (socket says 172.21.15.28:58342) 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.076+0000 7f7c69000700 1 -- 172.21.15.28:0/1540348864 learned_addr learned my addr 172.21.15.28:0/1540348864 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.078+0000 7f7c69000700 1 -- 172.21.15.28:0/1540348864 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64108470 msgr2=0x7f7c6410f7b0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.078+0000 7f7c69000700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64108470 0x7f7c6410f7b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.078+0000 7f7c69000700 1 -- 172.21.15.28:0/1540348864 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6406b790 msgr2=0x7f7c641166a0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-05-20T20:18:26.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.078+0000 7f7c69000700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6406b790 0x7f7c641166a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.078+0000 7f7c69000700 1 -- 172.21.15.28:0/1540348864 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7c641100b0 con 0x7f7c64105f80 2024-05-20T20:18:26.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.082+0000 7f7c69000700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 0x7f7c64116be0 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f7c5000d950 tx=0x7f7c5000dd10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.086 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.084+0000 7f7c62000700 1 -- 172.21.15.28:0/1540348864 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c50016070 con 0x7f7c64105f80 2024-05-20T20:18:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.084+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7c641103a0 con 0x7f7c64105f80 2024-05-20T20:18:26.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.085+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7c641143f0 con 0x7f7c64105f80 2024-05-20T20:18:26.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.088+0000 7f7c62000700 1 -- 172.21.15.28:0/1540348864 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f7c50010460 con 0x7f7c64105f80 2024-05-20T20:18:26.096 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.092+0000 7f7c62000700 1 -- 172.21.15.28:0/1540348864 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7c5000f620 con 0x7f7c64105f80 2024-05-20T20:18:26.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.096+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7c44005330 con 0x7f7c64105f80 2024-05-20T20:18:26.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.127+0000 7f7c62000700 1 -- 172.21.15.28:0/1540348864 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f7c50019070 con 0x7f7c64105f80 2024-05-20T20:18:26.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.138+0000 7f7c62000700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c4c0b2af0 0x7f7c4c0b4fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.142+0000 7f7c62000700 1 -- 172.21.15.28:0/1540348864 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f7c500e25c0 con 0x7f7c64105f80 2024-05-20T20:18:26.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.146+0000 7f7c69a00700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c4c0b2af0 0x7f7c4c0b4fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.168+0000 7f7c69a00700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c4c0b2af0 0x7f7c4c0b4fb0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f7c5c000f80 tx=0x7f7c5c00b7f0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.180 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.177+0000 7f7c62000700 1 -- 172.21.15.28:0/1540348864 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f7c500a9f30 con 0x7f7c64105f80 2024-05-20T20:18:26.728 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.726+0000 7f3b5da00640 1 -- 172.21.15.28:0/906465376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b5810dd10 msgr2=0x7f3b5810e170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:26.729 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.726+0000 7f3b5da00640 1 --2- 172.21.15.28:0/906465376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b5810dd10 0x7f3b5810e170 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f3b44009900 tx=0x7f3b4402a420 comp rx=0 tx=0).stop 2024-05-20T20:18:26.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.728+0000 7f3b5da00640 1 -- 172.21.15.28:0/906465376 shutdown_connections 2024-05-20T20:18:26.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.728+0000 7f3b5da00640 1 --2- 172.21.15.28:0/906465376 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b5810e8a0 0x7f3b5807ad60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.728+0000 7f3b5da00640 1 --2- 172.21.15.28:0/906465376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b5810dd10 0x7f3b5810e170 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.728+0000 7f3b5da00640 1 --2- 172.21.15.28:0/906465376 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b5810f3c0 0x7f3b5810f7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.728+0000 7f3b5da00640 1 -- 172.21.15.28:0/906465376 >> 172.21.15.28:0/906465376 conn(0x7f3b5806db20 msgr2=0x7f3b5806df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:26.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.728+0000 7f3b5da00640 1 -- 172.21.15.28:0/906465376 shutdown_connections 2024-05-20T20:18:26.730 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.728+0000 7f3b5da00640 1 -- 172.21.15.28:0/906465376 wait complete. 2024-05-20T20:18:26.731 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.730+0000 7f3b5da00640 1 Processor -- start 2024-05-20T20:18:26.733 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.732+0000 7f3b5da00640 1 -- start start 2024-05-20T20:18:26.735 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.734+0000 7f3b5da00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b5810dd10 0x7f3b5807ace0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.734+0000 7f3b5da00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b5810f3c0 0x7f3b58074f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.734+0000 7f3b5da00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b580755c0 0x7f3b581c53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.735+0000 7f3b56000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b5810f3c0 0x7f3b58074f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.735+0000 7f3b56000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b5810f3c0 0x7f3b58074f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:47976/0 (socket says 172.21.15.28:47976) 2024-05-20T20:18:26.736 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.735+0000 7f3b56a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b5810dd10 0x7f3b5807ace0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.735+0000 7f3b56a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b5810dd10 0x7f3b5807ace0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58348/0 (socket says 172.21.15.28:58348) 2024-05-20T20:18:26.737 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.737+0000 7f3b56000640 1 -- 172.21.15.28:0/2056194133 learned_addr learned my addr 172.21.15.28:0/2056194133 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:26.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.737+0000 7f3b57400640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b580755c0 0x7f3b581c53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.737+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3b5807f280 con 0x7f3b5810dd10 2024-05-20T20:18:26.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.739+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3b5807f100 con 0x7f3b5810f3c0 2024-05-20T20:18:26.739 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.739+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3b5807f400 con 0x7f3b580755c0 2024-05-20T20:18:26.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.739+0000 7f3b57400640 1 -- 172.21.15.28:0/2056194133 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b5810f3c0 msgr2=0x7f3b58074f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:26.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.739+0000 7f3b57400640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b5810f3c0 0x7f3b58074f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.739+0000 7f3b57400640 1 -- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b5810dd10 msgr2=0x7f3b5807ace0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:26.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.739+0000 7f3b57400640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b5810dd10 0x7f3b5807ace0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.741 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.739+0000 7f3b57400640 1 -- 172.21.15.28:0/2056194133 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3b581c5910 con 0x7f3b580755c0 2024-05-20T20:18:26.742 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.741+0000 7f3b57400640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b580755c0 0x7f3b581c53d0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f3b4800d650 tx=0x7f3b4800db10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.743+0000 7f3b4f400640 1 -- 172.21.15.28:0/2056194133 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b48017070 con 0x7f3b580755c0 2024-05-20T20:18:26.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.743+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3b581c5ae0 con 0x7f3b580755c0 2024-05-20T20:18:26.744 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.743+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3b581c5f50 con 0x7f3b580755c0 2024-05-20T20:18:26.751 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.750+0000 7f3b4f400640 1 -- 172.21.15.28:0/2056194133 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3b48009820 con 0x7f3b580755c0 2024-05-20T20:18:26.751 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.750+0000 7f3b4f400640 1 -- 172.21.15.28:0/2056194133 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b48017070 con 0x7f3b580755c0 2024-05-20T20:18:26.799 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.775+0000 7f3b60849740 1 Processor -- start 2024-05-20T20:18:26.799 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.779+0000 7f3b4f400640 1 -- 172.21.15.28:0/2056194133 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3b48004a60 con 0x7f3b580755c0 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.794+0000 7f3b60849740 1 -- start start 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.794+0000 7f3b60849740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c155070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.794+0000 7f3b60849740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.794+0000 7f3b60849740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c15fe70 0x55a64c162260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.794+0000 7f3b60849740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a64c0c5b90 con 0x55a64c15fe70 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.794+0000 7f3b60849740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a64c0b5730 con 0x55a64c1555b0 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.795+0000 7f3b60849740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a64c0b9770 con 0x55a64c154c90 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.795+0000 7f3b56000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c155070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.795+0000 7f3b56000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c155070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39278/0 (socket says 172.21.15.28:39278) 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.796+0000 7f3b56000640 1 -- 172.21.15.28:0/2627587429 learned_addr learned my addr 172.21.15.28:0/2627587429 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.797+0000 7f3b56a00640 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.800 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.797+0000 7f3b56000640 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c15fe70 0x55a64c162260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.797+0000 7f3b56a00640 1 -- 172.21.15.28:0/2627587429 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 msgr2=0x55a64c155070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.797+0000 7f3b56a00640 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c155070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.797+0000 7f3b56a00640 1 -- 172.21.15.28:0/2627587429 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c15fe70 msgr2=0x55a64c162260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.797+0000 7f3b56a00640 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c15fe70 0x55a64c162260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.797+0000 7f3b56a00640 1 -- 172.21.15.28:0/2627587429 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a64c04e8b0 con 0x55a64c1555b0 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.798+0000 7f3b56000640 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c15fe70 0x55a64c162260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.799+0000 7f3b56a00640 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f930 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7f3b50009ee0 tx=0x7f3b5000c7f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=80ea4966274ab943 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.801 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.799+0000 7f3b43e00640 1 -- 172.21.15.28:0/2627587429 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b5003c070 con 0x55a64c1555b0 2024-05-20T20:18:26.802 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.800+0000 7f3b43e00640 1 -- 172.21.15.28:0/2627587429 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3b500378d0 con 0x55a64c1555b0 2024-05-20T20:18:26.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.801+0000 7f3b60849740 1 -- 172.21.15.28:0/2627587429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 msgr2=0x55a64c15f930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:26.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.803+0000 7f3b43e00640 1 -- 172.21.15.28:0/2627587429 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b5003c070 con 0x55a64c1555b0 2024-05-20T20:18:26.803 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.803+0000 7f3b60849740 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f930 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7f3b50009ee0 tx=0x7f3b5000c7f0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.805 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.805+0000 7f3b4f400640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b3c0b2d50 0x7f3b3c0b5210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.805+0000 7f3b4f400640 1 -- 172.21.15.28:0/2056194133 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f3b480e1a30 con 0x7f3b580755c0 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.805+0000 7f3b60849740 1 -- 172.21.15.28:0/2627587429 shutdown_connections 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.806+0000 7f3b60849740 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f930 unknown :-1 s=CLOSED pgs=597 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.806+0000 7f3b60849740 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c15fe70 0x55a64c162260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.806+0000 7f3b60849740 1 --2- 172.21.15.28:0/2627587429 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c155070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.806+0000 7f3b60849740 1 -- 172.21.15.28:0/2627587429 >> 172.21.15.28:0/2627587429 conn(0x55a64c1511a0 msgr2=0x55a64c151580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.806+0000 7f3b56000640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b3c0b2d50 0x7f3b3c0b5210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.807 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.807+0000 7f3b60849740 1 -- 172.21.15.28:0/2627587429 shutdown_connections 2024-05-20T20:18:26.811 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.807+0000 7f3b60849740 1 -- 172.21.15.28:0/2627587429 wait complete. 2024-05-20T20:18:26.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.813+0000 7f3b60849740 1 -- start start 2024-05-20T20:18:26.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.814+0000 7f3b60849740 1 Processor -- start 2024-05-20T20:18:26.815 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.815+0000 7f3b56000640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b3c0b2d50 0x7f3b3c0b5210 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f3b44000c00 tx=0x7f3b4402a920 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.819 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.819+0000 7f3b60849740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c15f720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.821 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.819+0000 7f3b60849740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c15ca10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.819+0000 7f3b56a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c15f720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.822 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.819+0000 7f3b56a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c15f720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:47990/0 (socket says 172.21.15.28:47990) 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.819+0000 7f3b60849740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c15fe70 0x55a64c15cf50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.820+0000 7f3b57400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c15fe70 0x55a64c15cf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.820+0000 7f3b57400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c15fe70 0x55a64c15cf50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:39288/0 (socket says 172.21.15.28:39288) 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.820+0000 7f3b57400640 1 -- 172.21.15.28:0/1804576511 learned_addr learned my addr 172.21.15.28:0/1804576511 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.820+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a64bd9b790 con 0x55a64c1555b0 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.820+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a64c05dbc0 con 0x55a64c154c90 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.820+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a64c1629e0 con 0x55a64c15fe70 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.821+0000 7f3b60849740 10 client.0 _ll_register_callbacks cb 0x55a64c03b400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.821+0000 7f3b57400640 1 -- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 msgr2=0x55a64c15f720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.821+0000 7f3b57400640 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c15f720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.822+0000 7f3b57400640 1 -- 172.21.15.28:0/1804576511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 msgr2=0x55a64c15ca10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.822+0000 7f3b57400640 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c15ca10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.822+0000 7f3b57400640 1 -- 172.21.15.28:0/1804576511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a64bdca210 con 0x55a64c15fe70 2024-05-20T20:18:26.823 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.822+0000 7f3b56a00640 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c15f720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:26.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.825+0000 7f3b57400640 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c15fe70 0x55a64c15cf50 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f3b480a9a10 tx=0x7f3b480a9960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a335b727aa42dc85 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.825+0000 7f3b40c00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:26.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.825+0000 7f3b40c00640 1 -- 172.21.15.28:0/1804576511 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b48020040 con 0x55a64c15fe70 2024-05-20T20:18:26.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.825+0000 7f3b40c00640 1 -- 172.21.15.28:0/1804576511 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f3b480a8f70 con 0x55a64c15fe70 2024-05-20T20:18:26.826 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.826+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3b500378d0 con 0x55a64c15fe70 2024-05-20T20:18:26.829 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.826+0000 7f3b40c00640 1 -- 172.21.15.28:0/1804576511 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3b480df5a0 con 0x55a64c15fe70 2024-05-20T20:18:26.832 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.832+0000 7f3b60849740 10 client.27883 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:26.832 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.832+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a64be96620 con 0x55a64c15fe70 2024-05-20T20:18:26.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.832+0000 7f3b40c00640 1 -- 172.21.15.28:0/1804576511 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3b500378d0 con 0x55a64c15fe70 2024-05-20T20:18:26.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.832+0000 7f3b40c00640 1 -- 172.21.15.28:0/1804576511 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3b480c4070 con 0x55a64c15fe70 2024-05-20T20:18:26.833 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.833+0000 7f3b60849740 10 client.27883 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:26.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.833+0000 7f3b60849740 10 client.27883 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:26.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.833+0000 7f3b60849740 20 client.27883 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:26.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.833+0000 7f3b60849740 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c16f580 0x55a64c171a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.833+0000 7f3b60849740 4 client.27883 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:26.834 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.833+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a64c0c7ae0 con 0x55a64c16f580 2024-05-20T20:18:26.839 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.839+0000 7f3b56a00640 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c16f580 0x55a64c171a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.840 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.840+0000 7f3b56a00640 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c16f580 0x55a64c171a40 crc :-1 s=READY pgs=257 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c953bb8f0b31c356 server_cookie=86ab7e94af5aea1b in_seq=0 out_seq=0 2024-05-20T20:18:26.841 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.840+0000 7f3b40c00640 10 client.27883 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:26.841 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.841+0000 7f3b40c00640 1 -- 172.21.15.28:0/1804576511 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a64c0c7ae0 con 0x55a64c16f580 2024-05-20T20:18:26.842 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.841+0000 7f3b40c00640 10 client.27883 handle_command_reply: tid=0 2024-05-20T20:18:26.842 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.841+0000 7f3b60849740 1 client.27883 shutdown 2024-05-20T20:18:26.843 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c15fe70 msgr2=0x55a64c15cf50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:26.843 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c15fe70 0x55a64c15cf50 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f3b480a9a10 tx=0x7f3b480a9960 comp rx=0 tx=0).stop 2024-05-20T20:18:26.843 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 shutdown_connections 2024-05-20T20:18:26.843 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c16f580 0x55a64c171a40 crc :-1 s=READY pgs=257 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.843 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c15fe70 0x55a64c15cf50 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.843 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c15ca10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 --2- 172.21.15.28:0/1804576511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c15f720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.842+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 >> 172.21.15.28:0/1804576511 conn(0x55a64c1511a0 msgr2=0x55a64c156ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:26.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.843+0000 7f3b40c00640 0 client.27883 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:26.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.843+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 shutdown_connections 2024-05-20T20:18:26.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.843+0000 7f3b60849740 1 -- 172.21.15.28:0/1804576511 wait complete. 2024-05-20T20:18:26.844 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.844+0000 7f3b60849740 20 client.27883 trim_cache size 0 max 16384 2024-05-20T20:18:26.897 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 Processor -- start 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 -- start start 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1511a0 0x55a64c16bb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c15c9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3b3800e950 con 0x55a64c1511a0 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3b38010490 con 0x55a64c1555b0 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.896+0000 7f3b60849740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3b3800f020 con 0x55a64c154c90 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.897+0000 7f3b57400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.897+0000 7f3b57400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41556/0 (socket says 172.21.15.28:41556) 2024-05-20T20:18:26.898 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.897+0000 7f3b57400640 1 -- 172.21.15.28:0/3668085424 learned_addr learned my addr 172.21.15.28:0/3668085424 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:26.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.898+0000 7f3b57400640 1 -- 172.21.15.28:0/3668085424 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 msgr2=0x55a64c15c9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:26.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.898+0000 7f3b57400640 1 --2- 172.21.15.28:0/3668085424 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c15c9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.898+0000 7f3b57400640 1 -- 172.21.15.28:0/3668085424 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1511a0 msgr2=0x55a64c16bb20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:26.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.898+0000 7f3b57400640 1 --2- 172.21.15.28:0/3668085424 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1511a0 0x55a64c16bb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.898+0000 7f3b57400640 1 -- 172.21.15.28:0/3668085424 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a64c15d7f0 con 0x55a64c1555b0 2024-05-20T20:18:26.901 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.900+0000 7f3b57400640 1 --2- 172.21.15.28:0/3668085424 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f250 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7f3b480a6f80 tx=0x7f3b4801e000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b871e68d19e99624 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.904 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.900+0000 7f3b43400640 1 -- 172.21.15.28:0/3668085424 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b48089070 con 0x55a64c1555b0 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.901+0000 7f3b43400640 1 -- 172.21.15.28:0/3668085424 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3b480b2b70 con 0x55a64c1555b0 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.901+0000 7f3b43400640 1 -- 172.21.15.28:0/3668085424 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b48020030 con 0x55a64c1555b0 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.902+0000 7f3b60849740 1 -- 172.21.15.28:0/3668085424 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 msgr2=0x55a64c15f250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.902+0000 7f3b60849740 1 --2- 172.21.15.28:0/3668085424 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f250 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7f3b480a6f80 tx=0x7f3b4801e000 comp rx=0 tx=0).stop 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.903+0000 7f3b60849740 1 -- 172.21.15.28:0/3668085424 shutdown_connections 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.903+0000 7f3b60849740 1 --2- 172.21.15.28:0/3668085424 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c1555b0 0x55a64c15f250 unknown :-1 s=CLOSED pgs=598 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.903+0000 7f3b60849740 1 --2- 172.21.15.28:0/3668085424 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c154c90 0x55a64c15c9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.903+0000 7f3b60849740 1 --2- 172.21.15.28:0/3668085424 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1511a0 0x55a64c16bb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.903+0000 7f3b60849740 1 -- 172.21.15.28:0/3668085424 >> 172.21.15.28:0/3668085424 conn(0x55a64c15d0b0 msgr2=0x55a64c172b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.903+0000 7f3b60849740 1 -- 172.21.15.28:0/3668085424 shutdown_connections 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.903+0000 7f3b60849740 1 -- 172.21.15.28:0/3668085424 wait complete. 2024-05-20T20:18:26.905 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.904+0000 7f3b60849740 1 -- start start 2024-05-20T20:18:26.906 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.904+0000 7f3b60849740 1 Processor -- start 2024-05-20T20:18:26.906 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b60849740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c1511a0 0x55a64c168280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b60849740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c164ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b60849740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c165220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b60849740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3b3800fdc0 con 0x55a64c1555b0 2024-05-20T20:18:26.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b60849740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3b3800f6f0 con 0x55a64c154c90 2024-05-20T20:18:26.908 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b60849740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3b3800c950 con 0x55a64c1511a0 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b56000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c164ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b56000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c164ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41562/0 (socket says 172.21.15.28:41562) 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.905+0000 7f3b56000640 1 -- 172.21.15.28:0/2376820164 learned_addr learned my addr 172.21.15.28:0/2376820164 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.906+0000 7f3b60849740 10 client.0 _ll_register_callbacks cb 0x7f3b480a5bd0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.906+0000 7f3b57400640 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c165220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.906+0000 7f3b57400640 1 -- 172.21.15.28:0/2376820164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c1511a0 msgr2=0x55a64c168280 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.906+0000 7f3b57400640 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c1511a0 0x55a64c168280 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.906+0000 7f3b57400640 1 -- 172.21.15.28:0/2376820164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 msgr2=0x55a64c164ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.906+0000 7f3b57400640 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c164ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:26.909 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.906+0000 7f3b57400640 1 -- 172.21.15.28:0/2376820164 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a64c1657e0 con 0x55a64c1555b0 2024-05-20T20:18:26.910 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.910+0000 7f3b57400640 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c165220 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f3b480a8bc0 tx=0x7f3b480a5d00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=831c664cee9109a4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:26.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.911+0000 7f3b40c00640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:18:26.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.911+0000 7f3b40c00640 1 -- 172.21.15.28:0/2376820164 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3b48089020 con 0x55a64c1555b0 2024-05-20T20:18:26.912 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.911+0000 7f3b40c00640 1 -- 172.21.15.28:0/2376820164 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f3b48005430 con 0x55a64c1555b0 2024-05-20T20:18:26.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.912+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f3b480b2b70 con 0x55a64c1555b0 2024-05-20T20:18:26.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.913+0000 7f3b40c00640 1 -- 172.21.15.28:0/2376820164 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f3b48020070 con 0x55a64c1555b0 2024-05-20T20:18:26.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b60849740 10 client.17361 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:26.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a64c1da510 con 0x55a64c1555b0 2024-05-20T20:18:26.918 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b40c00640 1 -- 172.21.15.28:0/2376820164 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3b480b2b70 con 0x55a64c1555b0 2024-05-20T20:18:26.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b40c00640 1 -- 172.21.15.28:0/2376820164 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f3b480c4070 con 0x55a64c1555b0 2024-05-20T20:18:26.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b60849740 10 client.17361 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:26.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b60849740 10 client.17361 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:26.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b60849740 20 client.17361 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:26.919 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.914+0000 7f3b60849740 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c15fe70 0x55a64c1dc940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:26.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.925+0000 7f3b60849740 4 client.17361 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:18:26.925 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.925+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55a64c0c12c0 con 0x55a64c15fe70 2024-05-20T20:18:26.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.925+0000 7f3b56a00640 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c15fe70 0x55a64c1dc940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:26.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.925+0000 7f3b56a00640 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c15fe70 0x55a64c1dc940 crc :-1 s=READY pgs=258 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d8ed22b18d504acf server_cookie=4d64cd40f42f7e5 in_seq=0 out_seq=0 2024-05-20T20:18:26.926 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:26.925+0000 7f3b40c00640 10 client.17361 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:26.992 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.988+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f7c44005620 con 0x7f7c64105f80 2024-05-20T20:18:26.994 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:26 smithi028 ceph-mon[31178]: osdmap e95: 12 total, 12 up, 12 in 2024-05-20T20:18:26.994 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:26 smithi028 ceph-mon[31178]: pgmap v401: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 277 MiB/s wr, 1.93k op/s 2024-05-20T20:18:27.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:26.999+0000 7f7c62000700 1 -- 172.21.15.28:0/1540348864 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f7c500a9830 con 0x7f7c64105f80 2024-05-20T20:18:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:18:27.005 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:18:27.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.006+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c4c0b2af0 msgr2=0x7f7c4c0b4fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:27.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c4c0b2af0 0x7f7c4c0b4fb0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f7c5c000f80 tx=0x7f7c5c00b7f0 comp rx=0 tx=0).stop 2024-05-20T20:18:27.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 msgr2=0x7f7c64116be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:27.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 0x7f7c64116be0 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f7c5000d950 tx=0x7f7c5000dd10 comp rx=0 tx=0).stop 2024-05-20T20:18:27.008 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 shutdown_connections 2024-05-20T20:18:27.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f7c64108470 0x7f7c6410f7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:27.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f7c4c0b2af0 0x7f7c4c0b4fb0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:27.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f7c6406b790 0x7f7c641166a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:27.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 --2- 172.21.15.28:0/1540348864 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f7c64105f80 0x7f7c64116be0 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:27.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.007+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 >> 172.21.15.28:0/1540348864 conn(0x7f7c640793f0 msgr2=0x7f7c64104f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:27.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.011+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 shutdown_connections 2024-05-20T20:18:27.013 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:27.011+0000 7f7c6c400700 1 -- 172.21.15.28:0/1540348864 wait complete. 2024-05-20T20:18:27.018 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:18:27.126 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:26 smithi154 ceph-mon[36314]: osdmap e95: 12 total, 12 up, 12 in 2024-05-20T20:18:27.127 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:26 smithi154 ceph-mon[36314]: pgmap v401: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 277 MiB/s wr, 1.93k op/s 2024-05-20T20:18:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:26 smithi033 ceph-mon[36179]: osdmap e95: 12 total, 12 up, 12 in 2024-05-20T20:18:27.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:26 smithi033 ceph-mon[36179]: pgmap v401: 97 pgs: 97 active+clean; 58 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 277 MiB/s wr, 1.93k op/s 2024-05-20T20:18:28.056 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:27 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1540348864' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:28.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:27 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1540348864' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:28.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:27 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1540348864' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:28.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.757+0000 7f3b40c00640 1 -- 172.21.15.28:0/2376820164 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55a64c0c12c0 con 0x55a64c15fe70 2024-05-20T20:18:28.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.757+0000 7f3b40c00640 10 client.17361 handle_command_reply: tid=0 2024-05-20T20:18:28.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.757+0000 7f3b60849740 1 client.17361 shutdown 2024-05-20T20:18:28.758 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.757+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 msgr2=0x55a64c165220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.757+0000 7f3b60849740 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c165220 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f3b480a8bc0 tx=0x7f3b480a5d00 comp rx=0 tx=0).stop 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 shutdown_connections 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55a64c15fe70 0x55a64c1dc940 crc :-1 s=READY pgs=258 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a64c154c90 0x55a64c164ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b40c00640 0 client.17361 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a64c1555b0 0x55a64c165220 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 --2- 172.21.15.28:0/2376820164 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a64c1511a0 0x55a64c168280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.759 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 >> 172.21.15.28:0/2376820164 conn(0x55a64c15d0b0 msgr2=0x55a64c16ab10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:28.760 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:18:28.760 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:18:28.760 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 shutdown_connections 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 1 -- 172.21.15.28:0/2376820164 wait complete. 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.758+0000 7f3b60849740 20 client.17361 trim_cache size 0 max 16384 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.759+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b3c0b2d50 msgr2=0x7f3b3c0b5210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.759+0000 7f3b5da00640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b3c0b2d50 0x7f3b3c0b5210 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f3b44000c00 tx=0x7f3b4402a920 comp rx=0 tx=0).stop 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.759+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b580755c0 msgr2=0x7f3b581c53d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.759+0000 7f3b5da00640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b580755c0 0x7f3b581c53d0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f3b4800d650 tx=0x7f3b4800db10 comp rx=0 tx=0).stop 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 shutdown_connections 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3b5810f3c0 0x7f3b58074f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3b3c0b2d50 0x7f3b3c0b5210 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3b580755c0 0x7f3b581c53d0 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.761 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 --2- 172.21.15.28:0/2056194133 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3b5810dd10 0x7f3b5807ace0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:28.762 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 >> 172.21.15.28:0/2056194133 conn(0x7f3b5806db20 msgr2=0x7f3b5807c2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:28.762 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 shutdown_connections 2024-05-20T20:18:28.762 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:28.760+0000 7f3b5da00640 1 -- 172.21.15.28:0/2056194133 wait complete. 2024-05-20T20:18:29.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:28 smithi154 ceph-mon[36314]: pgmap v402: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 266 MiB/s wr, 1.85k op/s 2024-05-20T20:18:29.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:28 smithi033 ceph-mon[36179]: pgmap v402: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 266 MiB/s wr, 1.85k op/s 2024-05-20T20:18:29.166 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:28 smithi028 ceph-mon[31178]: pgmap v402: 97 pgs: 97 active+clean; 59 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 266 MiB/s wr, 1.85k op/s 2024-05-20T20:18:31.110 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db / --timeout 90 --expiration 137.9 --await-for 45.0' 2024-05-20T20:18:31.111 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db / --timeout 90 --expiration 137.9 --await-for 45.0 2024-05-20T20:18:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:31 smithi154 ceph-mon[36314]: pgmap v403: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 190 KiB/s rd, 343 MiB/s wr, 2.53k op/s 2024-05-20T20:18:31.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:31 smithi033 ceph-mon[36179]: pgmap v403: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 190 KiB/s rd, 343 MiB/s wr, 2.53k op/s 2024-05-20T20:18:31.807 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:31 smithi028 ceph-mon[31178]: pgmap v403: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 190 KiB/s rd, 343 MiB/s wr, 2.53k op/s 2024-05-20T20:18:33.015 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.355+0000 7f4b67200700 1 -- 172.21.15.28:0/3872132415 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010a540 msgr2=0x7f4b6010a9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.356+0000 7f4b67200700 1 --2- 172.21.15.28:0/3872132415 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010a540 0x7f4b6010a9c0 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7f4b4800ba80 tx=0x7f4b4800bd90 comp rx=0 tx=0).stop 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.357+0000 7f4b67200700 1 -- 172.21.15.28:0/3872132415 shutdown_connections 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.357+0000 7f4b67200700 1 --2- 172.21.15.28:0/3872132415 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4b6010af00 0x7f4b60116e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.357+0000 7f4b67200700 1 --2- 172.21.15.28:0/3872132415 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4b6010ef90 0x7f4b6010f370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.357+0000 7f4b67200700 1 --2- 172.21.15.28:0/3872132415 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010a540 0x7f4b6010a9c0 unknown :-1 s=CLOSED pgs=599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.357+0000 7f4b67200700 1 -- 172.21.15.28:0/3872132415 >> 172.21.15.28:0/3872132415 conn(0x7f4b6006ea90 msgr2=0x7f4b6006eea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:33.358 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.357+0000 7f4b67200700 1 -- 172.21.15.28:0/3872132415 shutdown_connections 2024-05-20T20:18:33.359 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.358+0000 7f4b67200700 1 -- 172.21.15.28:0/3872132415 wait complete. 2024-05-20T20:18:33.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.360+0000 7f4b67200700 1 Processor -- start 2024-05-20T20:18:33.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.362+0000 7f4b67200700 1 -- start start 2024-05-20T20:18:33.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.363+0000 7f4b67200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4b6010a540 0x7f4b601a4610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.363+0000 7f4b67200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4b6010af00 0x7f4b601a4b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.363+0000 7f4b67200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010ef90 0x7f4b601a8e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.364+0000 7f4b67200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4b60119e30 con 0x7f4b6010a540 2024-05-20T20:18:33.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.364+0000 7f4b65200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010ef90 0x7f4b601a8e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.364+0000 7f4b65200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010ef90 0x7f4b601a8e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41576/0 (socket says 172.21.15.28:41576) 2024-05-20T20:18:33.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.364+0000 7f4b65200700 1 -- 172.21.15.28:0/3187456688 learned_addr learned my addr 172.21.15.28:0/3187456688 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:33.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.364+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4b60119cb0 con 0x7f4b6010ef90 2024-05-20T20:18:33.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.364+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4b60119fb0 con 0x7f4b6010af00 2024-05-20T20:18:33.365 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.366+0000 7f4b57400700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4b6010af00 0x7f4b601a4b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.366+0000 7f4b65200700 1 -- 172.21.15.28:0/3187456688 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4b6010af00 msgr2=0x7f4b601a4b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.366 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.366+0000 7f4b65200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4b6010af00 0x7f4b601a4b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.366+0000 7f4b65200700 1 -- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4b6010a540 msgr2=0x7f4b601a4610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.367+0000 7f4b65200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4b6010a540 0x7f4b601a4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.367 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.367+0000 7f4b65200700 1 -- 172.21.15.28:0/3187456688 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4b601a95d0 con 0x7f4b6010ef90 2024-05-20T20:18:33.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.368+0000 7f4b57400700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4b6010af00 0x7f4b601a4b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:33.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.368+0000 7f4b65200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010ef90 0x7f4b601a8e90 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f4b5800c970 tx=0x7f4b5800cd30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:33.368 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.369+0000 7f4b5d600700 1 -- 172.21.15.28:0/3187456688 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b58004230 con 0x7f4b6010ef90 2024-05-20T20:18:33.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.369+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4b601a98c0 con 0x7f4b6010ef90 2024-05-20T20:18:33.370 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.369+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4b601b1140 con 0x7f4b6010ef90 2024-05-20T20:18:33.375 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.373+0000 7f4b5d600700 1 -- 172.21.15.28:0/3187456688 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4b58007db0 con 0x7f4b6010ef90 2024-05-20T20:18:33.376 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.373+0000 7f4b5d600700 1 -- 172.21.15.28:0/3187456688 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b58004230 con 0x7f4b6010ef90 2024-05-20T20:18:33.380 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.379+0000 7f4b71bd5b80 1 Processor -- start 2024-05-20T20:18:33.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.382+0000 7f4b5d600700 1 -- 172.21.15.28:0/3187456688 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4b58003bb0 con 0x7f4b6010ef90 2024-05-20T20:18:33.383 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.382+0000 7f4b71bd5b80 1 -- start start 2024-05-20T20:18:33.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.386+0000 7f4b5d600700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4b440b2ca0 0x7f4b440b5160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.386+0000 7f4b5d600700 1 -- 172.21.15.28:0/3187456688 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f4b580e21d0 con 0x7f4b6010ef90 2024-05-20T20:18:33.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.386+0000 7f4b57400700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4b440b2ca0 0x7f4b440b5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.387+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c648a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.387+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c642a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.387+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c643090 0x56202c643520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.387+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56202c598e40 con 0x56202c648660 2024-05-20T20:18:33.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.387+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56202c5889e0 con 0x56202c643090 2024-05-20T20:18:33.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.387+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56202c58ca20 con 0x56202c648f80 2024-05-20T20:18:33.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.388+0000 7f4b65200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c643090 0x56202c643520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.388+0000 7f4b65200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c643090 0x56202c643520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41592/0 (socket says 172.21.15.28:41592) 2024-05-20T20:18:33.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.388+0000 7f4b65200700 1 -- 172.21.15.28:0/307583939 learned_addr learned my addr 172.21.15.28:0/307583939 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:33.390 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.390+0000 7f4b57400700 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c642a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.391+0000 7f4b65200700 1 -- 172.21.15.28:0/307583939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 msgr2=0x56202c642a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.391+0000 7f4b65200700 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c642a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.391+0000 7f4b65200700 1 -- 172.21.15.28:0/307583939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 msgr2=0x56202c648a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.391+0000 7f4b65200700 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c648a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.391+0000 7f4b65200700 1 -- 172.21.15.28:0/307583939 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56202c551420 con 0x56202c643090 2024-05-20T20:18:33.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.391+0000 7f4b57400700 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c642a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:33.392 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.392+0000 7f4b65200700 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c643090 0x56202c643520 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7f4b580a71f0 tx=0x7f4b580a95f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b9acc2111343218 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:33.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.392+0000 7f4b57400700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4b440b2ca0 0x7f4b440b5160 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f4b4800d3b0 tx=0x7f4b48009240 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:33.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.393+0000 7f4b55600700 1 -- 172.21.15.28:0/307583939 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b580a9750 con 0x56202c643090 2024-05-20T20:18:33.393 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.393+0000 7f4b55600700 1 -- 172.21.15.28:0/307583939 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4b580a9d10 con 0x56202c643090 2024-05-20T20:18:33.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.394+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/307583939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c643090 msgr2=0x56202c643520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.394+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c643090 0x56202c643520 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7f4b580a71f0 tx=0x7f4b580a95f0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.395 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.396+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/307583939 shutdown_connections 2024-05-20T20:18:33.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.396+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c643090 0x56202c643520 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.396+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c648a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.396+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/307583939 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c642a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.396 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.396+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/307583939 >> 172.21.15.28:0/307583939 conn(0x56202c568be0 msgr2=0x56202c635010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:33.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.397+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/307583939 shutdown_connections 2024-05-20T20:18:33.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.398+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/307583939 wait complete. 2024-05-20T20:18:33.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.399+0000 7f4b71bd5b80 1 -- start start 2024-05-20T20:18:33.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.400+0000 7f4b71bd5b80 1 Processor -- start 2024-05-20T20:18:33.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.400+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648660 0x56202c64ead0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c64f010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c64bf30 0x56202c64c3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x56202c48efa0 con 0x56202c64bf30 2024-05-20T20:18:33.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x56202c529760 con 0x56202c648660 2024-05-20T20:18:33.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x56202c64faf0 con 0x56202c648f80 2024-05-20T20:18:33.402 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b71bd5b80 10 client.0 _ll_register_callbacks cb 0x56202c532c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:33.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b5fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648660 0x56202c64ead0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b5fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648660 0x56202c64ead0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41604/0 (socket says 172.21.15.28:41604) 2024-05-20T20:18:33.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.402+0000 7f4b5fe00700 1 -- 172.21.15.28:0/3761613837 learned_addr learned my addr 172.21.15.28:0/3761613837 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:33.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.403+0000 7f4b57400700 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c64f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.404+0000 7f4b5fe00700 1 -- 172.21.15.28:0/3761613837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 msgr2=0x56202c64f010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.404+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c64f010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.404 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.404+0000 7f4b5fe00700 1 -- 172.21.15.28:0/3761613837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c64bf30 msgr2=0x56202c64c3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.404+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c64bf30 0x56202c64c3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.405 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.404+0000 7f4b5fe00700 1 -- 172.21.15.28:0/3761613837 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56202c4d88d0 con 0x56202c648660 2024-05-20T20:18:33.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.406+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648660 0x56202c64ead0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f4b500078e0 tx=0x7f4b50007bf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1da8faba0b26497d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:33.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.406+0000 7f4b4e000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:33.406 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.407+0000 7f4b4e000700 1 -- 172.21.15.28:0/3761613837 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b5000d7e0 con 0x56202c648660 2024-05-20T20:18:33.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.407+0000 7f4b4e000700 1 -- 172.21.15.28:0/3761613837 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f4b5000d980 con 0x56202c648660 2024-05-20T20:18:33.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.407+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x56202c59ad90 con 0x56202c648660 2024-05-20T20:18:33.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.407+0000 7f4b4e000700 1 -- 172.21.15.28:0/3761613837 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4b5000dba0 con 0x56202c648660 2024-05-20T20:18:33.407 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.407+0000 7f4b71bd5b80 10 client.27962 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:33.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.408+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56202c559060 con 0x56202c648660 2024-05-20T20:18:33.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.408+0000 7f4b4e000700 1 -- 172.21.15.28:0/3761613837 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x56202c59ad90 con 0x56202c648660 2024-05-20T20:18:33.408 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.409+0000 7f4b4e000700 1 -- 172.21.15.28:0/3761613837 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4b5001d030 con 0x56202c648660 2024-05-20T20:18:33.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.410+0000 7f4b71bd5b80 10 client.27962 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:33.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.411+0000 7f4b71bd5b80 10 client.27962 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:33.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.411+0000 7f4b71bd5b80 20 client.27962 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:33.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.411+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c65eef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.411+0000 7f4b71bd5b80 4 client.27962 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:33.411 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.411+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x56202c594570 con 0x56202c65ca90 2024-05-20T20:18:33.412 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.412+0000 7f4b57400700 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c65eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.414+0000 7f4b57400700 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c65eef0 crc :-1 s=READY pgs=259 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5dfc7b00ddee784b server_cookie=b4d510a44d1dc3bb in_seq=0 out_seq=0 2024-05-20T20:18:33.413 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.414+0000 7f4b4e000700 10 client.27962 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:33.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.414+0000 7f4b4e000700 1 -- 172.21.15.28:0/3761613837 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x56202c594570 con 0x56202c65ca90 2024-05-20T20:18:33.414 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.415+0000 7f4b4e000700 10 client.27962 handle_command_reply: tid=0 2024-05-20T20:18:33.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.415+0000 7f4b71bd5b80 1 client.27962 shutdown 2024-05-20T20:18:33.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.416+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648660 msgr2=0x56202c64ead0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.415 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.416+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648660 0x56202c64ead0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f4b500078e0 tx=0x7f4b50007bf0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.419+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 shutdown_connections 2024-05-20T20:18:33.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.419+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c65eef0 crc :-1 s=READY pgs=259 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.419+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c64bf30 0x56202c64c3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.419+0000 7f4b4e000700 0 client.27962 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:33.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.419+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648660 0x56202c64ead0 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.419+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/3761613837 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c64f010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.419+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 >> 172.21.15.28:0/3761613837 conn(0x56202c568be0 msgr2=0x56202c639250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:33.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.420+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 shutdown_connections 2024-05-20T20:18:33.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.420+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/3761613837 wait complete. 2024-05-20T20:18:33.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.420+0000 7f4b71bd5b80 20 client.27962 trim_cache size 0 max 16384 2024-05-20T20:18:33.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.453+0000 7f4b71bd5b80 1 Processor -- start 2024-05-20T20:18:33.453 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.454+0000 7f4b71bd5b80 1 -- start start 2024-05-20T20:18:33.454 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.454+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c647f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.454+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648f80 0x56202c65eeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.454+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c64bf30 0x56202c658c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.454+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c00d680 con 0x56202c648660 2024-05-20T20:18:33.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.454+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c00f1c0 con 0x56202c648f80 2024-05-20T20:18:33.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.454+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c00dd50 con 0x56202c64bf30 2024-05-20T20:18:33.455 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.455+0000 7f4b57400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648f80 0x56202c65eeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.455+0000 7f4b65200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c64bf30 0x56202c658c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.455+0000 7f4b65200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c64bf30 0x56202c658c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57504/0 (socket says 172.21.15.28:57504) 2024-05-20T20:18:33.456 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.455+0000 7f4b65200700 1 -- 172.21.15.28:0/4178144983 learned_addr learned my addr 172.21.15.28:0/4178144983 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:33.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.456+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c647f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.457 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.457+0000 7f4b5fe00700 1 -- 172.21.15.28:0/4178144983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c64bf30 msgr2=0x56202c658c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.457+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c64bf30 0x56202c658c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.457+0000 7f4b5fe00700 1 -- 172.21.15.28:0/4178144983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648f80 msgr2=0x56202c65eeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.457+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648f80 0x56202c65eeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.457+0000 7f4b5fe00700 1 -- 172.21.15.28:0/4178144983 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x56202c64a360 con 0x56202c648660 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.458+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c647f20 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f4b50002230 tx=0x7f4b50009490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c67ad8b63c84444 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.459+0000 7f4b54c00700 1 -- 172.21.15.28:0/4178144983 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b5002b030 con 0x56202c648660 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.460+0000 7f4b54c00700 1 -- 172.21.15.28:0/4178144983 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4b50005c60 con 0x56202c648660 2024-05-20T20:18:33.460 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.460+0000 7f4b54c00700 1 -- 172.21.15.28:0/4178144983 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b5001ea30 con 0x56202c648660 2024-05-20T20:18:33.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.461+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/4178144983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 msgr2=0x56202c647f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.461 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.461+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c647f20 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f4b50002230 tx=0x7f4b50009490 comp rx=0 tx=0).stop 2024-05-20T20:18:33.462 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.462+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/4178144983 shutdown_connections 2024-05-20T20:18:33.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.462+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c64bf30 0x56202c658c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.462+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c647f20 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.462+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/4178144983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c648f80 0x56202c65eeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.463 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.462+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/4178144983 >> 172.21.15.28:0/4178144983 conn(0x56202c568be0 msgr2=0x56202c64ead0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:33.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.463+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/4178144983 shutdown_connections 2024-05-20T20:18:33.467 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.463+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/4178144983 wait complete. 2024-05-20T20:18:33.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.464+0000 7f4b71bd5b80 1 -- start start 2024-05-20T20:18:33.468 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.465+0000 7f4b71bd5b80 1 Processor -- start 2024-05-20T20:18:33.470 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.471+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c655680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.472+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c6641c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.472 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.473+0000 7f4b71bd5b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c64bf30 0x56202c664900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.473+0000 7f4b71bd5b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c00eaf0 con 0x56202c648660 2024-05-20T20:18:33.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.473+0000 7f4b57400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c6641c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.473+0000 7f4b57400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c6641c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:57510/0 (socket says 172.21.15.28:57510) 2024-05-20T20:18:33.473 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.473+0000 7f4b57400700 1 -- 172.21.15.28:0/515364124 learned_addr learned my addr 172.21.15.28:0/515364124 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:33.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.474+0000 7f4b65200700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c64bf30 0x56202c664900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.474+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c00e420 con 0x56202c64bf30 2024-05-20T20:18:33.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.474+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4b3c00b640 con 0x56202c648f80 2024-05-20T20:18:33.474 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.475+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c655680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.475 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.475+0000 7f4b71bd5b80 10 client.0 _ll_register_callbacks cb 0x56202c532c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:33.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.476+0000 7f4b57400700 1 -- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c64bf30 msgr2=0x56202c664900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.476+0000 7f4b57400700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c64bf30 0x56202c664900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.476 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.476+0000 7f4b57400700 1 -- 172.21.15.28:0/515364124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 msgr2=0x56202c655680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:33.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.476+0000 7f4b57400700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c655680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.476+0000 7f4b57400700 1 -- 172.21.15.28:0/515364124 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x56202c6c6690 con 0x56202c648f80 2024-05-20T20:18:33.477 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.477+0000 7f4b65200700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c64bf30 0x56202c664900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:18:33.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.477+0000 7f4b57400700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c6641c0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f4b4800bfd0 tx=0x7f4b480058e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6c51f7a7c16e0445 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:33.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.477+0000 7f4b4e000700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:33.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.478+0000 7f4b4e000700 1 -- 172.21.15.28:0/515364124 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b48005240 con 0x56202c648f80 2024-05-20T20:18:33.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.478+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f4b50005c60 con 0x56202c648f80 2024-05-20T20:18:33.478 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.478+0000 7f4b4e000700 1 -- 172.21.15.28:0/515364124 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f4b480053e0 con 0x56202c648f80 2024-05-20T20:18:33.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.479+0000 7f4b4e000700 1 -- 172.21.15.28:0/515364124 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4b480055e0 con 0x56202c648f80 2024-05-20T20:18:33.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.480+0000 7f4b71bd5b80 10 client.27898 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:33.479 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.480+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x56202c6c7850 con 0x56202c648f80 2024-05-20T20:18:33.480 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b4e000700 1 -- 172.21.15.28:0/515364124 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4b50005c60 con 0x56202c648f80 2024-05-20T20:18:33.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b4e000700 1 -- 172.21.15.28:0/515364124 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4b48013070 con 0x56202c648f80 2024-05-20T20:18:33.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b71bd5b80 10 client.27898 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:33.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b71bd5b80 10 client.27898 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:33.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b71bd5b80 20 client.27898 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:33.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c6ca000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:33.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b71bd5b80 4 client.27898 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:18:33.481 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.481+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x56202c40b250 con 0x56202c65ca90 2024-05-20T20:18:33.483 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.484+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c6ca000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:33.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.489+0000 7f4b5fe00700 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c6ca000 crc :-1 s=READY pgs=260 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6bdaff0a3dd18dc4 server_cookie=50e31e201a126ed1 in_seq=0 out_seq=0 2024-05-20T20:18:33.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.489+0000 7f4b4e000700 10 client.27898 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:33.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.491+0000 7f4b4e000700 1 -- 172.21.15.28:0/515364124 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x56202c40b250 con 0x56202c65ca90 2024-05-20T20:18:33.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.491+0000 7f4b4e000700 10 client.27898 handle_command_reply: tid=0 2024-05-20T20:18:33.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.492+0000 7f4b71bd5b80 1 client.27898 shutdown 2024-05-20T20:18:33.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.492+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 msgr2=0x56202c6641c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.492+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c6641c0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f4b4800bfd0 tx=0x7f4b480058e0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.493+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 shutdown_connections 2024-05-20T20:18:33.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.493+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x56202c65ca90 0x56202c6ca000 crc :-1 s=READY pgs=260 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.493+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x56202c64bf30 0x56202c664900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.493+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x56202c648660 0x56202c655680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.493+0000 7f4b71bd5b80 1 --2- 172.21.15.28:0/515364124 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x56202c648f80 0x56202c6641c0 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.493+0000 7f4b4e000700 0 client.27898 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:33.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.493+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 >> 172.21.15.28:0/515364124 conn(0x56202c568be0 msgr2=0x56202c64a6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:33.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.495+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 shutdown_connections 2024-05-20T20:18:33.498 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:33 smithi028 ceph-mon[31178]: pgmap v404: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 318 MiB/s wr, 2.35k op/s 2024-05-20T20:18:33.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.497+0000 7f4b71bd5b80 1 -- 172.21.15.28:0/515364124 wait complete. 2024-05-20T20:18:33.499 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.497+0000 7f4b71bd5b80 20 client.27898 trim_cache size 0 max 16384 2024-05-20T20:18:33.499 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:18:33.501 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.501+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4b440b2ca0 msgr2=0x7f4b440b5160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.503+0000 7f4b67200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4b440b2ca0 0x7f4b440b5160 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f4b4800d3b0 tx=0x7f4b48009240 comp rx=0 tx=0).stop 2024-05-20T20:18:33.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.503+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010ef90 msgr2=0x7f4b601a8e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:33.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.503+0000 7f4b67200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010ef90 0x7f4b601a8e90 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x7f4b5800c970 tx=0x7f4b5800cd30 comp rx=0 tx=0).stop 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.504+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 shutdown_connections 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.505+0000 7f4b67200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f4b440b2ca0 0x7f4b440b5160 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.505+0000 7f4b67200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f4b6010ef90 0x7f4b601a8e90 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.505+0000 7f4b67200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f4b6010a540 0x7f4b601a4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.505+0000 7f4b67200700 1 --2- 172.21.15.28:0/3187456688 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4b6010af00 0x7f4b601a4b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.505+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 >> 172.21.15.28:0/3187456688 conn(0x7f4b6006ea90 msgr2=0x7f4b600730b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.505+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 shutdown_connections 2024-05-20T20:18:33.505 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:33.506+0000 7f4b67200700 1 -- 172.21.15.28:0/3187456688 wait complete. 2024-05-20T20:18:33.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:33 smithi154 ceph-mon[36314]: pgmap v404: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 318 MiB/s wr, 2.35k op/s 2024-05-20T20:18:33.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:33 smithi033 ceph-mon[36179]: pgmap v404: 97 pgs: 97 active+clean; 60 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 318 MiB/s wr, 2.35k op/s 2024-05-20T20:18:35.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:35 smithi028 ceph-mon[31178]: pgmap v405: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 287 KiB/s rd, 396 MiB/s wr, 2.59k op/s 2024-05-20T20:18:35.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:35 smithi154 ceph-mon[36314]: pgmap v405: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 287 KiB/s rd, 396 MiB/s wr, 2.59k op/s 2024-05-20T20:18:35.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:35 smithi033 ceph-mon[36179]: pgmap v405: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 287 KiB/s rd, 396 MiB/s wr, 2.59k op/s 2024-05-20T20:18:35.983 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-05-20T20:18:35.984 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub start / force,recursive 2024-05-20T20:18:36.259 INFO:tasks.workunit.client.0.smithi028.stdout:stopping iogen 2024-05-20T20:18:36.259 INFO:tasks.workunit.client.0.smithi028.stderr:+ echo 'stopping iogen' 2024-05-20T20:18:36.259 INFO:tasks.workunit.client.0.smithi028.stderr:+ ./iogen -k 2024-05-20T20:18:36.278 INFO:tasks.workunit.client.0.smithi028.stdout:OK 2024-05-20T20:18:36.278 INFO:tasks.workunit.client.0.smithi028.stderr:+ echo OK 2024-05-20T20:18:37.446 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:37 smithi028 ceph-mon[31178]: pgmap v406: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 274 KiB/s rd, 378 MiB/s wr, 2.47k op/s 2024-05-20T20:18:37.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:37 smithi154 ceph-mon[36314]: pgmap v406: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 274 KiB/s rd, 378 MiB/s wr, 2.47k op/s 2024-05-20T20:18:37.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:37 smithi033 ceph-mon[36179]: pgmap v406: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 274 KiB/s rd, 378 MiB/s wr, 2.47k op/s 2024-05-20T20:18:37.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.830+0000 7f0c0d000700 1 -- 172.21.15.28:0/2305852031 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 msgr2=0x7f0c0810fcf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:37.891 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.830+0000 7f0c0d000700 1 --2- 172.21.15.28:0/2305852031 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 0x7f0c0810fcf0 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0013ef0 tx=0x7f0bf00097f0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.833+0000 7f0c0d000700 1 -- 172.21.15.28:0/2305852031 shutdown_connections 2024-05-20T20:18:37.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.833+0000 7f0c0d000700 1 --2- 172.21.15.28:0/2305852031 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c08110230 0x7f0c081109e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.833+0000 7f0c0d000700 1 --2- 172.21.15.28:0/2305852031 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 0x7f0c0810ad30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.892 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.833+0000 7f0c0d000700 1 --2- 172.21.15.28:0/2305852031 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 0x7f0c0810fcf0 unknown :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.833+0000 7f0c0d000700 1 -- 172.21.15.28:0/2305852031 >> 172.21.15.28:0/2305852031 conn(0x7f0c0806e6b0 msgr2=0x7f0c0806eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:37.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.837+0000 7f0c0d000700 1 -- 172.21.15.28:0/2305852031 shutdown_connections 2024-05-20T20:18:37.893 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.837+0000 7f0c0d000700 1 -- 172.21.15.28:0/2305852031 wait complete. 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.841+0000 7f0c0d000700 1 Processor -- start 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.842+0000 7f0c0d000700 1 -- start start 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.844+0000 7f0c0d000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 0x7f0c081a0210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.844+0000 7f0c0d000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 0x7f0c081a0750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.845+0000 7f0c0d000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c08110230 0x7f0c081a4ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.845+0000 7f0c0d000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0c08119eb0 con 0x7f0c08110230 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.845+0000 7f0c0d000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0c08119d30 con 0x7f0c0810a540 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.845+0000 7f0c0d000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0c0811a030 con 0x7f0c0810f870 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.846+0000 7f0c05e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 0x7f0c081a0210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.896 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.846+0000 7f0c05e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 0x7f0c081a0210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:55974/0 (socket says 172.21.15.28:55974) 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.846+0000 7f0c05e00700 1 -- 172.21.15.28:0/1238389344 learned_addr learned my addr 172.21.15.28:0/1238389344 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0bffe00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 0x7f0c081a0750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0c05e00700 1 -- 172.21.15.28:0/1238389344 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 msgr2=0x7f0c081a0750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0c06800700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c08110230 0x7f0c081a4ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0c05e00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 0x7f0c081a0750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0c05e00700 1 -- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c08110230 msgr2=0x7f0c081a4ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0c05e00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c08110230 0x7f0c081a4ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0c05e00700 1 -- 172.21.15.28:0/1238389344 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c081a5220 con 0x7f0c0810a540 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0bffe00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 0x7f0c081a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.847+0000 7f0c06800700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c08110230 0x7f0c081a4ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.848+0000 7f0c05e00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 0x7f0c081a0210 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f0bf800d980 tx=0x7f0bf800dd40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:37.897 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.854+0000 7f0bfea00700 1 -- 172.21.15.28:0/1238389344 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf8017070 con 0x7f0c0810a540 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.854+0000 7f0bfea00700 1 -- 172.21.15.28:0/1238389344 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0bf8004830 con 0x7f0c0810a540 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.854+0000 7f0bfea00700 1 -- 172.21.15.28:0/1238389344 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf8005440 con 0x7f0c0810a540 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.855+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c081a5510 con 0x7f0c0810a540 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.855+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c081acd90 con 0x7f0c0810a540 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.873+0000 7f0bfea00700 1 -- 172.21.15.28:0/1238389344 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0bf80055e0 con 0x7f0c0810a540 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.876+0000 7f0bfea00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0bec0c3c70 0x7f0bec0c6130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.881+0000 7f0c179d1b80 1 Processor -- start 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.882+0000 7f0c179d1b80 1 -- start start 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.883+0000 7f0bffe00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0bec0c3c70 0x7f0bec0c6130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.883+0000 7f0bfea00700 1 -- 172.21.15.28:0/1238389344 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f0bf80e2220 con 0x7f0c0810a540 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.886+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce79ca0 0x55659ce7a080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.898 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.887+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7a5c0 0x55659ce74060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.887+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce74680 0x55659ce74ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.887+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55659cdd1fd0 con 0x55659ce79ca0 2024-05-20T20:18:37.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.887+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55659cdc1b70 con 0x55659ce74680 2024-05-20T20:18:37.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.887+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55659cdc5bb0 con 0x55659ce7a5c0 2024-05-20T20:18:37.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.888+0000 7f0c06800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce74680 0x55659ce74ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.899 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.888+0000 7f0c06800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce74680 0x55659ce74ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:55984/0 (socket says 172.21.15.28:55984) 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.888+0000 7f0c06800700 1 -- 172.21.15.28:0/605825704 learned_addr learned my addr 172.21.15.28:0/605825704 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.885+0000 7f0bffe00700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0bec0c3c70 0x7f0bec0c6130 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0006010 tx=0x7f0bf0015040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.889+0000 7f0c06800700 1 -- 172.21.15.28:0/605825704 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7a5c0 msgr2=0x55659ce74060 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.889+0000 7f0c06800700 1 --2- 172.21.15.28:0/605825704 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7a5c0 0x55659ce74060 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.889+0000 7f0c06800700 1 -- 172.21.15.28:0/605825704 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce79ca0 msgr2=0x55659ce7a080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.889+0000 7f0c06800700 1 --2- 172.21.15.28:0/605825704 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce79ca0 0x55659ce7a080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.890+0000 7f0c06800700 1 -- 172.21.15.28:0/605825704 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55659cd8a5b0 con 0x55659ce74680 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.892+0000 7f0c06800700 1 --2- 172.21.15.28:0/605825704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce74680 0x55659ce74ae0 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7f0c00007f30 tx=0x7f0c0000c5a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3e30a9196099b0f6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.893+0000 7f0bf7400700 1 -- 172.21.15.28:0/605825704 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c00014070 con 0x55659ce74680 2024-05-20T20:18:37.900 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.894+0000 7f0bf7400700 1 -- 172.21.15.28:0/605825704 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0c0000faf0 con 0x55659ce74680 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.894+0000 7f0bf7400700 1 -- 172.21.15.28:0/605825704 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c0000eb10 con 0x55659ce74680 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.896+0000 7f0c179d1b80 1 -- 172.21.15.28:0/605825704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce74680 msgr2=0x55659ce74ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.896+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/605825704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce74680 0x55659ce74ae0 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x7f0c00007f30 tx=0x7f0c0000c5a0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.900+0000 7f0c179d1b80 1 -- 172.21.15.28:0/605825704 shutdown_connections 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.900+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/605825704 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce74680 0x55659ce74ae0 unknown :-1 s=CLOSED pgs=604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.900+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/605825704 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce79ca0 0x55659ce7a080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.900+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/605825704 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7a5c0 0x55659ce74060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.901 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.900+0000 7f0c179d1b80 1 -- 172.21.15.28:0/605825704 >> 172.21.15.28:0/605825704 conn(0x55659cda1d70 msgr2=0x55659ce6ef90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:37.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.902+0000 7f0c179d1b80 1 -- 172.21.15.28:0/605825704 shutdown_connections 2024-05-20T20:18:37.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.902+0000 7f0c179d1b80 1 -- 172.21.15.28:0/605825704 wait complete. 2024-05-20T20:18:37.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.902+0000 7f0c179d1b80 1 -- start start 2024-05-20T20:18:37.905 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.903+0000 7f0c179d1b80 1 Processor -- start 2024-05-20T20:18:37.907 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.907+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce80420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.909 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.909+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce7d7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.910+0000 7f0bffe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce7d7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.910+0000 7f0c05e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce80420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.910+0000 7f0bffe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce7d7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56000/0 (socket says 172.21.15.28:56000) 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.910+0000 7f0c05e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce80420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40472/0 (socket says 172.21.15.28:40472) 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.911+0000 7f0bffe00700 1 -- 172.21.15.28:0/1618649029 learned_addr learned my addr 172.21.15.28:0/1618649029 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.911+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7dd00 0x55659ce7e180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.911+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55659ccc8130 con 0x55659ce74680 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.911+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55659cd628f0 con 0x55659ce79ca0 2024-05-20T20:18:37.918 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.911+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55659ce81030 con 0x55659ce7dd00 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.911+0000 7f0c06800700 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7dd00 0x55659ce7e180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0c179d1b80 10 client.0 _ll_register_callbacks cb 0x55659cd6be00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0c06800700 1 -- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 msgr2=0x55659ce7d7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0c06800700 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce7d7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0c06800700 1 -- 172.21.15.28:0/1618649029 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 msgr2=0x55659ce80420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0c06800700 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce80420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0c06800700 1 -- 172.21.15.28:0/1618649029 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55659cd11a60 con 0x55659ce7dd00 2024-05-20T20:18:37.919 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0bffe00700 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce7d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.912+0000 7f0c06800700 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7dd00 0x55659ce7e180 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f0c0000c9e0 tx=0x7f0c00007a40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f041a73e8b931c25 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.914+0000 7f0bebe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.914+0000 7f0bebe00700 1 -- 172.21.15.28:0/1618649029 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c00014040 con 0x55659ce7dd00 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.914+0000 7f0bebe00700 1 -- 172.21.15.28:0/1618649029 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f0c0000e8b0 con 0x55659ce7dd00 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.914+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0c0000faf0 con 0x55659ce7dd00 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.915+0000 7f0bebe00700 1 -- 172.21.15.28:0/1618649029 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0c0002e050 con 0x55659ce7dd00 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.915+0000 7f0c179d1b80 10 client.27904 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.915+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55659cd921f0 con 0x55659ce7dd00 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.915+0000 7f0bebe00700 1 -- 172.21.15.28:0/1618649029 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0c0000faf0 con 0x55659ce7dd00 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.915+0000 7f0bebe00700 1 -- 172.21.15.28:0/1618649029 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0c00029030 con 0x55659ce7dd00 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.916+0000 7f0c179d1b80 10 client.27904 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:37.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.916+0000 7f0c179d1b80 10 client.27904 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.916+0000 7f0c179d1b80 20 client.27904 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.916+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659ce8e010 0x55659ce90400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.916+0000 7f0c179d1b80 4 client.27904 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.916+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55659cdd3f20 con 0x55659ce8e010 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.916+0000 7f0c05e00700 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659ce8e010 0x55659ce90400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.917+0000 7f0c05e00700 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659ce8e010 0x55659ce90400 crc :-1 s=READY pgs=261 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8abd18f6de74c87 server_cookie=6579ae2b994e4b68 in_seq=0 out_seq=0 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.917+0000 7f0bebe00700 10 client.27904 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:37.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.918+0000 7f0bebe00700 1 -- 172.21.15.28:0/1618649029 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55659cdd3f20 con 0x55659ce8e010 2024-05-20T20:18:37.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.918+0000 7f0bebe00700 10 client.27904 handle_command_reply: tid=0 2024-05-20T20:18:37.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.918+0000 7f0c179d1b80 1 client.27904 shutdown 2024-05-20T20:18:37.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.920+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7dd00 msgr2=0x55659ce7e180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:37.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.920+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7dd00 0x55659ce7e180 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f0c0000c9e0 tx=0x7f0c00007a40 comp rx=0 tx=0).stop 2024-05-20T20:18:37.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.924+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 shutdown_connections 2024-05-20T20:18:37.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.924+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659ce8e010 0x55659ce90400 crc :-1 s=READY pgs=261 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.924+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce7dd00 0x55659ce7e180 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.924+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce80420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.924+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1618649029 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce7d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:37.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.924+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 >> 172.21.15.28:0/1618649029 conn(0x55659cda1d70 msgr2=0x55659ce6e4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:37.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.924+0000 7f0bebe00700 0 client.27904 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:37.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.925+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 shutdown_connections 2024-05-20T20:18:37.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.926+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1618649029 wait complete. 2024-05-20T20:18:37.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.929+0000 7f0c179d1b80 20 client.27904 trim_cache size 0 max 16384 2024-05-20T20:18:37.990 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.991+0000 7f0c179d1b80 1 Processor -- start 2024-05-20T20:18:37.991 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.991+0000 7f0c179d1b80 1 -- start start 2024-05-20T20:18:37.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.993+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce74060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.993+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce79ca0 0x55659ce790b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.993+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce7dd00 0x55659ce7c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:37.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.993+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0c081b44a0 con 0x55659ce74680 2024-05-20T20:18:37.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.993+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0c081b5fe0 con 0x55659ce7dd00 2024-05-20T20:18:37.995 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.993+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0c081b4b70 con 0x55659ce79ca0 2024-05-20T20:18:37.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.999+0000 7f0c06800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce7dd00 0x55659ce7c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:37.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.999+0000 7f0c06800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce7dd00 0x55659ce7c7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56012/0 (socket says 172.21.15.28:56012) 2024-05-20T20:18:37.999 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:37.999+0000 7f0bffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce79ca0 0x55659ce790b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:38.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.000+0000 7f0bffe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce79ca0 0x55659ce790b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56522/0 (socket says 172.21.15.28:56522) 2024-05-20T20:18:38.001 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.000+0000 7f0bffe00700 1 -- 172.21.15.28:0/1416867995 learned_addr learned my addr 172.21.15.28:0/1416867995 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:38.006 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.005+0000 7f0c05e00700 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce74060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:38.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.005+0000 7f0c06800700 1 -- 172.21.15.28:0/1416867995 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce79ca0 msgr2=0x55659ce790b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:38.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.005+0000 7f0c06800700 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce79ca0 0x55659ce790b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.005+0000 7f0c06800700 1 -- 172.21.15.28:0/1416867995 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 msgr2=0x55659ce74060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:38.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.006+0000 7f0c06800700 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce74060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.007 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.007+0000 7f0c06800700 1 -- 172.21.15.28:0/1416867995 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55659ce66db0 con 0x55659ce7dd00 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.008+0000 7f0c06800700 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce7dd00 0x55659ce7c7c0 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7f0c00004c20 tx=0x7f0c00007780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2bedcf70e3e57a06 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.008+0000 7f0c05e00700 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce74060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.008+0000 7f0bf6a00700 1 -- 172.21.15.28:0/1416867995 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c00037070 con 0x55659ce7dd00 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.009+0000 7f0bf6a00700 1 -- 172.21.15.28:0/1416867995 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0c00019bf0 con 0x55659ce7dd00 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.009+0000 7f0bf6a00700 1 -- 172.21.15.28:0/1416867995 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c00014070 con 0x55659ce7dd00 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.009+0000 7f0bffe00700 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce79ca0 0x55659ce790b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.010+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1416867995 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce7dd00 msgr2=0x55659ce7c7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:38.010 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.010+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce7dd00 0x55659ce7c7c0 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7f0c00004c20 tx=0x7f0c00007780 comp rx=0 tx=0).stop 2024-05-20T20:18:38.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.011+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1416867995 shutdown_connections 2024-05-20T20:18:38.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.011+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce7dd00 0x55659ce7c7c0 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.011+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce74680 0x55659ce74060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.011+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/1416867995 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce79ca0 0x55659ce790b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.011 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.011+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1416867995 >> 172.21.15.28:0/1416867995 conn(0x55659cda1d70 msgr2=0x55659ce7c0b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:38.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.011+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1416867995 shutdown_connections 2024-05-20T20:18:38.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.012+0000 7f0c179d1b80 1 -- 172.21.15.28:0/1416867995 wait complete. 2024-05-20T20:18:38.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.012+0000 7f0c179d1b80 1 -- start start 2024-05-20T20:18:38.012 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 1 Processor -- start 2024-05-20T20:18:38.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce74680 0x55659ce868a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:38.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce81280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:38.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce7dd00 0x55659ce817c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:38.016 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0c081b5910 con 0x55659ce7dd00 2024-05-20T20:18:38.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0c081b5240 con 0x55659ce79ca0 2024-05-20T20:18:38.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0c081b2460 con 0x55659ce74680 2024-05-20T20:18:38.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.013+0000 7f0c179d1b80 10 client.0 _ll_register_callbacks cb 0x55659cd6be00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:38.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.015+0000 7f0bffe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce81280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:38.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.015+0000 7f0c05e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce74680 0x55659ce868a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:38.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.015+0000 7f0bffe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce81280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56018/0 (socket says 172.21.15.28:56018) 2024-05-20T20:18:38.017 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.015+0000 7f0bffe00700 1 -- 172.21.15.28:0/313955285 learned_addr learned my addr 172.21.15.28:0/313955285 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:38.018 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.018+0000 7f0c06800700 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce7dd00 0x55659ce817c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:38.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.019+0000 7f0bffe00700 1 -- 172.21.15.28:0/313955285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce74680 msgr2=0x55659ce868a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:38.019 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.020+0000 7f0bffe00700 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce74680 0x55659ce868a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.020+0000 7f0bffe00700 1 -- 172.21.15.28:0/313955285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce7dd00 msgr2=0x55659ce817c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:38.020 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.020+0000 7f0bffe00700 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce7dd00 0x55659ce817c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.021 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.021+0000 7f0bffe00700 1 -- 172.21.15.28:0/313955285 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55659ce81e10 con 0x55659ce79ca0 2024-05-20T20:18:38.025 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.025+0000 7f0bebe00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:38.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.025+0000 7f0bffe00700 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce81280 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0013e40 tx=0x7f0bf0009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f4c4da487c9dffaa server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:38.027 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.026+0000 7f0bebe00700 1 -- 172.21.15.28:0/313955285 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0011940 con 0x55659ce79ca0 2024-05-20T20:18:38.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.027+0000 7f0bebe00700 1 -- 172.21.15.28:0/313955285 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f0bf0011ae0 con 0x55659ce79ca0 2024-05-20T20:18:38.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.027+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f0c00019bf0 con 0x55659ce79ca0 2024-05-20T20:18:38.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.027+0000 7f0bebe00700 1 -- 172.21.15.28:0/313955285 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f0bf0011ce0 con 0x55659ce79ca0 2024-05-20T20:18:38.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 10 client.27983 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:38.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55659cef9650 con 0x55659ce79ca0 2024-05-20T20:18:38.028 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0bebe00700 1 -- 172.21.15.28:0/313955285 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0bf00318f0 con 0x55659ce79ca0 2024-05-20T20:18:38.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0bebe00700 1 -- 172.21.15.28:0/313955285 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f0bf002f020 con 0x55659ce79ca0 2024-05-20T20:18:38.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 10 client.27983 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:38.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 10 client.27983 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:18:38.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 20 client.27983 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:38.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659cef9e00 0x55659cefc220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:38.030 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 4 client.27983 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 137.9, "await_for": 45.0}] 2024-05-20T20:18:38.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.028+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "roots": ["/"], "timeout": 90.0, "expiration": 137.9, "await_for": 45.0}) -- 0x7f0c00019bf0 con 0x55659cef9e00 2024-05-20T20:18:38.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.029+0000 7f0c05e00700 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659cef9e00 0x55659cefc220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:38.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.029+0000 7f0c05e00700 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659cef9e00 0x55659cefc220 crc :-1 s=READY pgs=262 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1eeeb19615dfba63 server_cookie=94ba6c9ae37919e8 in_seq=0 out_seq=0 2024-05-20T20:18:38.031 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.029+0000 7f0bebe00700 10 client.27983 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:38.108 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0bebe00700 1 -- 172.21.15.28:0/313955285 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x7f0c00019bf0 con 0x55659cef9e00 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0bebe00700 10 client.27983 handle_command_reply: tid=0 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 client.27983 shutdown 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 msgr2=0x55659ce81280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce81280 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0013e40 tx=0x7f0bf0009310 comp rx=0 tx=0).stop 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 shutdown_connections 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55659cef9e00 0x55659cefc220 crc :-1 s=READY pgs=262 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55659ce7dd00 0x55659ce817c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.189 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55659ce74680 0x55659ce868a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 --2- 172.21.15.28:0/313955285 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55659ce79ca0 0x55659ce81280 unknown :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 >> 172.21.15.28:0/313955285 conn(0x55659cda1d70 msgr2=0x55659ce7c940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0bebe00700 0 client.27983 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.172+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 shutdown_connections 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.173+0000 7f0c179d1b80 1 -- 172.21.15.28:0/313955285 wait complete. 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.175+0000 7f0c179d1b80 20 client.27983 trim_cache size 0 max 16384 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:18:38.190 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 34, 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "ec1db510": { 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 34, 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 137.9, 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:18:38.191 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "QUIESCED", 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.186+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0bec0c3c70 msgr2=0x7f0bec0c6130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.186+0000 7f0c0d000700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0bec0c3c70 0x7f0bec0c6130 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0006010 tx=0x7f0bf0015040 comp rx=0 tx=0).stop 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.186+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 msgr2=0x7f0c081a0210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:38.192 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.186+0000 7f0c0d000700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 0x7f0c081a0210 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f0bf800d980 tx=0x7f0bf800dd40 comp rx=0 tx=0).stop 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 shutdown_connections 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c08110230 0x7f0c081a4ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0bec0c3c70 0x7f0bec0c6130 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c0810a540 0x7f0c081a0210 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 --2- 172.21.15.28:0/1238389344 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c0810f870 0x7f0c081a0750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 >> 172.21.15.28:0/1238389344 conn(0x7f0c0806e6b0 msgr2=0x7f0c08111af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 shutdown_connections 2024-05-20T20:18:38.193 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:38.187+0000 7f0c0d000700 1 -- 172.21.15.28:0/1238389344 wait complete. 2024-05-20T20:18:39.546 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:39 smithi028 ceph-mon[31178]: pgmap v407: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 224 KiB/s rd, 330 MiB/s wr, 2.15k op/s 2024-05-20T20:18:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:39 smithi154 ceph-mon[36314]: pgmap v407: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 224 KiB/s rd, 330 MiB/s wr, 2.15k op/s 2024-05-20T20:18:39.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:39 smithi033 ceph-mon[36179]: pgmap v407: 97 pgs: 97 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 224 KiB/s rd, 330 MiB/s wr, 2.15k op/s 2024-05-20T20:18:40.443 INFO:tasks.quiescer.fs.[cephfs]:Successfully quiesced set 'ec1db510', quiesce took 32.0 seconds. Will release after: -14.100000000000001 2024-05-20T20:18:40.444 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:18:41.721 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:41.803 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:41 smithi028 ceph-mon[31178]: pgmap v408: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 242 KiB/s rd, 410 MiB/s wr, 2.76k op/s 2024-05-20T20:18:41.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:41 smithi154 ceph-mon[36314]: pgmap v408: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 242 KiB/s rd, 410 MiB/s wr, 2.76k op/s 2024-05-20T20:18:41.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:41 smithi033 ceph-mon[36179]: pgmap v408: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 242 KiB/s rd, 410 MiB/s wr, 2.76k op/s 2024-05-20T20:18:42.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 -- 172.21.15.28:0/1329331838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010e3b0 msgr2=0x7fe19007ad60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 --2- 172.21.15.28:0/1329331838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010e3b0 0x7fe19007ad60 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7fe188009ca0 tx=0x7fe18802a8e0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 -- 172.21.15.28:0/1329331838 shutdown_connections 2024-05-20T20:18:42.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 --2- 172.21.15.28:0/1329331838 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010e3b0 0x7fe19007ad60 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.004 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 --2- 172.21.15.28:0/1329331838 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010f920 0x7fe19010dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 --2- 172.21.15.28:0/1329331838 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe19010ef70 0x7fe19010f350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 -- 172.21.15.28:0/1329331838 >> 172.21.15.28:0/1329331838 conn(0x7fe19006db20 msgr2=0x7fe19006df30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:42.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 -- 172.21.15.28:0/1329331838 shutdown_connections 2024-05-20T20:18:42.005 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.004+0000 7fe194a00640 1 -- 172.21.15.28:0/1329331838 wait complete. 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 Processor -- start 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 -- start start 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010e3b0 0x7fe1900791f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010ef70 0x7fe190079730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe19007ae90 0x7fe1901c53d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe19007bc90 con 0x7fe19010e3b0 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe19007bb10 con 0x7fe19010ef70 2024-05-20T20:18:42.006 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.006+0000 7fe194a00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe19007be10 con 0x7fe19007ae90 2024-05-20T20:18:42.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.007+0000 7fe18de00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010e3b0 0x7fe1900791f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.007+0000 7fe18d400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010ef70 0x7fe190079730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.007+0000 7fe18de00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010e3b0 0x7fe1900791f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40500/0 (socket says 172.21.15.28:40500) 2024-05-20T20:18:42.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.007+0000 7fe18de00640 1 -- 172.21.15.28:0/1074586 learned_addr learned my addr 172.21.15.28:0/1074586 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:42.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.007+0000 7fe18e800640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe19007ae90 0x7fe1901c53d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.007 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.008+0000 7fe18de00640 1 -- 172.21.15.28:0/1074586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe19007ae90 msgr2=0x7fe1901c53d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.008 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.008+0000 7fe18de00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe19007ae90 0x7fe1901c53d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.008 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.008+0000 7fe18de00640 1 -- 172.21.15.28:0/1074586 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010ef70 msgr2=0x7fe190079730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.008 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.008+0000 7fe18de00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010ef70 0x7fe190079730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.008 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.008+0000 7fe18de00640 1 -- 172.21.15.28:0/1074586 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe1901c5910 con 0x7fe19010e3b0 2024-05-20T20:18:42.008 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.009+0000 7fe18e800640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe19007ae90 0x7fe1901c53d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:42.008 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.009+0000 7fe18d400640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010ef70 0x7fe190079730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:42.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.009+0000 7fe18de00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010e3b0 0x7fe1900791f0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7fe18000ea60 tx=0x7fe18000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:42.009 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.009+0000 7fe186a00640 1 -- 172.21.15.28:0/1074586 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe18000cde0 con 0x7fe19010e3b0 2024-05-20T20:18:42.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.010+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe1901c5ae0 con 0x7fe19010e3b0 2024-05-20T20:18:42.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.010+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe1901c6020 con 0x7fe19010e3b0 2024-05-20T20:18:42.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.010+0000 7fe186a00640 1 -- 172.21.15.28:0/1074586 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe1800035f0 con 0x7fe19010e3b0 2024-05-20T20:18:42.010 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.010+0000 7fe186a00640 1 -- 172.21.15.28:0/1074586 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe180010690 con 0x7fe19010e3b0 2024-05-20T20:18:42.014 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.014+0000 7fe1979ba740 1 Processor -- start 2024-05-20T20:18:42.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.015+0000 7fe1979ba740 1 -- start start 2024-05-20T20:18:42.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.015+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee48293070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.015+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 0x55ee4829d930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.015+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 0x55ee482a0260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.015+0000 7fe1979ba740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ee48203b90 con 0x55ee4829de70 2024-05-20T20:18:42.015 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.015+0000 7fe1979ba740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ee481f3730 con 0x55ee482935b0 2024-05-20T20:18:42.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.015+0000 7fe1979ba740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ee481f7770 con 0x55ee48292c90 2024-05-20T20:18:42.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee48293070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee48293070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56572/0 (socket says 172.21.15.28:56572) 2024-05-20T20:18:42.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 -- 172.21.15.28:0/255388200 learned_addr learned my addr 172.21.15.28:0/255388200 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:42.016 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 -- 172.21.15.28:0/255388200 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 msgr2=0x55ee4829d930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 --2- 172.21.15.28:0/255388200 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 0x55ee4829d930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 -- 172.21.15.28:0/255388200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 msgr2=0x55ee482a0260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 --2- 172.21.15.28:0/255388200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 0x55ee482a0260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 -- 172.21.15.28:0/255388200 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ee4818c8b0 con 0x55ee48292c90 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.016+0000 7fe18e800640 1 --2- 172.21.15.28:0/255388200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee48293070 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7fe188009ca0 tx=0x7fe188009a20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5fa9f98ba1d62f32 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.017+0000 7fe17b000640 1 -- 172.21.15.28:0/255388200 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe188031d10 con 0x55ee48292c90 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.017+0000 7fe17b000640 1 -- 172.21.15.28:0/255388200 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe188036040 con 0x55ee48292c90 2024-05-20T20:18:42.017 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.017+0000 7fe17b000640 1 -- 172.21.15.28:0/255388200 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1880305e0 con 0x55ee48292c90 2024-05-20T20:18:42.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.018+0000 7fe186a00640 1 -- 172.21.15.28:0/1074586 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fe1800037a0 con 0x7fe19010e3b0 2024-05-20T20:18:42.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.018+0000 7fe1979ba740 1 -- 172.21.15.28:0/255388200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 msgr2=0x55ee48293070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.018+0000 7fe1979ba740 1 --2- 172.21.15.28:0/255388200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee48293070 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7fe188009ca0 tx=0x7fe188009a20 comp rx=0 tx=0).stop 2024-05-20T20:18:42.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.019+0000 7fe1979ba740 1 -- 172.21.15.28:0/255388200 shutdown_connections 2024-05-20T20:18:42.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.019+0000 7fe1979ba740 1 --2- 172.21.15.28:0/255388200 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 0x55ee4829d930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.018 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.019+0000 7fe1979ba740 1 --2- 172.21.15.28:0/255388200 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 0x55ee482a0260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.019+0000 7fe1979ba740 1 --2- 172.21.15.28:0/255388200 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee48293070 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.019+0000 7fe1979ba740 1 -- 172.21.15.28:0/255388200 >> 172.21.15.28:0/255388200 conn(0x55ee4828f1a0 msgr2=0x55ee4828f580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:42.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.019+0000 7fe1979ba740 1 -- 172.21.15.28:0/255388200 shutdown_connections 2024-05-20T20:18:42.019 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.020+0000 7fe1979ba740 1 -- 172.21.15.28:0/255388200 wait complete. 2024-05-20T20:18:42.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.020+0000 7fe1979ba740 1 -- start start 2024-05-20T20:18:42.020 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 1 Processor -- start 2024-05-20T20:18:42.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee4829d880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829ab70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee4829de70 0x55ee4829b180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55ee47ed9790 con 0x55ee482935b0 2024-05-20T20:18:42.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55ee4819bbc0 con 0x55ee4829de70 2024-05-20T20:18:42.021 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55ee482b1a50 con 0x55ee48292c90 2024-05-20T20:18:42.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.021+0000 7fe1979ba740 10 client.0 _ll_register_callbacks cb 0x55ee48179400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:42.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.022+0000 7fe186a00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1740b29b0 0x7fe1740b4e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.022+0000 7fe18e800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee4829d880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.022 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.022+0000 7fe186a00640 1 -- 172.21.15.28:0/1074586 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fe1800e2820 con 0x7fe19010e3b0 2024-05-20T20:18:42.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.022+0000 7fe18d400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee4829de70 0x55ee4829b180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.022+0000 7fe18de00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829ab70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.022+0000 7fe18de00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829ab70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40512/0 (socket says 172.21.15.28:40512) 2024-05-20T20:18:42.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.022+0000 7fe18de00640 1 -- 172.21.15.28:0/2937291579 learned_addr learned my addr 172.21.15.28:0/2937291579 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:42.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.023+0000 7fe18e800640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1740b29b0 0x7fe1740b4e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.023 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.023+0000 7fe18de00640 1 -- 172.21.15.28:0/2937291579 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 msgr2=0x55ee4829d880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:42.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.025+0000 7fe18de00640 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee4829d880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.025+0000 7fe18de00640 1 -- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee4829de70 msgr2=0x55ee4829b180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:42.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.025+0000 7fe18de00640 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee4829de70 0x55ee4829b180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.025+0000 7fe18de00640 1 -- 172.21.15.28:0/2937291579 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ee47f08210 con 0x55ee482935b0 2024-05-20T20:18:42.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.025+0000 7fe18d400640 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee4829de70 0x55ee4829b180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:42.025 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.026+0000 7fe18de00640 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829ab70 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7fe18000c760 tx=0x7fe1800a9200 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8d594d708e73508d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:42.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.026+0000 7fe16f400640 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:18:42.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.026+0000 7fe18e800640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1740b29b0 0x7fe1740b4e70 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fe188009620 tx=0x7fe18802a8e0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:42.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.026+0000 7fe16f400640 1 -- 172.21.15.28:0/2937291579 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1800aa190 con 0x55ee482935b0 2024-05-20T20:18:42.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.026+0000 7fe16f400640 1 -- 172.21.15.28:0/2937291579 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fe1800e0180 con 0x55ee482935b0 2024-05-20T20:18:42.026 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.027+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe188036040 con 0x55ee482935b0 2024-05-20T20:18:42.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.027+0000 7fe16f400640 1 -- 172.21.15.28:0/2937291579 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe1800e0380 con 0x55ee482935b0 2024-05-20T20:18:42.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.027+0000 7fe1979ba740 10 client.17379 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:42.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.027+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55ee47fd4620 con 0x55ee482935b0 2024-05-20T20:18:42.027 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe16f400640 1 -- 172.21.15.28:0/2937291579 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe188036040 con 0x55ee482935b0 2024-05-20T20:18:42.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe16f400640 1 -- 172.21.15.28:0/2937291579 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe1800a8e10 con 0x55ee482935b0 2024-05-20T20:18:42.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe1979ba740 10 client.17379 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:42.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe1979ba740 10 client.17379 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:42.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe1979ba740 20 client.17379 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:42.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee482be930 0x55ee482c0d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe1979ba740 4 client.17379 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:42.028 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.028+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55ee48205ae0 con 0x55ee482be930 2024-05-20T20:18:42.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.029+0000 7fe18d400640 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee482be930 0x55ee482c0d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.029+0000 7fe18d400640 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee482be930 0x55ee482c0d20 crc :-1 s=READY pgs=263 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=511c54f0f1a93a76 server_cookie=18af4d2f572bf934 in_seq=0 out_seq=0 2024-05-20T20:18:42.029 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.029+0000 7fe16f400640 10 client.17379 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:42.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.030+0000 7fe16f400640 1 -- 172.21.15.28:0/2937291579 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55ee48205ae0 con 0x55ee482be930 2024-05-20T20:18:42.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.030+0000 7fe16f400640 10 client.17379 handle_command_reply: tid=0 2024-05-20T20:18:42.030 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.031+0000 7fe1979ba740 1 client.17379 shutdown 2024-05-20T20:18:42.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.031+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 msgr2=0x55ee4829ab70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.031+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829ab70 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7fe18000c760 tx=0x7fe1800a9200 comp rx=0 tx=0).stop 2024-05-20T20:18:42.031 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 shutdown_connections 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee482be930 0x55ee482c0d20 crc :-1 s=READY pgs=263 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee4829de70 0x55ee4829b180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829ab70 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2937291579 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee4829d880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe16f400640 0 client.17379 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 >> 172.21.15.28:0/2937291579 conn(0x55ee4828f1a0 msgr2=0x55ee48294ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 shutdown_connections 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 1 -- 172.21.15.28:0/2937291579 wait complete. 2024-05-20T20:18:42.032 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.032+0000 7fe1979ba740 20 client.17379 trim_cache size 0 max 16384 2024-05-20T20:18:42.054 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.052+0000 7fe1979ba740 1 Processor -- start 2024-05-20T20:18:42.054 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.052+0000 7fe1979ba740 1 -- start start 2024-05-20T20:18:42.054 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.053+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee48292c90 0x55ee4828e550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.053+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829d3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.053+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee4829de70 0x55ee482c6580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.053+0000 7fe1979ba740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe1901bee90 con 0x55ee482935b0 2024-05-20T20:18:42.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.053+0000 7fe1979ba740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe1901c78e0 con 0x55ee48292c90 2024-05-20T20:18:42.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.053+0000 7fe1979ba740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe1901c6470 con 0x55ee4829de70 2024-05-20T20:18:42.055 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee4829de70 0x55ee482c6580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee4829de70 0x55ee482c6580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56592/0 (socket says 172.21.15.28:56592) 2024-05-20T20:18:42.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 -- 172.21.15.28:0/1482163218 learned_addr learned my addr 172.21.15.28:0/1482163218 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:42.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18d400640 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829d3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 -- 172.21.15.28:0/1482163218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee48292c90 msgr2=0x55ee4828e550 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:18:42.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee48292c90 0x55ee4828e550 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.056 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 -- 172.21.15.28:0/1482163218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 msgr2=0x55ee4829d3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:42.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829d3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.054+0000 7fe18e800640 1 -- 172.21.15.28:0/1482163218 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55ee482c1350 con 0x55ee4829de70 2024-05-20T20:18:42.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.056+0000 7fe18e800640 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee4829de70 0x55ee482c6580 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7fe188045000 tx=0x7fe1880044d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=551355d57c846e14 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:42.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.056+0000 7fe17a600640 1 -- 172.21.15.28:0/1482163218 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe188003e00 con 0x55ee4829de70 2024-05-20T20:18:42.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.056+0000 7fe17a600640 1 -- 172.21.15.28:0/1482163218 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe188036040 con 0x55ee4829de70 2024-05-20T20:18:42.057 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.056+0000 7fe17a600640 1 -- 172.21.15.28:0/1482163218 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe188045560 con 0x55ee4829de70 2024-05-20T20:18:42.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.057+0000 7fe1979ba740 1 -- 172.21.15.28:0/1482163218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee4829de70 msgr2=0x55ee482c6580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.057+0000 7fe1979ba740 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee4829de70 0x55ee482c6580 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7fe188045000 tx=0x7fe1880044d0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.058+0000 7fe1979ba740 1 -- 172.21.15.28:0/1482163218 shutdown_connections 2024-05-20T20:18:42.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.058+0000 7fe1979ba740 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee4829de70 0x55ee482c6580 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.058+0000 7fe1979ba740 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee482935b0 0x55ee4829d3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.058+0000 7fe1979ba740 1 --2- 172.21.15.28:0/1482163218 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee48292c90 0x55ee4828e550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.058 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.058+0000 7fe1979ba740 1 -- 172.21.15.28:0/1482163218 >> 172.21.15.28:0/1482163218 conn(0x55ee4828f1a0 msgr2=0x55ee48294ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:42.059 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.058+0000 7fe1979ba740 1 -- 172.21.15.28:0/1482163218 shutdown_connections 2024-05-20T20:18:42.059 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.058+0000 7fe1979ba740 1 -- 172.21.15.28:0/1482163218 wait complete. 2024-05-20T20:18:42.059 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.059+0000 7fe1979ba740 1 -- start start 2024-05-20T20:18:42.059 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.059+0000 7fe1979ba740 1 Processor -- start 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee482b2a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 0x55ee482b2f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe1979ba740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 0x55ee482b3490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe1979ba740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe1901c7210 con 0x55ee4829de70 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe1979ba740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe1901c6b40 con 0x55ee482935b0 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe1979ba740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe1900507e0 con 0x55ee48292c90 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe1979ba740 10 client.0 _ll_register_callbacks cb 0x7fe180004560 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe18de00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee482b2a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe18de00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee482b2a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56604/0 (socket says 172.21.15.28:56604) 2024-05-20T20:18:42.060 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe18de00640 1 -- 172.21.15.28:0/2896709217 learned_addr learned my addr 172.21.15.28:0/2896709217 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:42.061 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.060+0000 7fe18d400640 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 0x55ee482b2f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.061+0000 7fe18de00640 1 -- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 msgr2=0x55ee482b2f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.061+0000 7fe18de00640 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 0x55ee482b2f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.061+0000 7fe18de00640 1 -- 172.21.15.28:0/2896709217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 msgr2=0x55ee482b3490 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.061+0000 7fe18de00640 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 0x55ee482b3490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.061+0000 7fe18de00640 1 -- 172.21.15.28:0/2896709217 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55ee482bd4d0 con 0x55ee48292c90 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.061+0000 7fe18de00640 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee482b2a10 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fe180009920 tx=0x7fe1800a9330 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1964655936e27412 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.061+0000 7fe16f400640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.062+0000 7fe16f400640 1 -- 172.21.15.28:0/2896709217 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe1800ade40 con 0x55ee48292c90 2024-05-20T20:18:42.062 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.062+0000 7fe16f400640 1 -- 172.21.15.28:0/2896709217 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fe1800a8f00 con 0x55ee48292c90 2024-05-20T20:18:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.062+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe188036040 con 0x55ee48292c90 2024-05-20T20:18:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.062+0000 7fe16f400640 1 -- 172.21.15.28:0/2896709217 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe1800cc150 con 0x55ee48292c90 2024-05-20T20:18:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.062+0000 7fe1979ba740 10 client.27928 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.062+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55ee48329580 con 0x55ee48292c90 2024-05-20T20:18:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.062+0000 7fe16f400640 1 -- 172.21.15.28:0/2896709217 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe188036040 con 0x55ee48292c90 2024-05-20T20:18:42.063 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.063+0000 7fe16f400640 1 -- 172.21.15.28:0/2896709217 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe180004120 con 0x55ee48292c90 2024-05-20T20:18:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.063+0000 7fe1979ba740 10 client.27928 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.063+0000 7fe1979ba740 10 client.27928 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.063+0000 7fe1979ba740 20 client.27928 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.063+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee48329b20 0x55ee4832bf80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.063+0000 7fe1979ba740 4 client.27928 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:18:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.063+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55ee481ff2c0 con 0x55ee48329b20 2024-05-20T20:18:42.064 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.064+0000 7fe18d400640 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee48329b20 0x55ee4832bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:42.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.065+0000 7fe18d400640 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee48329b20 0x55ee4832bf80 crc :-1 s=READY pgs=264 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e1edd65d037547c4 server_cookie=2c8939df9848ca16 in_seq=0 out_seq=0 2024-05-20T20:18:42.065 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.065+0000 7fe16f400640 10 client.27928 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:42.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.114+0000 7fe16f400640 1 -- 172.21.15.28:0/2896709217 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55ee481ff2c0 con 0x55ee48329b20 2024-05-20T20:18:42.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.114+0000 7fe16f400640 10 client.27928 handle_command_reply: tid=0 2024-05-20T20:18:42.116 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.114+0000 7fe1979ba740 1 client.27928 shutdown 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 msgr2=0x55ee482b2a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee482b2a10 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fe180009920 tx=0x7fe1800a9330 comp rx=0 tx=0).stop 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 shutdown_connections 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55ee48329b20 0x55ee4832bf80 crc :-1 s=READY pgs=264 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55ee482935b0 0x55ee482b2f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe16f400640 0 client.27928 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55ee4829de70 0x55ee482b3490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 --2- 172.21.15.28:0/2896709217 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55ee48292c90 0x55ee482b2a10 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.117 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 >> 172.21.15.28:0/2896709217 conn(0x55ee4828f1a0 msgr2=0x55ee482990c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:42.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.115+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 shutdown_connections 2024-05-20T20:18:42.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.116+0000 7fe1979ba740 1 -- 172.21.15.28:0/2896709217 wait complete. 2024-05-20T20:18:42.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.116+0000 7fe1979ba740 20 client.27928 trim_cache size 0 max 16384 2024-05-20T20:18:42.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:18:42.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:18:42.118 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:18:42.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.117+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1740b29b0 msgr2=0x7fe1740b4e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.117+0000 7fe194a00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1740b29b0 0x7fe1740b4e70 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fe188009620 tx=0x7fe18802a8e0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.117+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010e3b0 msgr2=0x7fe1900791f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:42.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.117+0000 7fe194a00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010e3b0 0x7fe1900791f0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7fe18000ea60 tx=0x7fe18000ef20 comp rx=0 tx=0).stop 2024-05-20T20:18:42.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.118+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 shutdown_connections 2024-05-20T20:18:42.119 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.118+0000 7fe194a00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe19010ef70 0x7fe190079730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.118+0000 7fe194a00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe1740b29b0 0x7fe1740b4e70 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.118+0000 7fe194a00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe19007ae90 0x7fe1901c53d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.118+0000 7fe194a00640 1 --2- 172.21.15.28:0/1074586 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe19010e3b0 0x7fe1900791f0 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:42.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.118+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 >> 172.21.15.28:0/1074586 conn(0x7fe19006db20 msgr2=0x7fe190073540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:42.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.119+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 shutdown_connections 2024-05-20T20:18:42.120 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:42.119+0000 7fe194a00640 1 -- 172.21.15.28:0/1074586 wait complete. 2024-05-20T20:18:43.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:43 smithi028 ceph-mon[31178]: pgmap v409: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 270 MiB/s wr, 1.66k op/s 2024-05-20T20:18:43.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:43 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:18:43.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:43 smithi154 ceph-mon[36314]: pgmap v409: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 270 MiB/s wr, 1.66k op/s 2024-05-20T20:18:43.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:43 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:18:43.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:43 smithi033 ceph-mon[36179]: pgmap v409: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 270 MiB/s wr, 1.66k op/s 2024-05-20T20:18:43.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:43 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:18:44.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.782+0000 7f1b82c00700 1 -- 172.21.15.28:0/2939035623 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c109dd0 msgr2=0x7f1b7c10a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.782+0000 7f1b82c00700 1 --2- 172.21.15.28:0/2939035623 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c109dd0 0x7f1b7c10a1b0 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f1b64009990 tx=0x7f1b64009ca0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.782+0000 7f1b82c00700 1 -- 172.21.15.28:0/2939035623 shutdown_connections 2024-05-20T20:18:44.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.782+0000 7f1b82c00700 1 --2- 172.21.15.28:0/2939035623 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1b7c10eeb0 0x7f1b7c116a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.782+0000 7f1b82c00700 1 --2- 172.21.15.28:0/2939035623 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c109dd0 0x7f1b7c10a1b0 secure :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f1b64009990 tx=0x7f1b64009ca0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.782+0000 7f1b82c00700 1 --2- 172.21.15.28:0/2939035623 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1b7c10a780 0x7f1b7c10e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.782+0000 7f1b82c00700 1 -- 172.21.15.28:0/2939035623 >> 172.21.15.28:0/2939035623 conn(0x7f1b7c076d10 msgr2=0x7f1b7c079130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.784+0000 7f1b82c00700 1 -- 172.21.15.28:0/2939035623 shutdown_connections 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.784+0000 7f1b82c00700 1 -- 172.21.15.28:0/2939035623 wait complete. 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.786+0000 7f1b82c00700 1 Processor -- start 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.786+0000 7f1b82c00700 1 -- start start 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.787+0000 7f1b82c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1b7c10a780 0x7f1b7c1a0520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.787+0000 7f1b82c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1b7c10eeb0 0x7f1b7c1a0a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.787+0000 7f1b82c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c1a4df0 0x7f1b7c1a5270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.787+0000 7f1b82c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1b7c119e30 con 0x7f1b7c10eeb0 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.787+0000 7f1b82c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1b7c119cb0 con 0x7f1b7c1a4df0 2024-05-20T20:18:44.788 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.787+0000 7f1b82c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1b7c119fb0 con 0x7f1b7c10a780 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.788+0000 7f1b80c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c1a4df0 0x7f1b7c1a5270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.788+0000 7f1b80c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c1a4df0 0x7f1b7c1a5270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56076/0 (socket says 172.21.15.28:56076) 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.788+0000 7f1b80c00700 1 -- 172.21.15.28:0/1419967869 learned_addr learned my addr 172.21.15.28:0/1419967869 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.789+0000 7f1b7be00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1b7c10a780 0x7f1b7c1a0520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.789+0000 7f1b7b400700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1b7c10eeb0 0x7f1b7c1a0a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.789+0000 7f1b7be00700 1 -- 172.21.15.28:0/1419967869 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c1a4df0 msgr2=0x7f1b7c1a5270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.790+0000 7f1b7be00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c1a4df0 0x7f1b7c1a5270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.790+0000 7f1b7be00700 1 -- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1b7c10eeb0 msgr2=0x7f1b7c1a0a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.790+0000 7f1b7be00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1b7c10eeb0 0x7f1b7c1a0a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.790+0000 7f1b7be00700 1 -- 172.21.15.28:0/1419967869 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1b7c1a5910 con 0x7f1b7c10a780 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.790+0000 7f1b7b400700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1b7c10eeb0 0x7f1b7c1a0a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:44.792 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.790+0000 7f1b7be00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1b7c10a780 0x7f1b7c1a0520 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f1b64008000 tx=0x7f1b64009f50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:44.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.792+0000 7f1b78c00700 1 -- 172.21.15.28:0/1419967869 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b64003af0 con 0x7f1b7c10a780 2024-05-20T20:18:44.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.792+0000 7f1b78c00700 1 -- 172.21.15.28:0/1419967869 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b64004140 con 0x7f1b7c10a780 2024-05-20T20:18:44.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.792+0000 7f1b78c00700 1 -- 172.21.15.28:0/1419967869 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b6400fcd0 con 0x7f1b7c10a780 2024-05-20T20:18:44.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.792+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1b7c1a5ba0 con 0x7f1b7c10a780 2024-05-20T20:18:44.793 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.792+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1b7c06d130 con 0x7f1b7c10a780 2024-05-20T20:18:44.800 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.798+0000 7f1b78c00700 1 -- 172.21.15.28:0/1419967869 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f1b6400f460 con 0x7f1b7c10a780 2024-05-20T20:18:44.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.801+0000 7f1b78c00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b600b2b40 0x7f1b600b5000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.801 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.802+0000 7f1b78c00700 1 -- 172.21.15.28:0/1419967869 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f1b640f4630 con 0x7f1b7c10a780 2024-05-20T20:18:44.802 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.803+0000 7f1b7b400700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b600b2b40 0x7f1b600b5000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.803 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.804+0000 7f1b7b400700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b600b2b40 0x7f1b600b5000 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f1b7c077230 tx=0x7f1b6c00b410 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:44.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.806+0000 7f1b8d567b80 1 Processor -- start 2024-05-20T20:18:44.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.806+0000 7f1b8d567b80 1 -- start start 2024-05-20T20:18:44.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.807+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e396db80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.807+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e3967bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.807+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e39685c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.807+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e1e38b4ec0 con 0x55e1e3968140 2024-05-20T20:18:44.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.807+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e1e38a4a60 con 0x55e1e396d7a0 2024-05-20T20:18:44.807 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.807+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e1e38a8aa0 con 0x55e1e396e0c0 2024-05-20T20:18:44.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e396db80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e396db80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56078/0 (socket says 172.21.15.28:56078) 2024-05-20T20:18:44.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 -- 172.21.15.28:0/809340305 learned_addr learned my addr 172.21.15.28:0/809340305 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:44.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b80c00700 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e39685c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 -- 172.21.15.28:0/809340305 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 msgr2=0x55e1e3967bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e3967bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.808 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 -- 172.21.15.28:0/809340305 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 msgr2=0x55e1e39685c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e39685c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 -- 172.21.15.28:0/809340305 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e1e386d4a0 con 0x55e1e396d7a0 2024-05-20T20:18:44.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.808+0000 7f1b7be00700 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e396db80 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f1b640bb690 tx=0x7f1b64009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3456c9c50a56daee server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:44.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.809+0000 7f1b71600700 1 -- 172.21.15.28:0/809340305 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b640c40b0 con 0x55e1e396d7a0 2024-05-20T20:18:44.809 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.809+0000 7f1b71600700 1 -- 172.21.15.28:0/809340305 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b640053d0 con 0x55e1e396d7a0 2024-05-20T20:18:44.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.809+0000 7f1b71600700 1 -- 172.21.15.28:0/809340305 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b640bfd40 con 0x55e1e396d7a0 2024-05-20T20:18:44.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 -- 172.21.15.28:0/809340305 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 msgr2=0x55e1e396db80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e396db80 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f1b640bb690 tx=0x7f1b64009ca0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 -- 172.21.15.28:0/809340305 shutdown_connections 2024-05-20T20:18:44.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e39685c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e396db80 unknown :-1 s=CLOSED pgs=609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/809340305 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e3967bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 -- 172.21.15.28:0/809340305 >> 172.21.15.28:0/809340305 conn(0x55e1e3884c60 msgr2=0x55e1e3951710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:44.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 -- 172.21.15.28:0/809340305 shutdown_connections 2024-05-20T20:18:44.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.811+0000 7f1b8d567b80 1 -- 172.21.15.28:0/809340305 wait complete. 2024-05-20T20:18:44.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.812+0000 7f1b8d567b80 1 -- start start 2024-05-20T20:18:44.812 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.812+0000 7f1b8d567b80 1 Processor -- start 2024-05-20T20:18:44.813 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e3968140 0x55e1e3973ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396d7a0 0x55e1e3971270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 0x55e1e39717b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55e1e37ab020 con 0x55e1e396e0c0 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55e1e38457e0 con 0x55e1e3968140 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55e1e39748a0 con 0x55e1e396d7a0 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b8d567b80 10 client.0 _ll_register_callbacks cb 0x55e1e384ecf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b7be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e3968140 0x55e1e3973ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.815 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b7be00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e3968140 0x55e1e3973ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56090/0 (socket says 172.21.15.28:56090) 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.813+0000 7f1b7be00700 1 -- 172.21.15.28:0/888501641 learned_addr learned my addr 172.21.15.28:0/888501641 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.814+0000 7f1b7b400700 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396d7a0 0x55e1e3971270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.814+0000 7f1b7be00700 1 -- 172.21.15.28:0/888501641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396d7a0 msgr2=0x55e1e3971270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.814+0000 7f1b7be00700 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396d7a0 0x55e1e3971270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.814+0000 7f1b7be00700 1 -- 172.21.15.28:0/888501641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 msgr2=0x55e1e39717b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.814+0000 7f1b7be00700 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 0x55e1e39717b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.814+0000 7f1b7be00700 1 -- 172.21.15.28:0/888501641 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e1e37f4950 con 0x55e1e3968140 2024-05-20T20:18:44.816 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.815+0000 7f1b7be00700 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e3968140 0x55e1e3973ed0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f1b64006010 tx=0x7f1b640c2330 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f9ba0852e20cb349 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:44.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.815+0000 7f1b6a000700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:44.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.815+0000 7f1b6a000700 1 -- 172.21.15.28:0/888501641 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b640c40b0 con 0x55e1e3968140 2024-05-20T20:18:44.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.816+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55e1e38b6e10 con 0x55e1e3968140 2024-05-20T20:18:44.817 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.816+0000 7f1b6a000700 1 -- 172.21.15.28:0/888501641 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f1b640bbbf0 con 0x55e1e3968140 2024-05-20T20:18:44.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b6a000700 1 -- 172.21.15.28:0/888501641 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1b640bc090 con 0x55e1e3968140 2024-05-20T20:18:44.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b8d567b80 10 client.28007 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:44.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e1e38750e0 con 0x55e1e3968140 2024-05-20T20:18:44.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b6a000700 1 -- 172.21.15.28:0/888501641 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55e1e38b6e10 con 0x55e1e3968140 2024-05-20T20:18:44.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b6a000700 1 -- 172.21.15.28:0/888501641 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1b640c42d0 con 0x55e1e3968140 2024-05-20T20:18:44.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b8d567b80 10 client.28007 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:44.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b8d567b80 10 client.28007 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:44.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b8d567b80 20 client.28007 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:44.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.818+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e3981370 0x55e1e3983760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.819+0000 7f1b8d567b80 4 client.28007 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:44.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.819+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55e1e38b05f0 con 0x55e1e3981370 2024-05-20T20:18:44.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.819+0000 7f1b7b400700 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e3981370 0x55e1e3983760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.820+0000 7f1b7b400700 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e3981370 0x55e1e3983760 crc :-1 s=READY pgs=265 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d35ef534b0f027fa server_cookie=83c1929da3fd4cb8 in_seq=0 out_seq=0 2024-05-20T20:18:44.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.820+0000 7f1b6a000700 10 client.28007 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:44.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b6a000700 1 -- 172.21.15.28:0/888501641 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55e1e38b05f0 con 0x55e1e3981370 2024-05-20T20:18:44.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b6a000700 10 client.28007 handle_command_reply: tid=0 2024-05-20T20:18:44.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 client.28007 shutdown 2024-05-20T20:18:44.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e3968140 msgr2=0x55e1e3973ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e3968140 0x55e1e3973ed0 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f1b64006010 tx=0x7f1b640c2330 comp rx=0 tx=0).stop 2024-05-20T20:18:44.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 shutdown_connections 2024-05-20T20:18:44.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e3981370 0x55e1e3983760 crc :-1 s=READY pgs=265 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 0x55e1e39717b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e3968140 0x55e1e3973ed0 unknown :-1 s=CLOSED pgs=610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/888501641 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396d7a0 0x55e1e3971270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 >> 172.21.15.28:0/888501641 conn(0x55e1e3884c60 msgr2=0x55e1e3950df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:44.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.821+0000 7f1b6a000700 0 client.28007 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:44.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.822+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 shutdown_connections 2024-05-20T20:18:44.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.822+0000 7f1b8d567b80 1 -- 172.21.15.28:0/888501641 wait complete. 2024-05-20T20:18:44.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.822+0000 7f1b8d567b80 20 client.28007 trim_cache size 0 max 16384 2024-05-20T20:18:44.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.860+0000 7f1b8d567b80 1 Processor -- start 2024-05-20T20:18:44.861 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.860+0000 7f1b8d567b80 1 -- start start 2024-05-20T20:18:44.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.862+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e3968140 0x55e1e397d900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.862+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e3951570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.862+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 0x55e1e396c9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.862+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1b5800d620 con 0x55e1e396e0c0 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.862+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1b5800f160 con 0x55e1e396d7a0 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.862+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1b5800dcf0 con 0x55e1e3968140 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.863+0000 7f1b7be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e3968140 0x55e1e397d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.863+0000 7f1b7be00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e3968140 0x55e1e397d900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56662/0 (socket says 172.21.15.28:56662) 2024-05-20T20:18:44.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.863+0000 7f1b7be00700 1 -- 172.21.15.28:0/445409799 learned_addr learned my addr 172.21.15.28:0/445409799 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:44.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.863+0000 7f1b7b400700 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e3951570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.863+0000 7f1b80c00700 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 0x55e1e396c9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.864+0000 7f1b7be00700 1 -- 172.21.15.28:0/445409799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 msgr2=0x55e1e3951570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.864+0000 7f1b7be00700 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e3951570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.864+0000 7f1b7be00700 1 -- 172.21.15.28:0/445409799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 msgr2=0x55e1e396c9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.864+0000 7f1b7be00700 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 0x55e1e396c9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.864+0000 7f1b7be00700 1 -- 172.21.15.28:0/445409799 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55e1e396d3c0 con 0x55e1e3968140 2024-05-20T20:18:44.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.867+0000 7f1b7be00700 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e3968140 0x55e1e397d900 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f1b640bb920 tx=0x7f1b640bb100 comp rx=0 tx=0).ready entity=mon.1 client_cookie=db32f3806f5895fa server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:44.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.868+0000 7f1b70c00700 1 -- 172.21.15.28:0/445409799 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b640c40a0 con 0x55e1e3968140 2024-05-20T20:18:44.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.869+0000 7f1b70c00700 1 -- 172.21.15.28:0/445409799 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b640bb4a0 con 0x55e1e3968140 2024-05-20T20:18:44.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.869+0000 7f1b70c00700 1 -- 172.21.15.28:0/445409799 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b640bbbf0 con 0x55e1e3968140 2024-05-20T20:18:44.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 -- 172.21.15.28:0/445409799 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e3968140 msgr2=0x55e1e397d900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e3968140 0x55e1e397d900 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f1b640bb920 tx=0x7f1b640bb100 comp rx=0 tx=0).stop 2024-05-20T20:18:44.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 -- 172.21.15.28:0/445409799 shutdown_connections 2024-05-20T20:18:44.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e396e0c0 0x55e1e396c9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e3968140 0x55e1e397d900 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/445409799 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e3951570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 -- 172.21.15.28:0/445409799 >> 172.21.15.28:0/445409799 conn(0x55e1e3884c60 msgr2=0x55e1e3956420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:44.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.870+0000 7f1b8d567b80 1 -- 172.21.15.28:0/445409799 shutdown_connections 2024-05-20T20:18:44.871 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.871+0000 7f1b8d567b80 1 -- 172.21.15.28:0/445409799 wait complete. 2024-05-20T20:18:44.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.871+0000 7f1b8d567b80 1 -- start start 2024-05-20T20:18:44.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.872+0000 7f1b8d567b80 1 Processor -- start 2024-05-20T20:18:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.873+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e3977090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.873+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e39775d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.873+0000 7f1b8d567b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e39ebdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.873+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f1b5800ea90 con 0x55e1e3968140 2024-05-20T20:18:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.873+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f1b5800e3c0 con 0x55e1e396d7a0 2024-05-20T20:18:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.873+0000 7f1b8d567b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f1b5800b5e0 con 0x55e1e396e0c0 2024-05-20T20:18:44.876 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.873+0000 7f1b8d567b80 10 client.0 _ll_register_callbacks cb 0x55e1e384ecf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.875+0000 7f1b80c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e39ebdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.875+0000 7f1b80c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e39ebdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56666/0 (socket says 172.21.15.28:56666) 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.875+0000 7f1b80c00700 1 -- 172.21.15.28:0/2158396722 learned_addr learned my addr 172.21.15.28:0/2158396722 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.875+0000 7f1b7be00700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e3977090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.876+0000 7f1b7be00700 1 -- 172.21.15.28:0/2158396722 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 msgr2=0x55e1e39ebdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.876+0000 7f1b7be00700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e39ebdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.876+0000 7f1b7be00700 1 -- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 msgr2=0x55e1e39775d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.876+0000 7f1b7b400700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e39775d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.876+0000 7f1b7be00700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e39775d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.876+0000 7f1b7be00700 1 -- 172.21.15.28:0/2158396722 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55e1e39ec310 con 0x55e1e3968140 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.876+0000 7f1b7b400700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e39775d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-05-20T20:18:44.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.877+0000 7f1b7be00700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e3977090 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f1b64009990 tx=0x7f1b640dcda0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=224dd19d3cc294de server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:44.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.877+0000 7f1b6a000700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:18:44.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.877+0000 7f1b6a000700 1 -- 172.21.15.28:0/2158396722 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1b64098020 con 0x55e1e3968140 2024-05-20T20:18:44.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.877+0000 7f1b6a000700 1 -- 172.21.15.28:0/2158396722 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f1b640bac40 con 0x55e1e3968140 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.877+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f1b640bb4a0 con 0x55e1e3968140 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.878+0000 7f1b6a000700 1 -- 172.21.15.28:0/2158396722 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f1b640c40a0 con 0x55e1e3968140 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.878+0000 7f1b8d567b80 10 client.17394 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.878+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55e1e39ec310 con 0x55e1e3968140 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.878+0000 7f1b6a000700 1 -- 172.21.15.28:0/2158396722 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f1b640bb4a0 con 0x55e1e3968140 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b6a000700 1 -- 172.21.15.28:0/2158396722 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f1b640a3260 con 0x55e1e3968140 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b8d567b80 10 client.17394 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:44.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b8d567b80 10 client.17394 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:44.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b8d567b80 20 client.17394 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:44.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e39ed870 0x55e1e39efc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:44.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b8d567b80 4 client.17394 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-05-20T20:18:44.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) -- 0x55e1e37272d0 con 0x55e1e39ed870 2024-05-20T20:18:44.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.879+0000 7f1b7b400700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e39ed870 0x55e1e39efc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:44.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.880+0000 7f1b7b400700 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e39ed870 0x55e1e39efc60 crc :-1 s=READY pgs=266 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b6a3a34707dd500e server_cookie=523eb51adcce85c4 in_seq=0 out_seq=0 2024-05-20T20:18:44.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.880+0000 7f1b6a000700 10 client.17394 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:44.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.881+0000 7f1b6a000700 1 -- 172.21.15.28:0/2158396722 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+110 (crc 0 0 0) 0x55e1e37272d0 con 0x55e1e39ed870 2024-05-20T20:18:44.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.881+0000 7f1b6a000700 10 client.17394 handle_command_reply: tid=0 2024-05-20T20:18:44.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 client.17394 shutdown 2024-05-20T20:18:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 msgr2=0x55e1e3977090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e3977090 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f1b64009990 tx=0x7f1b640dcda0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 shutdown_connections 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55e1e39ed870 0x55e1e39efc60 crc :-1 s=READY pgs=266 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55e1e396e0c0 0x55e1e39ebdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55e1e3968140 0x55e1e3977090 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 --2- 172.21.15.28:0/2158396722 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55e1e396d7a0 0x55e1e39775d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 >> 172.21.15.28:0/2158396722 conn(0x55e1e3884c60 msgr2=0x55e1e3956420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.882+0000 7f1b6a000700 0 client.17394 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.883+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 shutdown_connections 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.883+0000 7f1b8d567b80 1 -- 172.21.15.28:0/2158396722 wait complete. 2024-05-20T20:18:44.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.883+0000 7f1b8d567b80 20 client.17394 trim_cache size 0 max 16384 2024-05-20T20:18:44.884 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:18:44.884 INFO:teuthology.orchestra.run.smithi028.stdout: "return_code": 0, 2024-05-20T20:18:44.884 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_tag": "a23ad208-9b58-4d60-be67-ea22afa52cb5", 2024-05-20T20:18:44.884 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": "asynchronous" 2024-05-20T20:18:44.884 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.884+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b600b2b40 msgr2=0x7f1b600b5000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.884+0000 7f1b82c00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b600b2b40 0x7f1b600b5000 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f1b7c077230 tx=0x7f1b6c00b410 comp rx=0 tx=0).stop 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.884+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1b7c10a780 msgr2=0x7f1b7c1a0520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.884+0000 7f1b82c00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1b7c10a780 0x7f1b7c1a0520 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f1b64008000 tx=0x7f1b64009f50 comp rx=0 tx=0).stop 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.885+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 shutdown_connections 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.885+0000 7f1b82c00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f1b600b2b40 0x7f1b600b5000 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.885+0000 7f1b82c00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f1b7c1a4df0 0x7f1b7c1a5270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.885+0000 7f1b82c00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f1b7c10a780 0x7f1b7c1a0520 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.885+0000 7f1b82c00700 1 --2- 172.21.15.28:0/1419967869 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f1b7c10eeb0 0x7f1b7c1a0a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:44.885 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.885+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 >> 172.21.15.28:0/1419967869 conn(0x7f1b7c076d10 msgr2=0x7f1b7c078100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:44.886 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.886+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 shutdown_connections 2024-05-20T20:18:44.887 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:44.887+0000 7f1b82c00700 1 -- 172.21.15.28:0/1419967869 wait complete. 2024-05-20T20:18:45.477 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: pgmap v410: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 125 KiB/s rd, 392 MiB/s wr, 2.71k op/s 2024-05-20T20:18:45.728 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:18:45.728 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:18:45.728 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:18:45.728 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: scrub queued for path: / 2024-05-20T20:18:45.728 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:18:45.728 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: scrub summary: active paths [/] 2024-05-20T20:18:45.728 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:45 smithi028 ceph-mon[31178]: scrub summary: idle+waiting paths [/] 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: pgmap v410: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 125 KiB/s rd, 392 MiB/s wr, 2.71k op/s 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: scrub queued for path: / 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: scrub summary: active paths [/] 2024-05-20T20:18:45.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:45 smithi154 ceph-mon[36314]: scrub summary: idle+waiting paths [/] 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: pgmap v410: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 125 KiB/s rd, 392 MiB/s wr, 2.71k op/s 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: scrub queued for path: / 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: scrub summary: active paths [/] 2024-05-20T20:18:45.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:45 smithi033 ceph-mon[36179]: scrub summary: idle+waiting paths [/] 2024-05-20T20:18:46.298 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 scrub status 2024-05-20T20:18:46.360 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.968+0000 7f0c49400700 1 -- 172.21.15.28:0/12451438 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 msgr2=0x7f0c44112330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.968+0000 7f0c49400700 1 --2- 172.21.15.28:0/12451438 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 0x7f0c44112330 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f0c3c00b0a0 tx=0x7f0c3c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.969+0000 7f0c49400700 1 -- 172.21.15.28:0/12451438 shutdown_connections 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.969+0000 7f0c49400700 1 --2- 172.21.15.28:0/12451438 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 0x7f0c44112330 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.969+0000 7f0c49400700 1 --2- 172.21.15.28:0/12451438 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 0x7f0c44075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.969+0000 7f0c49400700 1 --2- 172.21.15.28:0/12451438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c44076090 0x7f0c4410fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.969+0000 7f0c49400700 1 -- 172.21.15.28:0/12451438 >> 172.21.15.28:0/12451438 conn(0x7f0c4407a6a0 msgr2=0x7f0c4407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.969+0000 7f0c49400700 1 -- 172.21.15.28:0/12451438 shutdown_connections 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.970+0000 7f0c49400700 1 -- 172.21.15.28:0/12451438 wait complete. 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.971+0000 7f0c49400700 1 Processor -- start 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.971+0000 7f0c49400700 1 -- start start 2024-05-20T20:18:46.361 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c49400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 0x7f0c4419be30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c49400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c44076090 0x7f0c4419c370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c49400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 0x7f0c441a0700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c49400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f0c441159c0 con 0x7f0c4410ff40 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c49400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f0c44115840 con 0x7f0c44075770 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c49400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f0c44115b40 con 0x7f0c44076090 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c42800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 0x7f0c4419be30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c42800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 0x7f0c4419be30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:56132/0 (socket says 172.21.15.28:56132) 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c42800700 1 -- 172.21.15.28:0/2145147813 learned_addr learned my addr 172.21.15.28:0/2145147813 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.972+0000 7f0c41e00700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c44076090 0x7f0c4419c370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.973+0000 7f0c43200700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 0x7f0c441a0700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.973+0000 7f0c42800700 1 -- 172.21.15.28:0/2145147813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c44076090 msgr2=0x7f0c4419c370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.973+0000 7f0c42800700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c44076090 0x7f0c4419c370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.973+0000 7f0c42800700 1 -- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 msgr2=0x7f0c441a0700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.973+0000 7f0c42800700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 0x7f0c441a0700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.973+0000 7f0c42800700 1 -- 172.21.15.28:0/2145147813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c441a0e40 con 0x7f0c44075770 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.973+0000 7f0c42800700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 0x7f0c4419be30 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f0c3400b530 tx=0x7f0c3400b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.974+0000 7f0c3b400700 1 -- 172.21.15.28:0/2145147813 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c34015020 con 0x7f0c44075770 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.974+0000 7f0c3b400700 1 -- 172.21.15.28:0/2145147813 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f0c34011ce0 con 0x7f0c44075770 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.974+0000 7f0c3b400700 1 -- 172.21.15.28:0/2145147813 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c34010e40 con 0x7f0c44075770 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.974+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c441a1130 con 0x7f0c44075770 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.974+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c441a89b0 con 0x7f0c44075770 2024-05-20T20:18:46.363 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.975+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c4406a230 con 0x7f0c44075770 2024-05-20T20:18:46.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.978+0000 7f0c3b400700 1 -- 172.21.15.28:0/2145147813 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f0c34047d10 con 0x7f0c44075770 2024-05-20T20:18:46.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.979+0000 7f0c3b400700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2aa0 0x7f0c280b4f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:46.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.979+0000 7f0c3b400700 1 -- 172.21.15.28:0/2145147813 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f0c340e2360 con 0x7f0c44075770 2024-05-20T20:18:46.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.980+0000 7f0c41e00700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2aa0 0x7f0c280b4f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:46.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.981+0000 7f0c3b400700 1 -- 172.21.15.28:0/2145147813 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f0c3401f020 con 0x7f0c44075770 2024-05-20T20:18:46.364 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:45.983+0000 7f0c41e00700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2aa0 0x7f0c280b4f60 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f0c44111b90 tx=0x7f0c3000a300 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:46.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.502+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f0c4419d190 con 0x7f0c44075770 2024-05-20T20:18:46.503 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.503+0000 7f0c3b400700 1 -- 172.21.15.28:0/2145147813 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f0c340a9de0 con 0x7f0c44075770 2024-05-20T20:18:46.503 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:18:46.503 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:18:46.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.506+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2aa0 msgr2=0x7f0c280b4f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:46.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.506+0000 7f0c49400700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2aa0 0x7f0c280b4f60 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f0c44111b90 tx=0x7f0c3000a300 comp rx=0 tx=0).stop 2024-05-20T20:18:46.506 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.506+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 msgr2=0x7f0c4419be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.506+0000 7f0c49400700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 0x7f0c4419be30 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f0c3400b530 tx=0x7f0c3400b840 comp rx=0 tx=0).stop 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.506+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 shutdown_connections 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.506+0000 7f0c49400700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f0c4410ff40 0x7f0c441a0700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.507+0000 7f0c49400700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f0c280b2aa0 0x7f0c280b4f60 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.507+0000 7f0c49400700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f0c44075770 0x7f0c4419be30 secure :-1 s=CLOSED pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f0c3400b530 tx=0x7f0c3400b840 comp rx=0 tx=0).stop 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.507+0000 7f0c49400700 1 --2- 172.21.15.28:0/2145147813 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0c44076090 0x7f0c4419c370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.507+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 >> 172.21.15.28:0/2145147813 conn(0x7f0c4407a6a0 msgr2=0x7f0c441006b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.507+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 shutdown_connections 2024-05-20T20:18:46.507 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:46.507+0000 7f0c49400700 1 -- 172.21.15.28:0/2145147813 wait complete. 2024-05-20T20:18:46.508 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:18:46.900 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:46 smithi028 ceph-mon[31178]: pgmap v411: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 227 MiB/s wr, 1.83k op/s 2024-05-20T20:18:46.900 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:46 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2145147813' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:46 smithi154 ceph-mon[36314]: pgmap v411: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 227 MiB/s wr, 1.83k op/s 2024-05-20T20:18:47.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:46 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2145147813' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:47.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:46 smithi033 ceph-mon[36179]: pgmap v411: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 227 MiB/s wr, 1.83k op/s 2024-05-20T20:18:47.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:46 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2145147813' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:47.359 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --set-id ec1db510 --release --await' 2024-05-20T20:18:47.359 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --set-id ec1db510 --release --await 2024-05-20T20:18:47.409 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:48.237 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:48 smithi028 ceph-mon[31178]: scrub summary: idle 2024-05-20T20:18:48.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:48 smithi154 ceph-mon[36314]: scrub summary: idle 2024-05-20T20:18:48.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:48 smithi033 ceph-mon[36179]: scrub summary: idle 2024-05-20T20:18:49.391 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:49.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:49 smithi028 ceph-mon[31178]: pgmap v412: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 231 MiB/s wr, 1.87k op/s 2024-05-20T20:18:49.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:49 smithi154 ceph-mon[36314]: pgmap v412: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 231 MiB/s wr, 1.87k op/s 2024-05-20T20:18:49.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:49 smithi033 ceph-mon[36179]: pgmap v412: 97 pgs: 97 active+clean; 62 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 231 MiB/s wr, 1.87k op/s 2024-05-20T20:18:50.519 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 -- 172.21.15.28:0/2665034224 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbb041041a0 msgr2=0x7fbb041081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2665034224 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbb041041a0 0x7fbb041081c0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7fbaf0009920 tx=0x7fbaf0009c30 comp rx=0 tx=0).stop 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 -- 172.21.15.28:0/2665034224 shutdown_connections 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2665034224 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbb041088f0 0x7fbb04110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2665034224 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbb041037f0 0x7fbb04103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2665034224 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbb041041a0 0x7fbb041081c0 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 -- 172.21.15.28:0/2665034224 >> 172.21.15.28:0/2665034224 conn(0x7fbb0407a6a0 msgr2=0x7fbb0407aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 -- 172.21.15.28:0/2665034224 shutdown_connections 2024-05-20T20:18:50.520 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.518+0000 7fbb09e00700 1 -- 172.21.15.28:0/2665034224 wait complete. 2024-05-20T20:18:50.521 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 Processor -- start 2024-05-20T20:18:50.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 -- start start 2024-05-20T20:18:50.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbb041037f0 0x7fbb0419be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbb041041a0 0x7fbb0419c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbb041088f0 0x7fbb041a06f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbb041136b0 con 0x7fbb041037f0 2024-05-20T20:18:50.523 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fbb04113530 con 0x7fbb041041a0 2024-05-20T20:18:50.524 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.521+0000 7fbb09e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fbb04113830 con 0x7fbb041088f0 2024-05-20T20:18:50.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb02a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbb041037f0 0x7fbb0419be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb03400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbb041088f0 0x7fbb041a06f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.525 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb02a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbb041037f0 0x7fbb0419be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57944/0 (socket says 172.21.15.28:57944) 2024-05-20T20:18:50.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb02a00700 1 -- 172.21.15.28:0/2094010689 learned_addr learned my addr 172.21.15.28:0/2094010689 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:50.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb03400700 1 -- 172.21.15.28:0/2094010689 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbb041041a0 msgr2=0x7fbb0419c360 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-05-20T20:18:50.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb03400700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbb041041a0 0x7fbb0419c360 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb03400700 1 -- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbb041037f0 msgr2=0x7fbb0419be20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb03400700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbb041037f0 0x7fbb0419be20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.522+0000 7fbb03400700 1 -- 172.21.15.28:0/2094010689 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbb041a0e30 con 0x7fbb041088f0 2024-05-20T20:18:50.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.523+0000 7fbb03400700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbb041088f0 0x7fbb041a06f0 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fbafc00efc0 tx=0x7fbafc00c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:50.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.523+0000 7fbb08a00700 1 -- 172.21.15.28:0/2094010689 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbafc0092e0 con 0x7fbb041088f0 2024-05-20T20:18:50.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.523+0000 7fbb08a00700 1 -- 172.21.15.28:0/2094010689 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbafc004bb0 con 0x7fbb041088f0 2024-05-20T20:18:50.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.523+0000 7fbb08a00700 1 -- 172.21.15.28:0/2094010689 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbafc004280 con 0x7fbb041088f0 2024-05-20T20:18:50.527 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.523+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbb041a10c0 con 0x7fbb041088f0 2024-05-20T20:18:50.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.523+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fbb0406cf60 con 0x7fbb041088f0 2024-05-20T20:18:50.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.529+0000 7fbb146feb80 1 Processor -- start 2024-05-20T20:18:50.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.529+0000 7fbb08a00700 1 -- 172.21.15.28:0/2094010689 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fbafc01b030 con 0x7fbb041088f0 2024-05-20T20:18:50.531 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.530+0000 7fbb146feb80 1 -- start start 2024-05-20T20:18:50.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.531+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f70446eeb0 0x55f70446f290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.533 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.531+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704469260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.531+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704469ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.531+0000 7fbb146feb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55f7043c7e40 con 0x55f704469880 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.531+0000 7fbb146feb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55f7043b79e0 con 0x55f70446eeb0 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.531+0000 7fbb146feb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55f7043bba20 con 0x55f70446f7d0 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.531+0000 7fbb02000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f70446eeb0 0x55f70446f290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f70446eeb0 0x55f70446f290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53540/0 (socket says 172.21.15.28:53540) 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02000700 1 -- 172.21.15.28:0/337827583 learned_addr learned my addr 172.21.15.28:0/337827583 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02000700 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704469ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704469260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02000700 1 -- 172.21.15.28:0/337827583 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 msgr2=0x55f704469260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02000700 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704469260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb08a00700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbae80b2a70 0x7fbae80b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02000700 1 -- 172.21.15.28:0/337827583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 msgr2=0x55f704469ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.532+0000 7fbb02000700 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704469ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.533+0000 7fbb02000700 1 -- 172.21.15.28:0/337827583 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f704380420 con 0x55f70446eeb0 2024-05-20T20:18:50.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.533+0000 7fbb03400700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbae80b2a70 0x7fbae80b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.533+0000 7fbb02000700 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f70446eeb0 0x55f70446f290 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7fbaf0005bf0 tx=0x7fbaf0005b60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4169f58f214a7208 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:50.536 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.534+0000 7fbb08a00700 1 -- 172.21.15.28:0/2094010689 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fbafc0e20c0 con 0x7fbb041088f0 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.534+0000 7fbb03400700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbae80b2a70 0x7fbae80b4f30 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fbafc0df8b0 tx=0x7fbafc0e5920 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.534+0000 7fbaf8c00700 1 -- 172.21.15.28:0/337827583 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbaf000fbc0 con 0x55f70446eeb0 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.534+0000 7fbaf8c00700 1 -- 172.21.15.28:0/337827583 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbaf000fd60 con 0x55f70446eeb0 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.534+0000 7fbaf8c00700 1 -- 172.21.15.28:0/337827583 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbaf0003680 con 0x55f70446eeb0 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.535+0000 7fbb146feb80 1 -- 172.21.15.28:0/337827583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f70446eeb0 msgr2=0x55f70446f290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.535+0000 7fbb146feb80 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f70446eeb0 0x55f70446f290 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7fbaf0005bf0 tx=0x7fbaf0005b60 comp rx=0 tx=0).stop 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.535+0000 7fbb146feb80 1 -- 172.21.15.28:0/337827583 shutdown_connections 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.535+0000 7fbb146feb80 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704469ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.535+0000 7fbb146feb80 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f70446eeb0 0x55f70446f290 unknown :-1 s=CLOSED pgs=613 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.535+0000 7fbb146feb80 1 --2- 172.21.15.28:0/337827583 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704469260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.537 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.535+0000 7fbb146feb80 1 -- 172.21.15.28:0/337827583 >> 172.21.15.28:0/337827583 conn(0x55f704397be0 msgr2=0x55f704464010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:50.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.536+0000 7fbb146feb80 1 -- 172.21.15.28:0/337827583 shutdown_connections 2024-05-20T20:18:50.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.536+0000 7fbb146feb80 1 -- 172.21.15.28:0/337827583 wait complete. 2024-05-20T20:18:50.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.536+0000 7fbb146feb80 1 -- start start 2024-05-20T20:18:50.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 1 Processor -- start 2024-05-20T20:18:50.538 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704469880 0x55f7044754e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446eeb0 0x55f704472880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55f7042bdfa0 con 0x55f70446f7d0 2024-05-20T20:18:50.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55f704358760 con 0x55f704469880 2024-05-20T20:18:50.539 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55f7044761e0 con 0x55f70446eeb0 2024-05-20T20:18:50.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.537+0000 7fbb146feb80 10 client.0 _ll_register_callbacks cb 0x55f704361c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:50.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.538+0000 7fbb02000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.538+0000 7fbb02000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57962/0 (socket says 172.21.15.28:57962) 2024-05-20T20:18:50.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.538+0000 7fbb02000700 1 -- 172.21.15.28:0/2502789970 learned_addr learned my addr 172.21.15.28:0/2502789970 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:50.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.538+0000 7fbb02a00700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446eeb0 0x55f704472880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.538+0000 7fbb02000700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704469880 0x55f7044754e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.540 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbb02a00700 1 -- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704469880 msgr2=0x55f7044754e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbb02a00700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704469880 0x55f7044754e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbb02a00700 1 -- 172.21.15.28:0/2502789970 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 msgr2=0x55f704472dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbb02a00700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbb02a00700 1 -- 172.21.15.28:0/2502789970 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f7043078d0 con 0x55f70446eeb0 2024-05-20T20:18:50.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbb02000700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:50.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbb02a00700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446eeb0 0x55f704472880 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fbaf4009930 tx=0x7fbaf4012af0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bb2b1abaa926383c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:50.541 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.539+0000 7fbaed600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:50.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.540+0000 7fbaed600700 1 -- 172.21.15.28:0/2502789970 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbaf400f480 con 0x55f70446eeb0 2024-05-20T20:18:50.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.540+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbaf000fd60 con 0x55f70446eeb0 2024-05-20T20:18:50.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.540+0000 7fbaed600700 1 -- 172.21.15.28:0/2502789970 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fbaf400f620 con 0x55f70446eeb0 2024-05-20T20:18:50.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.540+0000 7fbaed600700 1 -- 172.21.15.28:0/2502789970 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fbaf400f820 con 0x55f70446eeb0 2024-05-20T20:18:50.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.540+0000 7fbb146feb80 10 client.27955 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:50.542 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.540+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55f704388060 con 0x55f70446eeb0 2024-05-20T20:18:50.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbaed600700 1 -- 172.21.15.28:0/2502789970 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbaf000fd60 con 0x55f70446eeb0 2024-05-20T20:18:50.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbaed600700 1 -- 172.21.15.28:0/2502789970 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fbaf4026a70 con 0x55f70446eeb0 2024-05-20T20:18:50.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbb146feb80 10 client.27955 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:50.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbb146feb80 10 client.27955 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:50.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbb146feb80 20 client.27955 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:50.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f70448b4c0 0x55f70448d8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbb146feb80 4 client.27955 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:50.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.541+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55f7043c9d90 con 0x55f70448b4c0 2024-05-20T20:18:50.544 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.542+0000 7fbb02000700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f70448b4c0 0x55f70448d8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.545 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.545+0000 7fbb02000700 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f70448b4c0 0x55f70448d8e0 crc :-1 s=READY pgs=267 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=35440249b2d54154 server_cookie=aeb70e1cad256a24 in_seq=0 out_seq=0 2024-05-20T20:18:50.546 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.545+0000 7fbaed600700 10 client.27955 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:50.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbaed600700 1 -- 172.21.15.28:0/2502789970 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55f7043c9d90 con 0x55f70448b4c0 2024-05-20T20:18:50.547 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbaed600700 10 client.27955 handle_command_reply: tid=0 2024-05-20T20:18:50.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 client.27955 shutdown 2024-05-20T20:18:50.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446eeb0 msgr2=0x55f704472880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446eeb0 0x55f704472880 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fbaf4009930 tx=0x7fbaf4012af0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.548 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 shutdown_connections 2024-05-20T20:18:50.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f70448b4c0 0x55f70448d8e0 crc :-1 s=READY pgs=267 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704469880 0x55f7044754e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2502789970 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446eeb0 0x55f704472880 secure :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7fbaf4009930 tx=0x7fbaf4012af0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.547+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 >> 172.21.15.28:0/2502789970 conn(0x55f704397be0 msgr2=0x55f704468250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:50.549 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.548+0000 7fbaed600700 0 client.27955 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:50.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.549+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 shutdown_connections 2024-05-20T20:18:50.550 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.549+0000 7fbb146feb80 1 -- 172.21.15.28:0/2502789970 wait complete. 2024-05-20T20:18:50.551 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.549+0000 7fbb146feb80 20 client.27955 trim_cache size 0 max 16384 2024-05-20T20:18:50.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 Processor -- start 2024-05-20T20:18:50.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 -- start start 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f704469880 0x55f704469260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 0x55f7044931e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbb040701d0 con 0x55f70446f7d0 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fbb04071d10 con 0x55f704474e90 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.584+0000 7fbb146feb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fbb040708a0 con 0x55f704469880 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.585+0000 7fbb02000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f704469880 0x55f704469260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.585+0000 7fbb02000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f704469880 0x55f704469260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44926/0 (socket says 172.21.15.28:44926) 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.585+0000 7fbb02000700 1 -- 172.21.15.28:0/2961616801 learned_addr learned my addr 172.21.15.28:0/2961616801 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:50.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.585+0000 7fbb02000700 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 0x55f7044931e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.586+0000 7fbb02000700 1 -- 172.21.15.28:0/2961616801 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 msgr2=0x55f7044931e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.586+0000 7fbb02000700 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 0x55f7044931e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.586+0000 7fbb02000700 1 -- 172.21.15.28:0/2961616801 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 msgr2=0x55f704472d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.586+0000 7fbb02000700 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.586+0000 7fbb02000700 1 -- 172.21.15.28:0/2961616801 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55f70445e1c0 con 0x55f704469880 2024-05-20T20:18:50.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.586+0000 7fbb02000700 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f704469880 0x55f704469260 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fbaf00098f0 tx=0x7fbaf0005700 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b09842ed7645a37c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:50.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.587+0000 7fbaefe00700 1 -- 172.21.15.28:0/2961616801 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbaf00050d0 con 0x55f704469880 2024-05-20T20:18:50.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.587+0000 7fbaefe00700 1 -- 172.21.15.28:0/2961616801 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbaf0005270 con 0x55f704469880 2024-05-20T20:18:50.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.587+0000 7fbaefe00700 1 -- 172.21.15.28:0/2961616801 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbaf00373d0 con 0x55f704469880 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 -- 172.21.15.28:0/2961616801 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f704469880 msgr2=0x55f704469260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f704469880 0x55f704469260 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7fbaf00098f0 tx=0x7fbaf0005700 comp rx=0 tx=0).stop 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 -- 172.21.15.28:0/2961616801 shutdown_connections 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 0x55f7044931e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f704469880 0x55f704469260 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 --2- 172.21.15.28:0/2961616801 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f70446f7d0 0x55f704472d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 -- 172.21.15.28:0/2961616801 >> 172.21.15.28:0/2961616801 conn(0x55f704397be0 msgr2=0x55f704472660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 -- 172.21.15.28:0/2961616801 shutdown_connections 2024-05-20T20:18:50.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.588+0000 7fbb146feb80 1 -- 172.21.15.28:0/2961616801 wait complete. 2024-05-20T20:18:50.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.589+0000 7fbb146feb80 1 -- start start 2024-05-20T20:18:50.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.589+0000 7fbb146feb80 1 Processor -- start 2024-05-20T20:18:50.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.590+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704483ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.590+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704484420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.590+0000 7fbb146feb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 0x55f704480cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.590+0000 7fbb146feb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fbb04071640 con 0x55f704469880 2024-05-20T20:18:50.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.590+0000 7fbb146feb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fbb04070f70 con 0x55f704474e90 2024-05-20T20:18:50.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.590+0000 7fbb146feb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fbb0410e580 con 0x55f70446f7d0 2024-05-20T20:18:50.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.590+0000 7fbb146feb80 10 client.0 _ll_register_callbacks cb 0x55f704361c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704484420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704484420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44930/0 (socket says 172.21.15.28:44930) 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704483ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 -- 172.21.15.28:0/3969819700 learned_addr learned my addr 172.21.15.28:0/3969819700 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 -- 172.21.15.28:0/3969819700 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 msgr2=0x55f704480cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 0x55f704480cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 -- 172.21.15.28:0/3969819700 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 msgr2=0x55f704483ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704483ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.591+0000 7fbb02a00700 1 -- 172.21.15.28:0/3969819700 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55f704481520 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.592+0000 7fbb02a00700 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704484420 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fbaf4026f10 tx=0x7fbaf4005e10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9be8fefc4bd755f5 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.592+0000 7fbaed600700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.592+0000 7fbaed600700 1 -- 172.21.15.28:0/3969819700 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbaf400fbd0 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.592+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbaf0005270 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.592+0000 7fbaed600700 1 -- 172.21.15.28:0/3969819700 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fbaf400fd70 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbaed600700 1 -- 172.21.15.28:0/3969819700 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fbaf4010900 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 10 client.27961 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55f7044f6bf0 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbaed600700 1 -- 172.21.15.28:0/3969819700 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbaf0005270 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbaed600700 1 -- 172.21.15.28:0/3969819700 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fbaf4003680 con 0x55f70446f7d0 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 10 client.27961 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:50.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 10 client.27961 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:50.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 20 client.27961 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:50.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f7044f72a0 0x55f7044f9690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:50.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 4 client.27961 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-05-20T20:18:50.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.593+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "scrub status"}) -- 0x55f7043c3570 con 0x55f7044f72a0 2024-05-20T20:18:50.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.594+0000 7fbb02000700 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f7044f72a0 0x55f7044f9690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:50.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbb02000700 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f7044f72a0 0x55f7044f9690 crc :-1 s=READY pgs=268 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4249b749621aa0b1 server_cookie=a3e84fc541fb9343 in_seq=0 out_seq=0 2024-05-20T20:18:50.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbaed600700 10 client.27961 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:50.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbaed600700 1 -- 172.21.15.28:0/3969819700 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+63 (crc 0 0 0) 0x55f7043c3570 con 0x55f7044f72a0 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbaed600700 10 client.27961 handle_command_reply: tid=0 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbb146feb80 1 client.27961 shutdown 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 msgr2=0x55f704484420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbb146feb80 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704484420 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fbaf4026f10 tx=0x7fbaf4005e10 comp rx=0 tx=0).stop 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 shutdown_connections 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.595+0000 7fbb146feb80 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55f7044f72a0 0x55f7044f9690 crc :-1 s=READY pgs=268 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbb146feb80 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55f704474e90 0x55f704480cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbb146feb80 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55f704469880 0x55f704483ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbb146feb80 1 --2- 172.21.15.28:0/3969819700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55f70446f7d0 0x55f704484420 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.597 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 >> 172.21.15.28:0/3969819700 conn(0x55f704397be0 msgr2=0x55f70446fbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:50.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbaed600700 0 client.27961 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:50.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 shutdown_connections 2024-05-20T20:18:50.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbb146feb80 1 -- 172.21.15.28:0/3969819700 wait complete. 2024-05-20T20:18:50.598 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.596+0000 7fbb146feb80 20 client.27961 trim_cache size 0 max 16384 2024-05-20T20:18:50.598 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:18:50.599 INFO:teuthology.orchestra.run.smithi028.stdout: "status": "no active scrubs running", 2024-05-20T20:18:50.599 INFO:teuthology.orchestra.run.smithi028.stdout: "scrubs": {} 2024-05-20T20:18:50.599 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:18:50.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbae80b2a70 msgr2=0x7fbae80b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbae80b2a70 0x7fbae80b4f30 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fbafc0df8b0 tx=0x7fbafc0e5920 comp rx=0 tx=0).stop 2024-05-20T20:18:50.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbb041088f0 msgr2=0x7fbb041a06f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:50.599 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbb041088f0 0x7fbb041a06f0 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7fbafc00efc0 tx=0x7fbafc00c3a0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 shutdown_connections 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbb041088f0 0x7fbb041a06f0 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fbb041037f0 0x7fbb0419be20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbae80b2a70 0x7fbae80b4f30 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 --2- 172.21.15.28:0/2094010689 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fbb041041a0 0x7fbb0419c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 >> 172.21.15.28:0/2094010689 conn(0x7fbb0407a6a0 msgr2=0x7fbb04101a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 shutdown_connections 2024-05-20T20:18:50.600 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:50.597+0000 7fbb09e00700 1 -- 172.21.15.28:0/2094010689 wait complete. 2024-05-20T20:18:51.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:51 smithi028 ceph-mon[31178]: pgmap v413: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 338 MiB/s wr, 2.79k op/s 2024-05-20T20:18:51.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:51 smithi154 ceph-mon[36314]: pgmap v413: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 338 MiB/s wr, 2.79k op/s 2024-05-20T20:18:51.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:51 smithi033 ceph-mon[36179]: pgmap v413: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 338 MiB/s wr, 2.79k op/s 2024-05-20T20:18:51.867 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-05-20T20:18:51.868 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:18:52.765 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:53.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.115+0000 7f41b9400700 1 -- 172.21.15.28:0/2204428737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 msgr2=0x7f41b4110430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.117 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.115+0000 7f41b9400700 1 --2- 172.21.15.28:0/2204428737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 0x7f41b4110430 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f41ac009f90 tx=0x7f41ac00b330 comp rx=0 tx=0).stop 2024-05-20T20:18:53.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.116+0000 7f41b9400700 1 -- 172.21.15.28:0/2204428737 shutdown_connections 2024-05-20T20:18:53.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.116+0000 7f41b9400700 1 --2- 172.21.15.28:0/2204428737 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 0x7f41b4110430 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.116+0000 7f41b9400700 1 --2- 172.21.15.28:0/2204428737 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b4109660 0x7f41b4109a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.116+0000 7f41b9400700 1 --2- 172.21.15.28:0/2204428737 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f41b41035f0 0x7f41b4103a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.116+0000 7f41b9400700 1 -- 172.21.15.28:0/2204428737 >> 172.21.15.28:0/2204428737 conn(0x7f41b407a6f0 msgr2=0x7f41b407ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:53.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.116+0000 7f41b9400700 1 -- 172.21.15.28:0/2204428737 shutdown_connections 2024-05-20T20:18:53.118 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.116+0000 7f41b9400700 1 -- 172.21.15.28:0/2204428737 wait complete. 2024-05-20T20:18:53.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.117+0000 7f41b9400700 1 Processor -- start 2024-05-20T20:18:53.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.117+0000 7f41b9400700 1 -- start start 2024-05-20T20:18:53.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b9400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b41035f0 0x7f41b4075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.119 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b9400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 0x7f41b4075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b9400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f41b4109660 0x7f41b40762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b9400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f41b41137c0 con 0x7f41b41035f0 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b9400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f41b4113640 con 0x7f41b4109660 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b9400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f41b4113940 con 0x7f41b4103fb0 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b2800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b41035f0 0x7f41b4075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b2800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b41035f0 0x7f41b4075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:57994/0 (socket says 172.21.15.28:57994) 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b2800700 1 -- 172.21.15.28:0/3887072540 learned_addr learned my addr 172.21.15.28:0/3887072540 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.118+0000 7f41b1e00700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 0x7f41b4075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.120 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b3200700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f41b4109660 0x7f41b40762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b2800700 1 -- 172.21.15.28:0/3887072540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 msgr2=0x7f41b4075d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b2800700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 0x7f41b4075d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b2800700 1 -- 172.21.15.28:0/3887072540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f41b4109660 msgr2=0x7f41b40762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b2800700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f41b4109660 0x7f41b40762c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.121 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b2800700 1 -- 172.21.15.28:0/3887072540 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f41b41a8200 con 0x7f41b41035f0 2024-05-20T20:18:53.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b1e00700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 0x7f41b4075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:53.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.119+0000 7f41b2800700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b41035f0 0x7f41b4075840 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f41a400ec10 tx=0x7f41a400ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:53.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.120+0000 7f41ab400700 1 -- 172.21.15.28:0/3887072540 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41a400cc90 con 0x7f41b41035f0 2024-05-20T20:18:53.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.120+0000 7f41ab400700 1 -- 172.21.15.28:0/3887072540 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f41a400ce30 con 0x7f41b41035f0 2024-05-20T20:18:53.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.120+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f41b41a84f0 con 0x7f41b41035f0 2024-05-20T20:18:53.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.120+0000 7f41ab400700 1 -- 172.21.15.28:0/3887072540 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41a4010430 con 0x7f41b41035f0 2024-05-20T20:18:53.122 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.120+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f41b41a8a30 con 0x7f41b41035f0 2024-05-20T20:18:53.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.124+0000 7f41ab400700 1 -- 172.21.15.28:0/3887072540 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f41a4004000 con 0x7f41b41035f0 2024-05-20T20:18:53.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 Processor -- start 2024-05-20T20:18:53.125 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 -- start start 2024-05-20T20:18:53.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abac10 0x557d26abaff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abb530 0x557d26ab4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ab5a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557d26a12f40 con 0x557d26abb530 2024-05-20T20:18:53.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557d26a02ae0 con 0x557d26abac10 2024-05-20T20:18:53.126 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.125+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557d26a06b20 con 0x557d26ab55f0 2024-05-20T20:18:53.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b2800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abb530 0x557d26ab4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ab5a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b3200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abac10 0x557d26abaff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ab5a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44952/0 (socket says 172.21.15.28:44952) 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 -- 172.21.15.28:0/867049225 learned_addr learned my addr 172.21.15.28:0/867049225 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41ab400700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f41980b29f0 0x7f41980b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 -- 172.21.15.28:0/867049225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abac10 msgr2=0x557d26abaff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 --2- 172.21.15.28:0/867049225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abac10 0x557d26abaff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 -- 172.21.15.28:0/867049225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abb530 msgr2=0x557d26ab4fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 --2- 172.21.15.28:0/867049225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abb530 0x557d26ab4fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.126+0000 7f41b1e00700 1 -- 172.21.15.28:0/867049225 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557d269cb520 con 0x557d26ab55f0 2024-05-20T20:18:53.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.127+0000 7f41b3200700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f41980b29f0 0x7f41980b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.127+0000 7f41ab400700 1 -- 172.21.15.28:0/3887072540 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f41a40e28e0 con 0x7f41b41035f0 2024-05-20T20:18:53.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.127+0000 7f41b1e00700 1 --2- 172.21.15.28:0/867049225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ab5a50 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f41a0011540 tx=0x7f41a0011850 comp rx=0 tx=0).ready entity=mon.1 client_cookie=30676e5d6b60747a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:53.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.127+0000 7f419fe00700 1 -- 172.21.15.28:0/867049225 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41a0004a30 con 0x557d26ab55f0 2024-05-20T20:18:53.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.127+0000 7f419fe00700 1 -- 172.21.15.28:0/867049225 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f41a00054d0 con 0x557d26ab55f0 2024-05-20T20:18:53.129 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.127+0000 7f419fe00700 1 -- 172.21.15.28:0/867049225 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41a000e8d0 con 0x557d26ab55f0 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 -- 172.21.15.28:0/867049225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 msgr2=0x557d26ab5a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/867049225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ab5a50 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f41a0011540 tx=0x7f41a0011850 comp rx=0 tx=0).stop 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41b3200700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f41980b29f0 0x7f41980b4eb0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f41ac009790 tx=0x7f41ac009920 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 -- 172.21.15.28:0/867049225 shutdown_connections 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/867049225 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ab5a50 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/867049225 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abac10 0x557d26abaff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/867049225 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abb530 0x557d26ab4fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 -- 172.21.15.28:0/867049225 >> 172.21.15.28:0/867049225 conn(0x557d269e2ce0 msgr2=0x557d26aaff00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:53.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 -- 172.21.15.28:0/867049225 shutdown_connections 2024-05-20T20:18:53.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.128+0000 7f41c3e57b80 1 -- 172.21.15.28:0/867049225 wait complete. 2024-05-20T20:18:53.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.129+0000 7f41c3e57b80 1 -- start start 2024-05-20T20:18:53.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.129+0000 7f41c3e57b80 1 Processor -- start 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ac1170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abac10 0x557d26abe510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26abeed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x557d269090a0 con 0x557d26abac10 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x557d269a3860 con 0x557d26abea50 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x557d26ac1fa0 con 0x557d26ab55f0 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41c3e57b80 10 client.0 _ll_register_callbacks cb 0x557d269acd70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abac10 0x557d26abe510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41b2800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ac1170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41b3200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26abeed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abac10 0x557d26abe510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58014/0 (socket says 172.21.15.28:58014) 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41b2800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ac1170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44956/0 (socket says 172.21.15.28:44956) 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.130+0000 7f41b1e00700 1 -- 172.21.15.28:0/3684588839 learned_addr learned my addr 172.21.15.28:0/3684588839 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41b3200700 1 -- 172.21.15.28:0/3684588839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 msgr2=0x557d26ac1170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41b3200700 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ac1170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41b3200700 1 -- 172.21.15.28:0/3684588839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abac10 msgr2=0x557d26abe510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41b3200700 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abac10 0x557d26abe510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41b3200700 1 -- 172.21.15.28:0/3684588839 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557d269529d0 con 0x557d26abea50 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41b1e00700 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abac10 0x557d26abe510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41b3200700 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26abeed0 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f41ac00b6c0 tx=0x7f41ac00b330 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b7fcf1571ea01210 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f419cc00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:53.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f419cc00700 1 -- 172.21.15.28:0/3684588839 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41ac007420 con 0x557d26abea50 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f419cc00700 1 -- 172.21.15.28:0/3684588839 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f41ac0075c0 con 0x557d26abea50 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.131+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f41a00054d0 con 0x557d26abea50 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f419cc00700 1 -- 172.21.15.28:0/3684588839 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f41ac0077c0 con 0x557d26abea50 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 10 client.28037 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557d269d3160 con 0x557d26abea50 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f419cc00700 1 -- 172.21.15.28:0/3684588839 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f41a00054d0 con 0x557d26abea50 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f419cc00700 1 -- 172.21.15.28:0/3684588839 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f41ac02b030 con 0x557d26abea50 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 10 client.28037 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 10 client.28037 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 20 client.28037 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26acee90 0x557d26ad1280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 4 client.28037 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:53.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.132+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x557d26a14e90 con 0x557d26acee90 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.133+0000 7f41b2800700 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26acee90 0x557d26ad1280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.133+0000 7f41b2800700 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26acee90 0x557d26ad1280 crc :-1 s=READY pgs=269 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b1e30d771b888d0e server_cookie=2a0f1caf640cc7ac in_seq=0 out_seq=0 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.133+0000 7f419cc00700 10 client.28037 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f419cc00700 1 -- 172.21.15.28:0/3684588839 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x557d26a14e90 con 0x557d26acee90 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f419cc00700 10 client.28037 handle_command_reply: tid=0 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 client.28037 shutdown 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 msgr2=0x557d26abeed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26abeed0 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f41ac00b6c0 tx=0x7f41ac00b330 comp rx=0 tx=0).stop 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 shutdown_connections 2024-05-20T20:18:53.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26acee90 0x557d26ad1280 crc :-1 s=READY pgs=269 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26abeed0 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26ab55f0 0x557d26ac1170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/3684588839 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abac10 0x557d26abe510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f419cc00700 0 client.28037 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 >> 172.21.15.28:0/3684588839 conn(0x557d269e2ce0 msgr2=0x557d26aaf430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 shutdown_connections 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.134+0000 7f41c3e57b80 1 -- 172.21.15.28:0/3684588839 wait complete. 2024-05-20T20:18:53.138 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.135+0000 7f41c3e57b80 20 client.28037 trim_cache size 0 max 16384 2024-05-20T20:18:53.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.157+0000 7f41c3e57b80 1 Processor -- start 2024-05-20T20:18:53.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.157+0000 7f41c3e57b80 1 -- start start 2024-05-20T20:18:53.159 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26ab55f0 0x557d26ab4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26abc030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26acb040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f41b41abca0 con 0x557d26ab55f0 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f41b41ad7e0 con 0x557d26abea50 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f41b41ac370 con 0x557d26abac10 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26abc030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26abc030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44968/0 (socket says 172.21.15.28:44968) 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41b1e00700 1 -- 172.21.15.28:0/1249840733 learned_addr learned my addr 172.21.15.28:0/1249840733 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41b2800700 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26ab55f0 0x557d26ab4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.160 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.158+0000 7f41b3200700 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26acb040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.159+0000 7f41b1e00700 1 -- 172.21.15.28:0/1249840733 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 msgr2=0x557d26acb040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.159+0000 7f41b1e00700 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26acb040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.159+0000 7f41b1e00700 1 -- 172.21.15.28:0/1249840733 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26ab55f0 msgr2=0x557d26ab4fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.161 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.159+0000 7f41b1e00700 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26ab55f0 0x557d26ab4fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.159+0000 7f41b1e00700 1 -- 172.21.15.28:0/1249840733 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x557d26ac0db0 con 0x557d26abac10 2024-05-20T20:18:53.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.159+0000 7f41b1e00700 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26abc030 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f41a0011460 tx=0x7f41a000e470 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bea309dc892f3e6 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:53.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.159+0000 7f419f400700 1 -- 172.21.15.28:0/1249840733 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41a00054d0 con 0x557d26abac10 2024-05-20T20:18:53.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.160+0000 7f419f400700 1 -- 172.21.15.28:0/1249840733 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f41a0005670 con 0x557d26abac10 2024-05-20T20:18:53.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.160+0000 7f419f400700 1 -- 172.21.15.28:0/1249840733 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41a000eb40 con 0x557d26abac10 2024-05-20T20:18:53.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.160+0000 7f41c3e57b80 1 -- 172.21.15.28:0/1249840733 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 msgr2=0x557d26abc030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.162 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.160+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26abc030 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f41a0011460 tx=0x7f41a000e470 comp rx=0 tx=0).stop 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 -- 172.21.15.28:0/1249840733 shutdown_connections 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26abea50 0x557d26acb040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26ab55f0 0x557d26ab4fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/1249840733 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26abc030 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 -- 172.21.15.28:0/1249840733 >> 172.21.15.28:0/1249840733 conn(0x557d269e2ce0 msgr2=0x557d26abb920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 -- 172.21.15.28:0/1249840733 shutdown_connections 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 -- 172.21.15.28:0/1249840733 wait complete. 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.161+0000 7f41c3e57b80 1 -- start start 2024-05-20T20:18:53.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 1 Processor -- start 2024-05-20T20:18:53.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26ab55f0 0x557d26ac7410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26ac7950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abea50 0x557d26ac4200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f41b41ad110 con 0x557d26abea50 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f41b41aca40 con 0x557d26ab55f0 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f41b419db80 con 0x557d26abac10 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41c3e57b80 10 client.0 _ll_register_callbacks cb 0x557d269acd70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41b3200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abea50 0x557d26ac4200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26ac7950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.162+0000 7f41b2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26ab55f0 0x557d26ac7410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b1e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26ac7950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:44974/0 (socket says 172.21.15.28:44974) 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b2800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26ab55f0 0x557d26ac7410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53620/0 (socket says 172.21.15.28:53620) 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b1e00700 1 -- 172.21.15.28:0/4138864338 learned_addr learned my addr 172.21.15.28:0/4138864338 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:53.165 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b2800700 1 -- 172.21.15.28:0/4138864338 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 msgr2=0x557d26ac7950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b2800700 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26ac7950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b2800700 1 -- 172.21.15.28:0/4138864338 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abea50 msgr2=0x557d26ac4200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b2800700 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abea50 0x557d26ac4200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b2800700 1 -- 172.21.15.28:0/4138864338 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x557d26ac4900 con 0x557d26ab55f0 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41b2800700 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26ab55f0 0x557d26ac7410 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f41a40dffc0 tx=0x7f41a40e8530 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a69e6528378f6da9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f419cc00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f419cc00700 1 -- 172.21.15.28:0/4138864338 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f41a40a94a0 con 0x557d26ab55f0 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f419cc00700 1 -- 172.21.15.28:0/4138864338 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f41a40a6140 con 0x557d26ab55f0 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.163+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f41a0005670 con 0x557d26ab55f0 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f419cc00700 1 -- 172.21.15.28:0/4138864338 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f41a40e0450 con 0x557d26ab55f0 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 10 client.28043 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:53.166 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x557d26b3a510 con 0x557d26ab55f0 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f419cc00700 1 -- 172.21.15.28:0/4138864338 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f41a0005670 con 0x557d26ab55f0 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f419cc00700 1 -- 172.21.15.28:0/4138864338 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f41a40042f0 con 0x557d26ab55f0 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 10 client.28043 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 10 client.28043 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 20 client.28043 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26b3abd0 0x557d26b3d010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 4 client.28043 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "set_id": "ec1db510", "release": true, "await": true}] 2024-05-20T20:18:53.167 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "set_id": "ec1db510", "release": true, "await": true}) -- 0x557d26a0e670 con 0x557d26b3abd0 2024-05-20T20:18:53.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.164+0000 7f41b1e00700 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26b3abd0 0x557d26b3d010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:53.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.165+0000 7f41b1e00700 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26b3abd0 0x557d26b3d010 crc :-1 s=READY pgs=270 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2aac65690dbaba6c server_cookie=60091784e9ab448d in_seq=0 out_seq=0 2024-05-20T20:18:53.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.165+0000 7f419cc00700 10 client.28043 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:53.170 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-05-20T20:18:53.170 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.168+0000 7f419cc00700 1 -- 172.21.15.28:0/4138864338 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+591 (crc 0 0 0) 0x557d26a0e670 con 0x557d26b3abd0 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.168+0000 7f419cc00700 10 client.28043 handle_command_reply: tid=0 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 client.28043 shutdown 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26ab55f0 msgr2=0x557d26ac7410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26ab55f0 0x557d26ac7410 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7f41a40dffc0 tx=0x7f41a40e8530 comp rx=0 tx=0).stop 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 shutdown_connections 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x557d26b3abd0 0x557d26b3d010 crc :-1 s=READY pgs=270 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x557d26abea50 0x557d26ac4200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x557d26ab55f0 0x557d26ac7410 unknown :-1 s=CLOSED pgs=615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 --2- 172.21.15.28:0/4138864338 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x557d26abac10 0x557d26ac7950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.171 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f419cc00700 0 client.28043 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 >> 172.21.15.28:0/4138864338 conn(0x557d269e2ce0 msgr2=0x557d26abce00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 shutdown_connections 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 1 -- 172.21.15.28:0/4138864338 wait complete. 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.169+0000 7f41c3e57b80 20 client.28043 trim_cache size 0 max 16384 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 36, 2024-05-20T20:18:53.172 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": { 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "ec1db510": { 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "version": 36, 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "age_ref": 0.0, 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "timeout": 90.0, 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "expiration": 137.9, 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "members": { 2024-05-20T20:18:53.173 INFO:teuthology.orchestra.run.smithi028.stdout: "file:/": { 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: "excluded": false, 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: "state": { 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RELEASED", 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: "age": 0.0 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:18:53.174 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:18:53.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f41980b29f0 msgr2=0x7f41980b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f41980b29f0 0x7f41980b4eb0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f41ac009790 tx=0x7f41ac009920 comp rx=0 tx=0).stop 2024-05-20T20:18:53.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b41035f0 msgr2=0x7f41b4075840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:53.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b41035f0 0x7f41b4075840 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f41a400ec10 tx=0x7f41a400ef20 comp rx=0 tx=0).stop 2024-05-20T20:18:53.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 shutdown_connections 2024-05-20T20:18:53.175 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f41980b29f0 0x7f41980b4eb0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f41b4109660 0x7f41b40762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f41b41035f0 0x7f41b4075840 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 --2- 172.21.15.28:0/3887072540 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f41b4103fb0 0x7f41b4075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:53.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.170+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 >> 172.21.15.28:0/3887072540 conn(0x7f41b407a6f0 msgr2=0x7f41b41022e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:53.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.171+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 shutdown_connections 2024-05-20T20:18:53.176 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:53.171+0000 7f41b9400700 1 -- 172.21.15.28:0/3887072540 wait complete. 2024-05-20T20:18:53.354 INFO:tasks.workunit:Stopping ['suites/iogen.sh'] on client.0... 2024-05-20T20:18:53.354 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-05-20T20:18:53.413 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:53 smithi028 ceph-mon[31178]: pgmap v414: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 240 MiB/s wr, 2.05k op/s 2024-05-20T20:18:53.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:53 smithi154 ceph-mon[36314]: pgmap v414: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 240 MiB/s wr, 2.05k op/s 2024-05-20T20:18:53.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:53 smithi033 ceph-mon[36179]: pgmap v414: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 240 MiB/s wr, 2.05k op/s 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 -- 172.21.15.28:0/2779951643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 msgr2=0x7fc008105280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 --2- 172.21.15.28:0/2779951643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc008105280 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7fbff8009900 tx=0x7fbff802a410 comp rx=0 tx=0).stop 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 -- 172.21.15.28:0/2779951643 shutdown_connections 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 --2- 172.21.15.28:0/2779951643 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc008105850 0x7fc008109890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 --2- 172.21.15.28:0/2779951643 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc008109f70 0x7fc008111b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 --2- 172.21.15.28:0/2779951643 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc008105280 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 -- 172.21.15.28:0/2779951643 >> 172.21.15.28:0/2779951643 conn(0x7fc008100740 msgr2=0x7fc008102b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 -- 172.21.15.28:0/2779951643 shutdown_connections 2024-05-20T20:18:55.273 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.272+0000 7fc00d200640 1 -- 172.21.15.28:0/2779951643 wait complete. 2024-05-20T20:18:55.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.273+0000 7fc00d200640 1 Processor -- start 2024-05-20T20:18:55.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.273+0000 7fc00d200640 1 -- start start 2024-05-20T20:18:55.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.273+0000 7fc00d200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc00819b400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.273+0000 7fc00d200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc008109f70 0x7fc00819b940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.275 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.273+0000 7fc00d200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc00819fc70 0x7fc0081a0120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc00d200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc008114f20 con 0x7fc008104ea0 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc00d200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc008114da0 con 0x7fc008109f70 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc00d200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc0081150a0 con 0x7fc00819fc70 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc00819b400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc006800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc00819fc70 0x7fc0081a0120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc00819b400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58044/0 (socket says 172.21.15.28:58044) 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 -- 172.21.15.28:0/2828563418 learned_addr learned my addr 172.21.15.28:0/2828563418 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005400640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc008109f70 0x7fc00819b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 -- 172.21.15.28:0/2828563418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc00819fc70 msgr2=0x7fc0081a0120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc00819fc70 0x7fc0081a0120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 -- 172.21.15.28:0/2828563418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc008109f70 msgr2=0x7fc00819b940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc008109f70 0x7fc00819b940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.276 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005e00640 1 -- 172.21.15.28:0/2828563418 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc0081a0760 con 0x7fc008104ea0 2024-05-20T20:18:55.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.274+0000 7fc005400640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc008109f70 0x7fc00819b940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:55.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.275+0000 7fc005e00640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc00819b400 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7fbff8005ab0 tx=0x7fbff8003c80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:55.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.275+0000 7fbffea00640 1 -- 172.21.15.28:0/2828563418 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff803f070 con 0x7fc008104ea0 2024-05-20T20:18:55.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.275+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc0081a09f0 con 0x7fc008104ea0 2024-05-20T20:18:55.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.275+0000 7fbffea00640 1 -- 172.21.15.28:0/2828563418 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbff8031d60 con 0x7fc008104ea0 2024-05-20T20:18:55.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.275+0000 7fbffea00640 1 -- 172.21.15.28:0/2828563418 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff8035030 con 0x7fc008104ea0 2024-05-20T20:18:55.277 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.275+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc0081a81c0 con 0x7fc008104ea0 2024-05-20T20:18:55.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.280+0000 7fc01003a740 1 Processor -- start 2024-05-20T20:18:55.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.280+0000 7fc01003a740 1 -- start start 2024-05-20T20:18:55.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc01003a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c99d070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.281 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc01003a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a7930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc01003a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9aa260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc01003a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55953c90db90 con 0x55953c99cc90 2024-05-20T20:18:55.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc01003a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55953c8fd730 con 0x55953c9a7e70 2024-05-20T20:18:55.282 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc01003a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55953c901770 con 0x55953c99d5b0 2024-05-20T20:18:55.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fbffea00640 1 -- 172.21.15.28:0/2828563418 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fbff8042070 con 0x7fc008104ea0 2024-05-20T20:18:55.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc006800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c99d070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc005400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9aa260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc006800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c99d070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58056/0 (socket says 172.21.15.28:58056) 2024-05-20T20:18:55.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc006800640 1 -- 172.21.15.28:0/1085365024 learned_addr learned my addr 172.21.15.28:0/1085365024 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:55.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.281+0000 7fc005400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9aa260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53654/0 (socket says 172.21.15.28:53654) 2024-05-20T20:18:55.283 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc005e00640 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a7930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc005400640 1 -- 172.21.15.28:0/1085365024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 msgr2=0x55953c9a7930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc005400640 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a7930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc005400640 1 -- 172.21.15.28:0/1085365024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 msgr2=0x55953c99d070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc005400640 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c99d070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc005400640 1 -- 172.21.15.28:0/1085365024 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55953c8968b0 con 0x55953c9a7e70 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc006800640 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c99d070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fc005400640 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9aa260 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7fc00806c1c0 tx=0x7fbff40118d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=91c7d30164bf2403 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fbff3400640 1 -- 172.21.15.28:0/1085365024 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff4004030 con 0x55953c9a7e70 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fbff3400640 1 -- 172.21.15.28:0/1085365024 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbff40054d0 con 0x55953c9a7e70 2024-05-20T20:18:55.284 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.282+0000 7fbff3400640 1 -- 172.21.15.28:0/1085365024 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff4004ce0 con 0x55953c9a7e70 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.283+0000 7fbffea00640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbfec0b2a10 0x7fbfec0b4ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.283+0000 7fbffea00640 1 -- 172.21.15.28:0/2828563418 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fbff8102d70 con 0x7fc008104ea0 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.283+0000 7fc01003a740 1 -- 172.21.15.28:0/1085365024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 msgr2=0x55953c9aa260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.283+0000 7fc01003a740 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9aa260 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7fc00806c1c0 tx=0x7fbff40118d0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.283+0000 7fc006800640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbfec0b2a10 0x7fbfec0b4ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 -- 172.21.15.28:0/1085365024 shutdown_connections 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9aa260 unknown :-1 s=CLOSED pgs=616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.286 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 --2- 172.21.15.28:0/1085365024 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c99d070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 -- 172.21.15.28:0/1085365024 >> 172.21.15.28:0/1085365024 conn(0x55953c9991a0 msgr2=0x55953c999580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 -- 172.21.15.28:0/1085365024 shutdown_connections 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 -- 172.21.15.28:0/1085365024 wait complete. 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc006800640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbfec0b2a10 0x7fbfec0b4ed0 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fc000009d50 tx=0x7fc0000096a0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.284+0000 7fc01003a740 1 -- start start 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 1 Processor -- start 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9a7820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a4b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9a50b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55953c5e3790 con 0x55953c99cc90 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55953c8a5bc0 con 0x55953c99d5b0 2024-05-20T20:18:55.287 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55953c9aaaa0 con 0x55953c9a7e70 2024-05-20T20:18:55.288 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.285+0000 7fc01003a740 10 client.0 _ll_register_callbacks cb 0x55953c883400 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:55.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a4b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc006800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9a50b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9a7820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a4b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:53668/0 (socket says 172.21.15.28:53668) 2024-05-20T20:18:55.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 -- 172.21.15.28:0/1202232502 learned_addr learned my addr 172.21.15.28:0/1202232502 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:55.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 -- 172.21.15.28:0/1202232502 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 msgr2=0x55953c9a50b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.290 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9a50b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 -- 172.21.15.28:0/1202232502 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 msgr2=0x55953c9a7820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9a7820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005400640 1 -- 172.21.15.28:0/1202232502 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55953c612210 con 0x55953c99d5b0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.286+0000 7fc005e00640 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9a7820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fc005400640 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a4b10 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fbff4009350 tx=0x7fbff40050f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=abd8a29607d47ed2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fbfe7e00640 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fbfe7e00640 1 -- 172.21.15.28:0/1202232502 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff4029020 con 0x55953c99d5b0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbff40054d0 con 0x55953c99d5b0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fbfe7e00640 1 -- 172.21.15.28:0/1202232502 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fbff400f750 con 0x55953c99d5b0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fbfe7e00640 1 -- 172.21.15.28:0/1202232502 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fbff40052e0 con 0x55953c99d5b0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fc01003a740 10 client.28055 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.287+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55953c6de620 con 0x55953c99d5b0 2024-05-20T20:18:55.291 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fbfe7e00640 1 -- 172.21.15.28:0/1202232502 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbff40054d0 con 0x55953c99d5b0 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fbfe7e00640 1 -- 172.21.15.28:0/1202232502 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fbff4025650 con 0x55953c99d5b0 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fc01003a740 10 client.28055 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fc01003a740 10 client.28055 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fc01003a740 20 client.28055 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fc01003a740 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953c9b77a0 0x55953c9b9b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fc01003a740 4 client.28055 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.288+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55953c90fae0 con 0x55953c9b77a0 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.289+0000 7fc005e00640 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953c9b77a0 0x55953c9b9b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.289+0000 7fc005e00640 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953c9b77a0 0x55953c9b9b90 crc :-1 s=READY pgs=271 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a077185cd3e71206 server_cookie=bd9545bdafd93294 in_seq=0 out_seq=0 2024-05-20T20:18:55.292 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.289+0000 7fbfe7e00640 10 client.28055 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:55.293 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fbfe7e00640 1 -- 172.21.15.28:0/1202232502 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55953c90fae0 con 0x55953c9b77a0 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fbfe7e00640 10 client.28055 handle_command_reply: tid=0 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 client.28055 shutdown 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 msgr2=0x55953c9a4b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a4b10 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fbff4009350 tx=0x7fbff40050f0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 shutdown_connections 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953c9b77a0 0x55953c9b9b90 crc :-1 s=READY pgs=271 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a4b10 unknown :-1 s=CLOSED pgs=617 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9a50b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 --2- 172.21.15.28:0/1202232502 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9a7820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 >> 172.21.15.28:0/1202232502 conn(0x55953c9991a0 msgr2=0x55953c99ef20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:55.294 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fbfe7e00640 0 client.28055 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:55.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 shutdown_connections 2024-05-20T20:18:55.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.290+0000 7fc01003a740 1 -- 172.21.15.28:0/1202232502 wait complete. 2024-05-20T20:18:55.295 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.291+0000 7fc01003a740 20 client.28055 trim_cache size 0 max 16384 2024-05-20T20:18:55.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.307+0000 7fc01003a740 1 Processor -- start 2024-05-20T20:18:55.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.307+0000 7fc01003a740 1 -- start start 2024-05-20T20:18:55.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc01003a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b3b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc01003a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a7350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc01003a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9bf320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.308 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc01003a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc0081b0870 con 0x55953c99cc90 2024-05-20T20:18:55.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc01003a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc0081b23b0 con 0x55953c99d5b0 2024-05-20T20:18:55.309 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc01003a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc0081b0f40 con 0x55953c9a7e70 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc005e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b3b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc005e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b3b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58070/0 (socket says 172.21.15.28:58070) 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc005e00640 1 -- 172.21.15.28:0/534775655 learned_addr learned my addr 172.21.15.28:0/534775655 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc006800640 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9bf320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.308+0000 7fc005400640 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a7350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fc005e00640 1 -- 172.21.15.28:0/534775655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 msgr2=0x55953c9bf320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fc005e00640 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9bf320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fc005e00640 1 -- 172.21.15.28:0/534775655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 msgr2=0x55953c9a7350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.310 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fc005e00640 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a7350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fc005e00640 1 -- 172.21.15.28:0/534775655 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55953c9976b0 con 0x55953c99cc90 2024-05-20T20:18:55.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fc005e00640 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b3b00 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7fbff8100ab0 tx=0x7fbff803b870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b32dd93c416d446 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:55.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fbff2a00640 1 -- 172.21.15.28:0/534775655 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff8049040 con 0x55953c99cc90 2024-05-20T20:18:55.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fbff2a00640 1 -- 172.21.15.28:0/534775655 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbff80ca0f0 con 0x55953c99cc90 2024-05-20T20:18:55.311 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.309+0000 7fbff2a00640 1 -- 172.21.15.28:0/534775655 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff803b460 con 0x55953c99cc90 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 -- 172.21.15.28:0/534775655 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 msgr2=0x55953c9b3b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b3b00 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7fbff8100ab0 tx=0x7fbff803b870 comp rx=0 tx=0).stop 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 -- 172.21.15.28:0/534775655 shutdown_connections 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c99d5b0 0x55953c9a7350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c9a7e70 0x55953c9bf320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 --2- 172.21.15.28:0/534775655 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b3b00 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 -- 172.21.15.28:0/534775655 >> 172.21.15.28:0/534775655 conn(0x55953c9991a0 msgr2=0x55953c9a0160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 -- 172.21.15.28:0/534775655 shutdown_connections 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.310+0000 7fc01003a740 1 -- 172.21.15.28:0/534775655 wait complete. 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 -- start start 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 Processor -- start 2024-05-20T20:18:55.313 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b03b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a1860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9a1da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fc0081b1ce0 con 0x55953c99cc90 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fc0081b1610 con 0x55953c9a7e70 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.311+0000 7fc01003a740 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fc0081ae870 con 0x55953c99d5b0 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc01003a740 10 client.0 _ll_register_callbacks cb 0x7fc000009a00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b03b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a1860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b03b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:58086/0 (socket says 172.21.15.28:58086) 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc006800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9a1da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005e00640 1 -- 172.21.15.28:0/687856887 learned_addr learned my addr 172.21.15.28:0/687856887 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005400640 1 -- 172.21.15.28:0/687856887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 msgr2=0x55953c9a1da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.314 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005400640 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9a1da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005400640 1 -- 172.21.15.28:0/687856887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 msgr2=0x55953c9b03b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:18:55.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005400640 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b03b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.315 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.312+0000 7fc005400640 1 -- 172.21.15.28:0/687856887 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55953c9a2580 con 0x55953c99d5b0 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.313+0000 7fc005400640 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a1860 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7fbff4025af0 tx=0x7fbff4009e90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9aa61b2833ebfd62 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.313+0000 7fbfe7e00640 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.314+0000 7fbfe7e00640 1 -- 172.21.15.28:0/687856887 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbff4029040 con 0x55953c99d5b0 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.314+0000 7fbfe7e00640 1 -- 172.21.15.28:0/687856887 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fbff4004040 con 0x55953c99d5b0 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.314+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fbff80ca0f0 con 0x55953c99d5b0 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.315+0000 7fbfe7e00640 1 -- 172.21.15.28:0/687856887 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fbff4025880 con 0x55953c99d5b0 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.315+0000 7fc01003a740 10 client.27997 fetch_fsmap learned FSMap version 29 2024-05-20T20:18:55.316 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.315+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55953ca22210 con 0x55953c99d5b0 2024-05-20T20:18:55.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fbfe7e00640 1 -- 172.21.15.28:0/687856887 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fbff80ca0f0 con 0x55953c99d5b0 2024-05-20T20:18:55.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fbfe7e00640 1 -- 172.21.15.28:0/687856887 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fbff4025ce0 con 0x55953c99d5b0 2024-05-20T20:18:55.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fc01003a740 10 client.27997 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:18:55.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fc01003a740 10 client.27997 resolve_mds: resolved cephfs:0 to role '1:0' aka daemon mds.b 2024-05-20T20:18:55.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fc01003a740 20 client.27997 populate_metadata read hostname 'smithi028' 2024-05-20T20:18:55.317 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fc01003a740 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953ca228d0 0x55953ca24cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:55.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fc01003a740 4 client.27997 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "flush journal"}] 2024-05-20T20:18:55.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "flush journal"}) -- 0x55953c9092c0 con 0x55953ca228d0 2024-05-20T20:18:55.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.316+0000 7fc005e00640 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953ca228d0 0x55953ca24cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:55.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.317+0000 7fc005e00640 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953ca228d0 0x55953ca24cc0 crc :-1 s=READY pgs=272 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3458dd9d9b186105 server_cookie=6b54792d406ec458 in_seq=0 out_seq=0 2024-05-20T20:18:55.318 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.317+0000 7fbfe7e00640 10 client.27997 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:55.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fbfe7e00640 1 -- 172.21.15.28:0/687856887 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+25 (crc 0 0 0) 0x55953c9092c0 con 0x55953ca228d0 2024-05-20T20:18:55.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fbfe7e00640 10 client.27997 handle_command_reply: tid=0 2024-05-20T20:18:55.346 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 client.27997 shutdown 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 msgr2=0x55953c9a1860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a1860 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7fbff4025af0 tx=0x7fbff4009e90 comp rx=0 tx=0).stop 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 shutdown_connections 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55953ca228d0 0x55953ca24cc0 crc :-1 s=READY pgs=272 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55953c9a7e70 0x55953c9a1da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55953c99d5b0 0x55953c9a1860 unknown :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fbfe7e00640 0 client.27997 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 --2- 172.21.15.28:0/687856887 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55953c99cc90 0x55953c9b03b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.345+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 >> 172.21.15.28:0/687856887 conn(0x55953c9991a0 msgr2=0x55953c99f840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.346+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 shutdown_connections 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.346+0000 7fc01003a740 1 -- 172.21.15.28:0/687856887 wait complete. 2024-05-20T20:18:55.347 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.346+0000 7fc01003a740 20 client.27997 trim_cache size 0 max 16384 2024-05-20T20:18:55.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:{ 2024-05-20T20:18:55.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout: "return_code": 0 2024-05-20T20:18:55.348 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stdout:} 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbfec0b2a10 msgr2=0x7fbfec0b4ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbfec0b2a10 0x7fbfec0b4ed0 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fc000009d50 tx=0x7fc0000096a0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 msgr2=0x7fc00819b400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc00819b400 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7fbff8005ab0 tx=0x7fbff8003c80 comp rx=0 tx=0).stop 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 shutdown_connections 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fc008109f70 0x7fc00819b940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fbfec0b2a10 0x7fbfec0b4ed0 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc00819fc70 0x7fc0081a0120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 --2- 172.21.15.28:0/2828563418 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fc008104ea0 0x7fc00819b400 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 >> 172.21.15.28:0/2828563418 conn(0x7fc008100740 msgr2=0x7fc008102a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:55.349 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 shutdown_connections 2024-05-20T20:18:55.350 INFO:teuthology.task.background_exec.ubuntu@smithi028.front.sepia.ceph.com.smithi028.stderr:2024-05-20T20:18:55.347+0000 7fc00d200640 1 -- 172.21.15.28:0/2828563418 wait complete. 2024-05-20T20:18:55.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:55 smithi154 ceph-mon[36314]: pgmap v415: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 339 MiB/s wr, 2.75k op/s 2024-05-20T20:18:55.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:55 smithi033 ceph-mon[36179]: pgmap v415: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 339 MiB/s wr, 2.75k op/s 2024-05-20T20:18:55.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:55 smithi028 ceph-mon[31178]: pgmap v415: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 339 MiB/s wr, 2.75k op/s 2024-05-20T20:18:55.856 DEBUG:teuthology.parallel:result is None 2024-05-20T20:18:55.857 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-05-20T20:18:55.949 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-05-20T20:18:55.949 DEBUG:teuthology.run_tasks:Unwinding manager quiescer 2024-05-20T20:18:55.965 INFO:tasks.quiescer:joining Quiescers 2024-05-20T20:18:55.966 WARNING:tasks.quiescer:The quiescer is requested to stop, running cancel all 2024-05-20T20:18:55.966 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:18:55.979 INFO:tasks.quiescer.fs.[cephfs]:Successfully released set 'ec1db510', total seconds elapsed: 47.5 2024-05-20T20:18:56.758 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:18:57.397 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:57 smithi028 ceph-mon[31178]: pgmap v416: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 217 MiB/s wr, 1.71k op/s 2024-05-20T20:18:57.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 -- 172.21.15.28:0/937363580 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33741185d0 msgr2=0x7f337411a9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:57.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 --2- 172.21.15.28:0/937363580 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33741185d0 0x7f337411a9c0 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f336c00b0a0 tx=0x7f336c00b3b0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 -- 172.21.15.28:0/937363580 shutdown_connections 2024-05-20T20:18:57.590 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 --2- 172.21.15.28:0/937363580 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f33741185d0 0x7f337411a9c0 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 --2- 172.21.15.28:0/937363580 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f337410dc70 0x7f337410a540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 --2- 172.21.15.28:0/937363580 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f337410aa80 0x7f337410aee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 -- 172.21.15.28:0/937363580 >> 172.21.15.28:0/937363580 conn(0x7f337406ea90 msgr2=0x7f337406eea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 -- 172.21.15.28:0/937363580 shutdown_connections 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.567+0000 7f337ba00700 1 -- 172.21.15.28:0/937363580 wait complete. 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.568+0000 7f337ba00700 1 Processor -- start 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f337ba00700 1 -- start start 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f337ba00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f337410aa80 0x7f33741a4650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f337ba00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f337410dc70 0x7f33741a4b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f337ba00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33741185d0 0x7f33741a8ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f337ba00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f337411e140 con 0x7f337410dc70 2024-05-20T20:18:57.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f337ba00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f337411dfc0 con 0x7f33741185d0 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f337ba00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f337411e2c0 con 0x7f337410aa80 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f3373e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f337410dc70 0x7f33741a4b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f3373e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f337410dc70 0x7f33741a4b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40966/0 (socket says 172.21.15.28:40966) 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f3373e00700 1 -- 172.21.15.28:0/695219021 learned_addr learned my addr 172.21.15.28:0/695219021 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.569+0000 7f3379000700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f337410aa80 0x7f33741a4650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3379a00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33741185d0 0x7f33741a8ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3373e00700 1 -- 172.21.15.28:0/695219021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f337410aa80 msgr2=0x7f33741a4650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3373e00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f337410aa80 0x7f33741a4650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3373e00700 1 -- 172.21.15.28:0/695219021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33741185d0 msgr2=0x7f33741a8ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3373e00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33741185d0 0x7f33741a8ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3373e00700 1 -- 172.21.15.28:0/695219021 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f33741a9610 con 0x7f337410dc70 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3379000700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f337410aa80 0x7f33741a4650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:18:57.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3379a00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33741185d0 0x7f33741a8ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3373e00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f337410dc70 0x7f33741a4b90 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f336000e9a0 tx=0x7f336000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.570+0000 7f3371600700 1 -- 172.21.15.28:0/695219021 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f336000cbd0 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.571+0000 7f3371600700 1 -- 172.21.15.28:0/695219021 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f336000cd70 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.571+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f33741a9900 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.571+0000 7f3371600700 1 -- 172.21.15.28:0/695219021 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3360018680 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.571+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f33741b1180 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.572+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f337406a230 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.575+0000 7f3371600700 1 -- 172.21.15.28:0/695219021 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3360010430 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.577+0000 7f3371600700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f335c0b29f0 0x7f335c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.577+0000 7f3371600700 1 -- 172.21.15.28:0/695219021 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f33600e2dc0 con 0x7f337410dc70 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.577+0000 7f3379000700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f335c0b29f0 0x7f335c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.578+0000 7f3379000700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f335c0b29f0 0x7f335c0b4eb0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f3368009fd0 tx=0x7f3368009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:18:57.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.579+0000 7f3371600700 1 -- 172.21.15.28:0/695219021 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f33600aa730 con 0x7f337410dc70 2024-05-20T20:18:57.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:57 smithi154 ceph-mon[36314]: pgmap v416: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 217 MiB/s wr, 1.71k op/s 2024-05-20T20:18:57.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:57 smithi033 ceph-mon[36179]: pgmap v416: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 217 MiB/s wr, 1.71k op/s 2024-05-20T20:18:57.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.875+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f33741b1380 con 0x7f337410dc70 2024-05-20T20:18:57.877 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.876+0000 7f3371600700 1 -- 172.21.15.28:0/695219021 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f33600aa030 con 0x7f337410dc70 2024-05-20T20:18:57.877 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:18:57.877 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:18:57.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f335c0b29f0 msgr2=0x7f335c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f335c0b29f0 0x7f335c0b4eb0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f3368009fd0 tx=0x7f3368009260 comp rx=0 tx=0).stop 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f337410dc70 msgr2=0x7f33741a4b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f337410dc70 0x7f33741a4b90 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f336000e9a0 tx=0x7f336000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 shutdown_connections 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f335c0b29f0 0x7f335c0b4eb0 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f33741185d0 0x7f33741a8ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f337410aa80 0x7f33741a4650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 --2- 172.21.15.28:0/695219021 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f337410dc70 0x7f33741a4b90 unknown :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 >> 172.21.15.28:0/695219021 conn(0x7f337406ea90 msgr2=0x7f3374119d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:18:57.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.879+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 shutdown_connections 2024-05-20T20:18:57.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:18:57.880+0000 7f337ba00700 1 -- 172.21.15.28:0/695219021 wait complete. 2024-05-20T20:18:57.882 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:18:58.610 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:58 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/695219021' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:58.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:58 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/695219021' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:58.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:58 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/695219021' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:18:59.429 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:4 damage ls 2024-05-20T20:18:59.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:18:59 smithi154 ceph-mon[36314]: pgmap v417: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 217 MiB/s wr, 1.71k op/s 2024-05-20T20:18:59.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:18:59 smithi033 ceph-mon[36179]: pgmap v417: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 217 MiB/s wr, 1.71k op/s 2024-05-20T20:18:59.725 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:18:59 smithi028 ceph-mon[31178]: pgmap v417: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 217 MiB/s wr, 1.71k op/s 2024-05-20T20:19:00.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 -- 172.21.15.28:0/3751971634 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c109670 msgr2=0x7f665c109a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:00.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 --2- 172.21.15.28:0/3751971634 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c109670 0x7f665c109a50 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f6650009920 tx=0x7f6650009c30 comp rx=0 tx=0).stop 2024-05-20T20:19:00.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 -- 172.21.15.28:0/3751971634 shutdown_connections 2024-05-20T20:19:00.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 --2- 172.21.15.28:0/3751971634 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f665c103f90 0x7f665c110440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 --2- 172.21.15.28:0/3751971634 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c109670 0x7f665c109a50 unknown :-1 s=CLOSED pgs=618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.039 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 --2- 172.21.15.28:0/3751971634 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f665c1035d0 0x7f665c103a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 -- 172.21.15.28:0/3751971634 >> 172.21.15.28:0/3751971634 conn(0x7f665c07a730 msgr2=0x7f665c07ab40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:00.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 -- 172.21.15.28:0/3751971634 shutdown_connections 2024-05-20T20:19:00.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.038+0000 7f6663a00700 1 -- 172.21.15.28:0/3751971634 wait complete. 2024-05-20T20:19:00.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.039+0000 7f6663a00700 1 Processor -- start 2024-05-20T20:19:00.040 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6663a00700 1 -- start start 2024-05-20T20:19:00.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6663a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c1035d0 0x7f665c075840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:00.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6663a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f665c103f90 0x7f665c075d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:00.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6663a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f665c109670 0x7f665c0762c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:00.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6663a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f665c1137e0 con 0x7f665c103f90 2024-05-20T20:19:00.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6663a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f665c113660 con 0x7f665c1035d0 2024-05-20T20:19:00.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6663a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f665c113960 con 0x7f665c109670 2024-05-20T20:19:00.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6661000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c1035d0 0x7f665c075840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:00.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.040+0000 7f6661000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c1035d0 0x7f665c075840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41462/0 (socket says 172.21.15.28:41462) 2024-05-20T20:19:00.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661000700 1 -- 172.21.15.28:0/2332077110 learned_addr learned my addr 172.21.15.28:0/2332077110 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:00.042 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f665be00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f665c103f90 0x7f665c075d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661a00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f665c109670 0x7f665c0762c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661000700 1 -- 172.21.15.28:0/2332077110 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f665c109670 msgr2=0x7f665c0762c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661000700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f665c109670 0x7f665c0762c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661000700 1 -- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f665c103f90 msgr2=0x7f665c075d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661000700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f665c103f90 0x7f665c075d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661000700 1 -- 172.21.15.28:0/2332077110 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f665c1a81e0 con 0x7f665c1035d0 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f665be00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f665c103f90 0x7f665c075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.041+0000 7f6661000700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c1035d0 0x7f665c075840 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f6650000c00 tx=0x7f665000fa40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.042+0000 7f6659600700 1 -- 172.21.15.28:0/2332077110 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6650027070 con 0x7f665c1035d0 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.042+0000 7f6659600700 1 -- 172.21.15.28:0/2332077110 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f6650005670 con 0x7f665c1035d0 2024-05-20T20:19:00.043 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.042+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f665c1a8470 con 0x7f665c1035d0 2024-05-20T20:19:00.044 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.042+0000 7f6659600700 1 -- 172.21.15.28:0/2332077110 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f66500049c0 con 0x7f665c1035d0 2024-05-20T20:19:00.044 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.042+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f665c1a88c0 con 0x7f665c1035d0 2024-05-20T20:19:00.044 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.043+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f665c10bb00 con 0x7f665c1035d0 2024-05-20T20:19:00.047 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.046+0000 7f6659600700 1 -- 172.21.15.28:0/2332077110 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f665002a070 con 0x7f665c1035d0 2024-05-20T20:19:00.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.048+0000 7f6659600700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66440b2a70 0x7f66440b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:00.048 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.048+0000 7f6659600700 1 -- 172.21.15.28:0/2332077110 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f66500e3440 con 0x7f665c1035d0 2024-05-20T20:19:00.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.048+0000 7f665be00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66440b2a70 0x7f66440b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:00.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.049+0000 7f665be00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66440b2a70 0x7f66440b4f30 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f665c07ad80 tx=0x7f6648009310 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:00.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.049+0000 7f6659600700 1 -- 172.21.15.28:0/2332077110 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f6650031020 con 0x7f665c1035d0 2024-05-20T20:19:00.080 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:00.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.347+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f665c10b4e0 con 0x7f665c1035d0 2024-05-20T20:19:00.348 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.348+0000 7f6659600700 1 -- 172.21.15.28:0/2332077110 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f6650013030 con 0x7f665c1035d0 2024-05-20T20:19:00.349 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:19:00.349 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:19:00.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66440b2a70 msgr2=0x7f66440b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:00.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66440b2a70 0x7f66440b4f30 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f665c07ad80 tx=0x7f6648009310 comp rx=0 tx=0).stop 2024-05-20T20:19:00.352 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c1035d0 msgr2=0x7f665c075840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c1035d0 0x7f665c075840 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f6650000c00 tx=0x7f665000fa40 comp rx=0 tx=0).stop 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 shutdown_connections 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f665c109670 0x7f665c0762c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f66440b2a70 0x7f66440b4f30 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f665c1035d0 0x7f665c075840 unknown :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 --2- 172.21.15.28:0/2332077110 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f665c103f90 0x7f665c075d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.351+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 >> 172.21.15.28:0/2332077110 conn(0x7f665c07a730 msgr2=0x7f665c102380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.352+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 shutdown_connections 2024-05-20T20:19:00.353 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:00.352+0000 7f6663a00700 1 -- 172.21.15.28:0/2332077110 wait complete. 2024-05-20T20:19:00.354 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:19:01.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:01 smithi028 ceph-mon[31178]: pgmap v418: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 212 MiB/s wr, 1.69k op/s 2024-05-20T20:19:01.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:01 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2332077110' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:19:01.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:01 smithi154 ceph-mon[36314]: pgmap v418: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 212 MiB/s wr, 1.69k op/s 2024-05-20T20:19:01.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:01 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2332077110' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:19:01.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:01 smithi033 ceph-mon[36179]: pgmap v418: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 212 MiB/s wr, 1.69k op/s 2024-05-20T20:19:01.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:01 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2332077110' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:19:01.734 DEBUG:tasks.quiescer.fs.[cephfs]:Running ceph command: 'tell mds.24542 quiesce db --cancel --all' 2024-05-20T20:19:01.734 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- timeout 105 ceph tell mds.24542 quiesce db --cancel --all 2024-05-20T20:19:02.389 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 -- 172.21.15.28:0/2658450790 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc114710 msgr2=0x7fe8fc116b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 --2- 172.21.15.28:0/2658450790 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc114710 0x7fe8fc116b00 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7fe8f400b0a0 tx=0x7fe8f400b3b0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 -- 172.21.15.28:0/2658450790 shutdown_connections 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 --2- 172.21.15.28:0/2658450790 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc114710 0x7fe8fc116b00 unknown :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 --2- 172.21.15.28:0/2658450790 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe8fc075770 0x7fe8fc075b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 --2- 172.21.15.28:0/2658450790 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe8fc076090 0x7fe8fc1141d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 -- 172.21.15.28:0/2658450790 >> 172.21.15.28:0/2658450790 conn(0x7fe8fc07a6a0 msgr2=0x7fe8fc07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:03.246 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.189+0000 7fe900e00700 1 -- 172.21.15.28:0/2658450790 shutdown_connections 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.190+0000 7fe900e00700 1 -- 172.21.15.28:0/2658450790 wait complete. 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.190+0000 7fe900e00700 1 Processor -- start 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe900e00700 1 -- start start 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe900e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc075770 0x7fe8fc1a01d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe900e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe8fc076090 0x7fe8fc1a0710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe900e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe8fc114710 0x7fe8fc1a4aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe900e00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc119ce0 con 0x7fe8fc075770 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe900e00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc119b60 con 0x7fe8fc076090 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe900e00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc119e60 con 0x7fe8fc114710 2024-05-20T20:19:03.247 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.191+0000 7fe8f9e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc075770 0x7fe8fc1a01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc075770 0x7fe8fc1a01d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41014/0 (socket says 172.21.15.28:41014) 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9e00700 1 -- 172.21.15.28:0/346898921 learned_addr learned my addr 172.21.15.28:0/346898921 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9400700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe8fc076090 0x7fe8fc1a0710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8fa800700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe8fc114710 0x7fe8fc1a4aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9400700 1 -- 172.21.15.28:0/346898921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe8fc114710 msgr2=0x7fe8fc1a4aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9400700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe8fc114710 0x7fe8fc1a4aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9400700 1 -- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc075770 msgr2=0x7fe8fc1a01d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9400700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc075770 0x7fe8fc1a01d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9400700 1 -- 172.21.15.28:0/346898921 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe8fc1a51e0 con 0x7fe8fc076090 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc075770 0x7fe8fc1a01d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.192+0000 7fe8f9400700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe8fc076090 0x7fe8fc1a0710 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7fe8e800e9f0 tx=0x7fe8e800ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.193+0000 7fe8f2a00700 1 -- 172.21.15.28:0/346898921 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8e800c9c0 con 0x7fe8fc076090 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.193+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe8fc1a54d0 con 0x7fe8fc076090 2024-05-20T20:19:03.248 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.193+0000 7fe8f2a00700 1 -- 172.21.15.28:0/346898921 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe8e800cb60 con 0x7fe8fc076090 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.193+0000 7fe8f2a00700 1 -- 172.21.15.28:0/346898921 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8e8018680 con 0x7fe8fc076090 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.193+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe8fc1acd50 con 0x7fe8fc076090 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.197+0000 7fe8f2a00700 1 -- 172.21.15.28:0/346898921 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fe8e8010430 con 0x7fe8fc076090 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 Processor -- start 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 -- start start 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b4290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22ae260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22aece0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.249 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555ac220ce40 con 0x555ac22ae880 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555ac21fc9e0 con 0x555ac22b3eb0 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.198+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555ac2200a20 con 0x555ac22b47d0 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.199+0000 7fe8fa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b4290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.199+0000 7fe8fa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b4290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:41476/0 (socket says 172.21.15.28:41476) 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.199+0000 7fe8fa800700 1 -- 172.21.15.28:0/3066271117 learned_addr learned my addr 172.21.15.28:0/3066271117 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.199+0000 7fe8f9400700 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22aece0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.199+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22ae260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.199+0000 7fe8f2a00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe8e00b29f0 0x7fe8e00b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8fa800700 1 -- 172.21.15.28:0/3066271117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 msgr2=0x555ac22ae260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8fa800700 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22ae260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8f2a00700 1 -- 172.21.15.28:0/346898921 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fe8e80e2770 con 0x7fe8fc076090 2024-05-20T20:19:03.250 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8fa800700 1 -- 172.21.15.28:0/3066271117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 msgr2=0x555ac22aece0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8fa800700 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22aece0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8fa800700 1 -- 172.21.15.28:0/3066271117 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555ac21c5420 con 0x555ac22b3eb0 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22ae260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8f9400700 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22aece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.200+0000 7fe8fa800700 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b4290 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7fe8f4015000 tx=0x7fe8f4009360 comp rx=0 tx=0).ready entity=mon.2 client_cookie=65e63f31c6e64993 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.201+0000 7fe8e7400700 1 -- 172.21.15.28:0/3066271117 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8f400bcb0 con 0x555ac22b3eb0 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.201+0000 7fe8fa800700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe8e00b29f0 0x7fe8e00b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.201+0000 7fe8e7400700 1 -- 172.21.15.28:0/3066271117 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe8f40074e0 con 0x555ac22b3eb0 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.201+0000 7fe8e7400700 1 -- 172.21.15.28:0/3066271117 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8f40052a0 con 0x555ac22b3eb0 2024-05-20T20:19:03.251 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3066271117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 msgr2=0x555ac22b4290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b4290 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7fe8f4015000 tx=0x7fe8f4009360 comp rx=0 tx=0).stop 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe8fa800700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe8e00b29f0 0x7fe8e00b4eb0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fe8f400bfa0 tx=0x7fe8f402a040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3066271117 shutdown_connections 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22aece0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b4290 unknown :-1 s=CLOSED pgs=621 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3066271117 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22ae260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3066271117 >> 172.21.15.28:0/3066271117 conn(0x555ac21dcbe0 msgr2=0x555ac22a9010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3066271117 shutdown_connections 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.202+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3066271117 wait complete. 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 -- start start 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 Processor -- start 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22ba4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.252 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b7890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22b7dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x555ac2102fa0 con 0x555ac22ae880 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x555ac219d760 con 0x555ac22b3eb0 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.203+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x555ac22bb1e0 con 0x555ac22b47d0 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe90b6aeb80 10 client.0 _ll_register_callbacks cb 0x555ac21a6c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8f9e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22ba4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22b7dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22b7dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37916/0 (socket says 172.21.15.28:37916) 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 -- 172.21.15.28:0/3355474551 learned_addr learned my addr 172.21.15.28:0/3355474551 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:03.253 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8f9400700 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b7890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 -- 172.21.15.28:0/3355474551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 msgr2=0x555ac22b7890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b7890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 -- 172.21.15.28:0/3355474551 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 msgr2=0x555ac22ba4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22ba4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.204+0000 7fe8fa800700 1 -- 172.21.15.28:0/3355474551 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555ac214c8d0 con 0x555ac22b47d0 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.205+0000 7fe8f9400700 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b7890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.205+0000 7fe8fa800700 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22b7dd0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7fe8f4026000 tx=0x7fe8f400b720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f95c91e973707bea server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.205+0000 7fe8dbe00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:19:03.254 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.205+0000 7fe8dbe00700 1 -- 172.21.15.28:0/3355474551 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8f40052a0 con 0x555ac22b47d0 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.205+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe8f40074e0 con 0x555ac22b47d0 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.205+0000 7fe8dbe00700 1 -- 172.21.15.28:0/3355474551 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fe8f400f040 con 0x555ac22b47d0 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe8dbe00700 1 -- 172.21.15.28:0/3355474551 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe8f400bcc0 con 0x555ac22b47d0 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 10 client.28021 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555ac21cd060 con 0x555ac22b47d0 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe8dbe00700 1 -- 172.21.15.28:0/3355474551 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe8f40074e0 con 0x555ac22b47d0 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe8dbe00700 1 -- 172.21.15.28:0/3355474551 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe8f4011070 con 0x555ac22b47d0 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 10 client.28021 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 10 client.28021 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 20 client.28021 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac22c7ba0 0x555ac22c9f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.255 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 4 client.28021 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.206+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x555ac220ed90 con 0x555ac22c7ba0 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.207+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac22c7ba0 0x555ac22c9f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.207+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac22c7ba0 0x555ac22c9f90 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=ddeb8b4b2e615a73 server_cookie=cbb22b3b0da3117c in_seq=0 out_seq=0 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.207+0000 7fe8dbe00700 10 client.28021 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.208+0000 7fe8dbe00700 1 -- 172.21.15.28:0/3355474551 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x555ac220ed90 con 0x555ac22c7ba0 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.208+0000 7fe8dbe00700 10 client.28021 handle_command_reply: tid=0 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.208+0000 7fe90b6aeb80 1 client.28021 shutdown 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.208+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 msgr2=0x555ac22b7dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.208+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22b7dd0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7fe8f4026000 tx=0x7fe8f400b720 comp rx=0 tx=0).stop 2024-05-20T20:19:03.256 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 shutdown_connections 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac22c7ba0 0x555ac22c9f90 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22b47d0 0x555ac22b7dd0 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22ae880 0x555ac22ba4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/3355474551 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b3eb0 0x555ac22b7890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe8dbe00700 0 client.28021 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 >> 172.21.15.28:0/3355474551 conn(0x555ac21dcbe0 msgr2=0x555ac22ad250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 shutdown_connections 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/3355474551 wait complete. 2024-05-20T20:19:03.257 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.209+0000 7fe90b6aeb80 20 client.28021 trim_cache size 0 max 16384 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.227+0000 7fe90b6aeb80 1 Processor -- start 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.227+0000 7fe90b6aeb80 1 -- start start 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22ae260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22ba4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22aec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc1b4490 con 0x555ac22b3eb0 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc1b5fd0 con 0x555ac22b47d0 2024-05-20T20:19:03.258 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc1b4b60 con 0x555ac22ae880 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe8f9400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22ba4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe8fa800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22aec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe8f9400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22ba4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41034/0 (socket says 172.21.15.28:41034) 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe8f9400700 1 -- 172.21.15.28:0/1389668758 learned_addr learned my addr 172.21.15.28:0/1389668758 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.228+0000 7fe8f9e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22ae260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8f9e00700 1 -- 172.21.15.28:0/1389668758 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 msgr2=0x555ac22aec60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22aec60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.259 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8f9e00700 1 -- 172.21.15.28:0/1389668758 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 msgr2=0x555ac22ba4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22ba4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8f9e00700 1 -- 172.21.15.28:0/1389668758 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x555ac22a34c0 con 0x555ac22ae880 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8fa800700 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22aec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8f9400700 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22ba4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22ae260 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x555ac22bae40 tx=0x7fe8ec009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7721ccb4bb878dc4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8e6a00700 1 -- 172.21.15.28:0/1389668758 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8ec003fc0 con 0x555ac22ae880 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8e6a00700 1 -- 172.21.15.28:0/1389668758 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe8ec004ce0 con 0x555ac22ae880 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.229+0000 7fe8e6a00700 1 -- 172.21.15.28:0/1389668758 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8ec010690 con 0x555ac22ae880 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/1389668758 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 msgr2=0x555ac22ae260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22ae260 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x555ac22bae40 tx=0x7fe8ec009260 comp rx=0 tx=0).stop 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/1389668758 shutdown_connections 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22aec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.260 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22ae260 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/1389668758 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22ba4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/1389668758 >> 172.21.15.28:0/1389668758 conn(0x555ac21dcbe0 msgr2=0x555ac22b61c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/1389668758 shutdown_connections 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.230+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/1389668758 wait complete. 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 -- start start 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 Processor -- start 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22cd6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22caa40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22caf80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc1b5900 con 0x555ac22b3eb0 2024-05-20T20:19:03.261 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc1b5230 con 0x555ac22b47d0 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.231+0000 7fe90b6aeb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe8fc1b2450 con 0x555ac22ae880 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe90b6aeb80 10 client.0 _ll_register_callbacks cb 0x555ac21a6c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22caa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22caa40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41040/0 (socket says 172.21.15.28:41040) 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 -- 172.21.15.28:0/786291900 learned_addr learned my addr 172.21.15.28:0/786291900 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8fa800700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22caf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22cd6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 -- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 msgr2=0x555ac22cd6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22cd6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 -- 172.21.15.28:0/786291900 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 msgr2=0x555ac22caf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22caf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.262 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8f9400700 1 -- 172.21.15.28:0/786291900 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x555ac22cb790 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.232+0000 7fe8fa800700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22caf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe8f9400700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22caa40 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fe8e80a5a60 tx=0x7fe8e80a6050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c67c1592deeb8fee server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe8dbe00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe8dbe00700 1 -- 172.21.15.28:0/786291900 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8e80a58a0 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe8dbe00700 1 -- 172.21.15.28:0/786291900 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7fe8e8010720 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fe8ec004ce0 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe8dbe00700 1 -- 172.21.15.28:0/786291900 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fe8e80a5170 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe90b6aeb80 10 client.17481 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x555ac2332470 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.233+0000 7fe8dbe00700 1 -- 172.21.15.28:0/786291900 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe8ec004ce0 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe8dbe00700 1 -- 172.21.15.28:0/786291900 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fe8e80a5350 con 0x555ac22b3eb0 2024-05-20T20:19:03.263 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe90b6aeb80 10 client.17481 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe90b6aeb80 10 client.17481 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.c 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe90b6aeb80 20 client.17481 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac2332ad0 0x555ac2334ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe90b6aeb80 4 client.17481 mds_command: new command op to 24475 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 --> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] -- command(tid 0: {"prefix": "damage ls"}) -- 0x555ac2208570 con 0x555ac2332ad0 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.234+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac2332ad0 0x555ac2334ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe8f9e00700 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac2332ad0 0x555ac2334ec0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=79c0f2de6e0fd4f3 server_cookie=a3bf9576dedf8ce8 in_seq=0 out_seq=0 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe8dbe00700 10 client.17481 ms_handle_connect on v2:172.21.15.154:6836/2808233682 2024-05-20T20:19:03.264 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe8dbe00700 1 -- 172.21.15.28:0/786291900 <== mds.4 v2:172.21.15.154:6836/2808233682 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x555ac2208570 con 0x555ac2332ad0 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe8dbe00700 10 client.17481 handle_command_reply: tid=0 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 client.17481 shutdown 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 msgr2=0x555ac22caa40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22caa40 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fe8e80a5a60 tx=0x7fe8e80a6050 comp rx=0 tx=0).stop 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 shutdown_connections 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x555ac22b47d0 0x555ac22caf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:6836/2808233682,v1:172.21.15.154:6837/2808233682] conn(0x555ac2332ad0 0x555ac2334ec0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x555ac22ae880 0x555ac22cd6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.265 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 --2- 172.21.15.28:0/786291900 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x555ac22b3eb0 0x555ac22caa40 unknown :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe8dbe00700 0 client.17481 ms_handle_reset on v2:172.21.15.154:6836/2808233682 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 >> 172.21.15.28:0/786291900 conn(0x555ac21dcbe0 msgr2=0x555ac22b6140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 shutdown_connections 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.235+0000 7fe90b6aeb80 1 -- 172.21.15.28:0/786291900 wait complete. 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.236+0000 7fe90b6aeb80 20 client.17481 trim_cache size 0 max 16384 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.236+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe8e00b29f0 msgr2=0x7fe8e00b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.236+0000 7fe900e00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe8e00b29f0 0x7fe8e00b4eb0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fe8f400bfa0 tx=0x7fe8f402a040 comp rx=0 tx=0).stop 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.236+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe8fc076090 msgr2=0x7fe8fc1a0710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.236+0000 7fe900e00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe8fc076090 0x7fe8fc1a0710 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7fe8e800e9f0 tx=0x7fe8e800ed00 comp rx=0 tx=0).stop 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 shutdown_connections 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe8fc114710 0x7fe8fc1a4aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.266 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe8fc075770 0x7fe8fc1a01d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe8e00b29f0 0x7fe8e00b4eb0 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 --2- 172.21.15.28:0/346898921 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe8fc076090 0x7fe8fc1a0710 unknown :-1 s=CLOSED pgs=620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:03.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 >> 172.21.15.28:0/346898921 conn(0x7fe8fc07a6a0 msgr2=0x7fe8fc1101b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:03.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 shutdown_connections 2024-05-20T20:19:03.267 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:03.237+0000 7fe900e00700 1 -- 172.21.15.28:0/346898921 wait complete. 2024-05-20T20:19:03.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:03 smithi028 ceph-mon[31178]: pgmap v419: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 98 MiB/s wr, 718 op/s 2024-05-20T20:19:03.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:03 smithi154 ceph-mon[36314]: pgmap v419: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 98 MiB/s wr, 718 op/s 2024-05-20T20:19:03.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:03 smithi033 ceph-mon[36179]: pgmap v419: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 98 MiB/s wr, 718 op/s 2024-05-20T20:19:05.246 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:3 damage ls 2024-05-20T20:19:05.463 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:05 smithi028 ceph-mon[31178]: pgmap v420: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 98 MiB/s wr, 741 op/s 2024-05-20T20:19:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:05 smithi154 ceph-mon[36314]: pgmap v420: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 98 MiB/s wr, 741 op/s 2024-05-20T20:19:05.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:05 smithi033 ceph-mon[36179]: pgmap v420: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 98 MiB/s wr, 741 op/s 2024-05-20T20:19:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.816+0000 7efd90c00700 1 -- 172.21.15.28:0/2993424179 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c103f80 msgr2=0x7efd8c110410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.816+0000 7efd90c00700 1 --2- 172.21.15.28:0/2993424179 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c103f80 0x7efd8c110410 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7efd84009f90 tx=0x7efd8400b330 comp rx=0 tx=0).stop 2024-05-20T20:19:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.816+0000 7efd90c00700 1 -- 172.21.15.28:0/2993424179 shutdown_connections 2024-05-20T20:19:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.816+0000 7efd90c00700 1 --2- 172.21.15.28:0/2993424179 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c103f80 0x7efd8c110410 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.816+0000 7efd90c00700 1 --2- 172.21.15.28:0/2993424179 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efd8c1095d0 0x7efd8c1099b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.816+0000 7efd90c00700 1 --2- 172.21.15.28:0/2993424179 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efd8c1035c0 0x7efd8c103a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.818 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.816+0000 7efd90c00700 1 -- 172.21.15.28:0/2993424179 >> 172.21.15.28:0/2993424179 conn(0x7efd8c07a610 msgr2=0x7efd8c07aa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.817+0000 7efd90c00700 1 -- 172.21.15.28:0/2993424179 shutdown_connections 2024-05-20T20:19:05.819 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.817+0000 7efd90c00700 1 -- 172.21.15.28:0/2993424179 wait complete. 2024-05-20T20:19:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.818+0000 7efd90c00700 1 Processor -- start 2024-05-20T20:19:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.818+0000 7efd90c00700 1 -- start start 2024-05-20T20:19:05.820 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.818+0000 7efd90c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c1035c0 0x7efd8c19f7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.818+0000 7efd90c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efd8c103f80 0x7efd8c19fd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.818+0000 7efd90c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efd8c1095d0 0x7efd8c199980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd90c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7efd8c113780 con 0x7efd8c1095d0 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd90c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7efd8c113600 con 0x7efd8c103f80 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd90c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7efd8c113900 con 0x7efd8c1035c0 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c1035c0 0x7efd8c19f7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c1035c0 0x7efd8c19f7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37960/0 (socket says 172.21.15.28:37960) 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89e00700 1 -- 172.21.15.28:0/1836987336 learned_addr learned my addr 172.21.15.28:0/1836987336 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efd8c103f80 0x7efd8c19fd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.821 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd8a800700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efd8c1095d0 0x7efd8c199980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89400700 1 -- 172.21.15.28:0/1836987336 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c1035c0 msgr2=0x7efd8c19f7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89400700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c1035c0 0x7efd8c19f7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89400700 1 -- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efd8c1095d0 msgr2=0x7efd8c199980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89400700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efd8c1095d0 0x7efd8c199980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.819+0000 7efd89400700 1 -- 172.21.15.28:0/1836987336 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efd8c19a0e0 con 0x7efd8c103f80 2024-05-20T20:19:05.822 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.820+0000 7efd89400700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efd8c103f80 0x7efd8c19fd00 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7efd7800e9c0 tx=0x7efd7800ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.820+0000 7efd82a00700 1 -- 172.21.15.28:0/1836987336 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd7800c9c0 con 0x7efd8c103f80 2024-05-20T20:19:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.820+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efd8c19a3d0 con 0x7efd8c103f80 2024-05-20T20:19:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.820+0000 7efd82a00700 1 -- 172.21.15.28:0/1836987336 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efd7800cb60 con 0x7efd8c103f80 2024-05-20T20:19:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.820+0000 7efd82a00700 1 -- 172.21.15.28:0/1836987336 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd78018680 con 0x7efd8c103f80 2024-05-20T20:19:05.823 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.820+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efd8c1a6690 con 0x7efd8c103f80 2024-05-20T20:19:05.826 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.825+0000 7efd82a00700 1 -- 172.21.15.28:0/1836987336 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7efd78010430 con 0x7efd8c103f80 2024-05-20T20:19:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 Processor -- start 2024-05-20T20:19:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 -- start start 2024-05-20T20:19:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f01db40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f017ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.827 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f018760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bc5ef71eb0 con 0x55bc5f018300 2024-05-20T20:19:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bc5ef61a50 con 0x55bc5f01d760 2024-05-20T20:19:05.828 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.826+0000 7efd9b618b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bc5ef65a90 con 0x55bc5f01e080 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd89400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f018760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd89e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f017ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd8a800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f01db40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd89400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f018760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41068/0 (socket says 172.21.15.28:41068) 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd82a00700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efd700b29f0 0x7efd700b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd89400700 1 -- 172.21.15.28:0/1303947600 learned_addr learned my addr 172.21.15.28:0/1303947600 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd82a00700 1 -- 172.21.15.28:0/1836987336 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7efd780e2790 con 0x7efd8c103f80 2024-05-20T20:19:05.829 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.827+0000 7efd8a800700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efd700b29f0 0x7efd700b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd89e00700 1 -- 172.21.15.28:0/1303947600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 msgr2=0x55bc5f01db40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd89e00700 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f01db40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd89e00700 1 -- 172.21.15.28:0/1303947600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 msgr2=0x55bc5f018760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd89e00700 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f018760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd89e00700 1 -- 172.21.15.28:0/1303947600 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bc5ef2a490 con 0x55bc5f01e080 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd8a800700 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f01db40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd89400700 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f018760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:05.830 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.828+0000 7efd89e00700 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f017ce0 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7efd7c009d10 tx=0x7efd7c011720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=de3e23d20a4ec88 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:05.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.829+0000 7efd77400700 1 -- 172.21.15.28:0/1303947600 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd7c004230 con 0x55bc5f01e080 2024-05-20T20:19:05.831 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.829+0000 7efd8a800700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efd700b29f0 0x7efd700b4eb0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7efd8400b6c0 tx=0x7efd8400b330 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.829+0000 7efd77400700 1 -- 172.21.15.28:0/1303947600 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efd7c0043d0 con 0x55bc5f01e080 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.829+0000 7efd77400700 1 -- 172.21.15.28:0/1303947600 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd7c0197c0 con 0x55bc5f01e080 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 -- 172.21.15.28:0/1303947600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 msgr2=0x55bc5f017ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f017ce0 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7efd7c009d10 tx=0x7efd7c011720 comp rx=0 tx=0).stop 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 -- 172.21.15.28:0/1303947600 shutdown_connections 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f018760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f01db40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 --2- 172.21.15.28:0/1303947600 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f017ce0 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 -- 172.21.15.28:0/1303947600 >> 172.21.15.28:0/1303947600 conn(0x55bc5ef41c50 msgr2=0x55bc5f00e390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 -- 172.21.15.28:0/1303947600 shutdown_connections 2024-05-20T20:19:05.832 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 -- 172.21.15.28:0/1303947600 wait complete. 2024-05-20T20:19:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.830+0000 7efd9b618b80 1 -- start start 2024-05-20T20:19:05.833 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.831+0000 7efd9b618b80 1 Processor -- start 2024-05-20T20:19:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.831+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f018300 0x55bc5f023cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.831+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f021060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.831+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f0215a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.831+0000 7efd9b618b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55bc5ee68010 con 0x55bc5f01d760 2024-05-20T20:19:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.831+0000 7efd9b618b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55bc5ef027d0 con 0x55bc5f018300 2024-05-20T20:19:05.835 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.831+0000 7efd9b618b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55bc5f024b80 con 0x55bc5f01e080 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.832+0000 7efd9b618b80 10 client.0 _ll_register_callbacks cb 0x55bc5ef0bce0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.832+0000 7efd89400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f021060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.832+0000 7efd89400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f021060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41070/0 (socket says 172.21.15.28:41070) 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.832+0000 7efd89400700 1 -- 172.21.15.28:0/4010183246 learned_addr learned my addr 172.21.15.28:0/4010183246 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.832+0000 7efd89e00700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f018300 0x55bc5f023cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.832+0000 7efd8a800700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f0215a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.832+0000 7efd89400700 1 -- 172.21.15.28:0/4010183246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 msgr2=0x55bc5f0215a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd89400700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f0215a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd89400700 1 -- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f018300 msgr2=0x55bc5f023cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.836 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd89400700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f018300 0x55bc5f023cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd89400700 1 -- 172.21.15.28:0/4010183246 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bc5eeb1940 con 0x55bc5f01d760 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd89e00700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f018300 0x55bc5f023cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd8a800700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f0215a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd89400700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f021060 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7efd78009740 tx=0x7efd780a9f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e28d1aa9b177d05 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd6be00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd6be00700 1 -- 172.21.15.28:0/4010183246 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd780b21a0 con 0x55bc5f01d760 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd6be00700 1 -- 172.21.15.28:0/4010183246 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7efd780cc2e0 con 0x55bc5f01d760 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.833+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efd7c0043d0 con 0x55bc5f01d760 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd6be00700 1 -- 172.21.15.28:0/4010183246 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7efd780a9510 con 0x55bc5f01d760 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 10 client.17493 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:05.837 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55bc5ef320d0 con 0x55bc5f01d760 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd6be00700 1 -- 172.21.15.28:0/4010183246 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efd7c0043d0 con 0x55bc5f01d760 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd6be00700 1 -- 172.21.15.28:0/4010183246 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7efd780e2b80 con 0x55bc5f01d760 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 10 client.17493 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 10 client.17493 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 20 client.17493 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f0316d0 0x55bc5f033b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 4 client.17493 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:19:05.838 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.834+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55bc5ef73e00 con 0x55bc5f0316d0 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.835+0000 7efd89e00700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f0316d0 0x55bc5f033b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.835+0000 7efd89e00700 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f0316d0 0x55bc5f033b30 crc :-1 s=READY pgs=273 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b32cb6f0ac8cb7ff server_cookie=3e1481daf0615d48 in_seq=0 out_seq=0 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.836+0000 7efd6be00700 10 client.17493 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.836+0000 7efd6be00700 1 -- 172.21.15.28:0/4010183246 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55bc5ef73e00 con 0x55bc5f0316d0 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.836+0000 7efd6be00700 10 client.17493 handle_command_reply: tid=0 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.836+0000 7efd9b618b80 1 client.17493 shutdown 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 msgr2=0x55bc5f021060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.840 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f021060 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7efd78009740 tx=0x7efd780a9f40 comp rx=0 tx=0).stop 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 shutdown_connections 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f0316d0 0x55bc5f033b30 crc :-1 s=READY pgs=273 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f0215a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f018300 0x55bc5f023cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 --2- 172.21.15.28:0/4010183246 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f021060 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 >> 172.21.15.28:0/4010183246 conn(0x55bc5ef41c50 msgr2=0x55bc5f0128c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd6be00700 0 client.17493 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 shutdown_connections 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 1 -- 172.21.15.28:0/4010183246 wait complete. 2024-05-20T20:19:05.841 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.837+0000 7efd9b618b80 20 client.17493 trim_cache size 0 max 16384 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.860+0000 7efd9b618b80 1 Processor -- start 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.860+0000 7efd9b618b80 1 -- start start 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f018300 0x55bc5f01b530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f012fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01e080 0x55bc5f02dcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd9b618b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7efd6c00d620 con 0x55bc5f01d760 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd9b618b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7efd6c00f160 con 0x55bc5f01e080 2024-05-20T20:19:05.862 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd9b618b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7efd6c00dcf0 con 0x55bc5f018300 2024-05-20T20:19:05.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd89400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f012fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd8a800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01e080 0x55bc5f02dcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd89400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f012fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:41082/0 (socket says 172.21.15.28:41082) 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd89e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f018300 0x55bc5f01b530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.861+0000 7efd89400700 1 -- 172.21.15.28:0/2836556102 learned_addr learned my addr 172.21.15.28:0/2836556102 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.862+0000 7efd8a800700 1 -- 172.21.15.28:0/2836556102 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f018300 msgr2=0x55bc5f01b530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.862+0000 7efd8a800700 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f018300 0x55bc5f01b530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.862+0000 7efd8a800700 1 -- 172.21.15.28:0/2836556102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 msgr2=0x55bc5f012fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.862+0000 7efd8a800700 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f012fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.862+0000 7efd8a800700 1 -- 172.21.15.28:0/2836556102 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55bc5f0244c0 con 0x55bc5f01e080 2024-05-20T20:19:05.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.862+0000 7efd89400700 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f012fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:05.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.863+0000 7efd8a800700 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01e080 0x55bc5f02dcb0 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7efd84009530 tx=0x7efd84020000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aafced9c5ab68152 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:05.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.863+0000 7efd76a00700 1 -- 172.21.15.28:0/2836556102 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd8402e070 con 0x55bc5f01e080 2024-05-20T20:19:05.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.863+0000 7efd76a00700 1 -- 172.21.15.28:0/2836556102 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efd84007ab0 con 0x55bc5f01e080 2024-05-20T20:19:05.865 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.863+0000 7efd76a00700 1 -- 172.21.15.28:0/2836556102 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd8402a6a0 con 0x55bc5f01e080 2024-05-20T20:19:05.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 -- 172.21.15.28:0/2836556102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01e080 msgr2=0x55bc5f02dcb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01e080 0x55bc5f02dcb0 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7efd84009530 tx=0x7efd84020000 comp rx=0 tx=0).stop 2024-05-20T20:19:05.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 -- 172.21.15.28:0/2836556102 shutdown_connections 2024-05-20T20:19:05.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01e080 0x55bc5f02dcb0 unknown :-1 s=CLOSED pgs=623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f018300 0x55bc5f01b530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.866 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 --2- 172.21.15.28:0/2836556102 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f01d760 0x55bc5f012fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 -- 172.21.15.28:0/2836556102 >> 172.21.15.28:0/2836556102 conn(0x55bc5ef41c50 msgr2=0x55bc5f0128c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:05.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 -- 172.21.15.28:0/2836556102 shutdown_connections 2024-05-20T20:19:05.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.864+0000 7efd9b618b80 1 -- 172.21.15.28:0/2836556102 wait complete. 2024-05-20T20:19:05.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.865+0000 7efd9b618b80 1 -- start start 2024-05-20T20:19:05.867 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.865+0000 7efd9b618b80 1 Processor -- start 2024-05-20T20:19:05.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f02a080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f02a5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd9b618b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f026e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd9b618b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7efd6c00ea90 con 0x55bc5f018300 2024-05-20T20:19:05.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd9b618b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7efd6c00e3c0 con 0x55bc5f01d760 2024-05-20T20:19:05.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd9b618b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7efd6c00b5e0 con 0x55bc5f01e080 2024-05-20T20:19:05.868 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd9b618b80 10 client.0 _ll_register_callbacks cb 0x55bc5f00e9f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd8a800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f026e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd8a800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f026e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:37982/0 (socket says 172.21.15.28:37982) 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd8a800700 1 -- 172.21.15.28:0/3656940231 learned_addr learned my addr 172.21.15.28:0/3656940231 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd89e00700 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f02a080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.866+0000 7efd89400700 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f02a5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd8a800700 1 -- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 msgr2=0x55bc5f02a5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd8a800700 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f02a5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd8a800700 1 -- 172.21.15.28:0/3656940231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 msgr2=0x55bc5f02a080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd8a800700 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f02a080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd8a800700 1 -- 172.21.15.28:0/3656940231 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55bc5f027680 con 0x55bc5f01e080 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd8a800700 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f026e70 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7efd84004fd0 tx=0x7efd84004db0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bd27ce768c2b629e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:05.869 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd6be00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:19:05.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd6be00700 1 -- 172.21.15.28:0/3656940231 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efd8402e040 con 0x55bc5f01e080 2024-05-20T20:19:05.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.867+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7efd84007ab0 con 0x55bc5f01e080 2024-05-20T20:19:05.870 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.868+0000 7efd6be00700 1 -- 172.21.15.28:0/3656940231 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7efd8401f850 con 0x55bc5f01e080 2024-05-20T20:19:05.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.868+0000 7efd6be00700 1 -- 172.21.15.28:0/3656940231 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7efd8401fa50 con 0x55bc5f01e080 2024-05-20T20:19:05.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.868+0000 7efd9b618b80 10 client.28045 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:05.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.868+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55bc5f09ce00 con 0x55bc5f01e080 2024-05-20T20:19:05.872 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.868+0000 7efd6be00700 1 -- 172.21.15.28:0/3656940231 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efd84007ab0 con 0x55bc5f01e080 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd6be00700 1 -- 172.21.15.28:0/3656940231 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7efd84011070 con 0x55bc5f01e080 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd9b618b80 10 client.28045 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd9b618b80 10 client.28045 resolve_mds: validated gid 24542 aka daemon mds.b 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd9b618b80 20 client.28045 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd9b618b80 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f09d460 0x55bc5f09f880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd9b618b80 4 client.28045 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "quiesce db", "cancel": true, "all": true}] 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "quiesce db", "cancel": true, "all": true}) -- 0x55bc5ef6d5e0 con 0x55bc5f09d460 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.869+0000 7efd89e00700 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f09d460 0x55bc5f09f880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.870+0000 7efd6be00700 10 client.28045 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.870+0000 7efd89e00700 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f09d460 0x55bc5f09f880 crc :-1 s=READY pgs=274 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3db18460ac0ea99f server_cookie=e9ce5a00adb8079b in_seq=0 out_seq=0 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.870+0000 7efd6be00700 1 -- 172.21.15.28:0/3656940231 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+80 (crc 0 0 0) 0x55bc5ef6d5e0 con 0x55bc5f09d460 2024-05-20T20:19:05.873 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.870+0000 7efd6be00700 10 client.28045 handle_command_reply: tid=0 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.870+0000 7efd9b618b80 1 client.28045 shutdown 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.870+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 msgr2=0x55bc5f026e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.870+0000 7efd9b618b80 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f026e70 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7efd84004fd0 tx=0x7efd84004db0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 shutdown_connections 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x55bc5f09d460 0x55bc5f09f880 crc :-1 s=READY pgs=274 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55bc5f01e080 0x55bc5f026e70 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55bc5f018300 0x55bc5f02a080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 --2- 172.21.15.28:0/3656940231 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55bc5f01d760 0x55bc5f02a5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 >> 172.21.15.28:0/3656940231 conn(0x55bc5ef41c50 msgr2=0x55bc5f01e460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:05.874 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd6be00700 0 client.28045 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:05.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 shutdown_connections 2024-05-20T20:19:05.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 1 -- 172.21.15.28:0/3656940231 wait complete. 2024-05-20T20:19:05.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.871+0000 7efd9b618b80 20 client.28045 trim_cache size 0 max 16384 2024-05-20T20:19:05.878 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:19:05.878 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 29, 2024-05-20T20:19:05.879 INFO:teuthology.orchestra.run.smithi028.stdout: "leader": 24542, 2024-05-20T20:19:05.879 INFO:teuthology.orchestra.run.smithi028.stdout: "set_version": 36, 2024-05-20T20:19:05.879 INFO:teuthology.orchestra.run.smithi028.stdout: "sets": {} 2024-05-20T20:19:05.879 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.872+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efd700b29f0 msgr2=0x7efd700b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.872+0000 7efd90c00700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efd700b29f0 0x7efd700b4eb0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7efd8400b6c0 tx=0x7efd8400b330 comp rx=0 tx=0).stop 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.872+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efd8c103f80 msgr2=0x7efd8c19fd00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.872+0000 7efd90c00700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efd8c103f80 0x7efd8c19fd00 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7efd7800e9c0 tx=0x7efd7800ecd0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 shutdown_connections 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efd8c1095d0 0x7efd8c199980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efd8c1035c0 0x7efd8c19f7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efd700b29f0 0x7efd700b4eb0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 --2- 172.21.15.28:0/1836987336 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efd8c103f80 0x7efd8c19fd00 unknown :-1 s=CLOSED pgs=622 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 >> 172.21.15.28:0/1836987336 conn(0x7efd8c07a610 msgr2=0x7efd8c102230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:05.880 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 shutdown_connections 2024-05-20T20:19:05.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:05.873+0000 7efd90c00700 1 -- 172.21.15.28:0/1836987336 wait complete. 2024-05-20T20:19:05.898 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:06.835 INFO:tasks.quiescer:done joining Quiescers 2024-05-20T20:19:06.836 DEBUG:teuthology.run_tasks:Unwinding manager background_exec 2024-05-20T20:19:06.846 INFO:teuthology.task.background_exec:Stopping background command on ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:19:06.873 DEBUG:teuthology.run_tasks:Unwinding manager full_sequential_finally 2024-05-20T20:19:06.883 INFO:teuthology.task.full_sequential_finally:In full_sequential_finally, running task exec... 2024-05-20T20:19:06.883 INFO:teuthology.task.exec:Executing custom commands... 2024-05-20T20:19:06.883 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:19:06.884 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > END_TIME' 2024-05-20T20:19:07.014 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi' 2024-05-20T20:19:07.107 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 -- 172.21.15.28:0/280197005 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a4104a60 msgr2=0x7f39a4104ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 --2- 172.21.15.28:0/280197005 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a4104a60 0x7f39a4104ee0 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7f3990009930 tx=0x7f399002a420 comp rx=0 tx=0).stop 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 -- 172.21.15.28:0/280197005 shutdown_connections 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 --2- 172.21.15.28:0/280197005 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39a4105420 0x7f39a4111930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 --2- 172.21.15.28:0/280197005 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a4104a60 0x7f39a4104ee0 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 --2- 172.21.15.28:0/280197005 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39a410aa70 0x7f39a410ae50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 -- 172.21.15.28:0/280197005 >> 172.21.15.28:0/280197005 conn(0x7f39a4100570 msgr2=0x7f39a4102990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 -- 172.21.15.28:0/280197005 shutdown_connections 2024-05-20T20:19:07.291 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.290+0000 7f39aa800640 1 -- 172.21.15.28:0/280197005 wait complete. 2024-05-20T20:19:07.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 Processor -- start 2024-05-20T20:19:07.293 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 -- start start 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39a4104a60 0x7f39a419b320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39a4105420 0x7f39a419b860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a410aa70 0x7f39a419fbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f39a4114d80 con 0x7f39a410aa70 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f39a4114c00 con 0x7f39a4105420 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.291+0000 7f39aa800640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f39a4114f00 con 0x7f39a4104a60 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a9e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a410aa70 0x7f39a419fbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a8a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39a4105420 0x7f39a419b860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a9e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a410aa70 0x7f39a419fbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52486/0 (socket says 172.21.15.28:52486) 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a9e00640 1 -- 172.21.15.28:0/1471144661 learned_addr learned my addr 172.21.15.28:0/1471144661 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:07.294 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a9400640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39a4104a60 0x7f39a419b320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a8a00640 1 -- 172.21.15.28:0/1471144661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39a4104a60 msgr2=0x7f39a419b320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a8a00640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39a4104a60 0x7f39a419b320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a8a00640 1 -- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a410aa70 msgr2=0x7f39a419fbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a8a00640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a410aa70 0x7f39a419fbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a8a00640 1 -- 172.21.15.28:0/1471144661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f39a41a0310 con 0x7f39a4105420 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a9400640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39a4104a60 0x7f39a419b320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.292+0000 7f39a9e00640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a410aa70 0x7f39a419fbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.293+0000 7f39a8a00640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39a4105420 0x7f39a419b860 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f399002a940 tx=0x7f3990003c10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.293+0000 7f39a2000640 1 -- 172.21.15.28:0/1471144661 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f399003f070 con 0x7f39a4105420 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.293+0000 7f39a2000640 1 -- 172.21.15.28:0/1471144661 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3990031b60 con 0x7f39a4105420 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.293+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f39a41a05a0 con 0x7f39a4105420 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.293+0000 7f39a2000640 1 -- 172.21.15.28:0/1471144661 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3990035030 con 0x7f39a4105420 2024-05-20T20:19:07.295 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.293+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f39a41a7e90 con 0x7f39a4105420 2024-05-20T20:19:07.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.294+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f39a41061a0 con 0x7f39a4105420 2024-05-20T20:19:07.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.297+0000 7f39a2000640 1 -- 172.21.15.28:0/1471144661 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3990042070 con 0x7f39a4105420 2024-05-20T20:19:07.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.298+0000 7f39a2000640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f398c0b2950 0x7f398c0b4e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.298+0000 7f39a2000640 1 -- 172.21.15.28:0/1471144661 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f39901030b0 con 0x7f39a4105420 2024-05-20T20:19:07.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.299+0000 7f39a9400640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f398c0b2950 0x7f398c0b4e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.299+0000 7f39a9400640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f398c0b2950 0x7f398c0b4e10 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f3998009f40 tx=0x7f3998005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:07.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.301+0000 7f39a2000640 1 -- 172.21.15.28:0/1471144661 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f39900ca3d0 con 0x7f39a4105420 2024-05-20T20:19:07.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.544+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) -- 0x7f39a410d000 con 0x7f398c0b2950 2024-05-20T20:19:07.558 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.558+0000 7f39a2000640 1 -- 172.21.15.28:0/1471144661 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+350 (secure 0 0 0) 0x7f39a410d000 con 0x7f398c0b2950 2024-05-20T20:19:07.559 INFO:teuthology.orchestra.run.smithi028.stdout:{"fs": "cephfs", "subvol": null, "group": null, "path": "/", "rel_path": "/", "schedule": "1m", "retention": {"m": 6, "h": 3}, "start": "2024-05-20T00:00:00", "created": "2024-05-20T20:05:56", "first": "2024-05-20T20:06:01", "last": "2024-05-20T20:19:00", "last_pruned": "2024-05-20T20:19:00", "created_count": 14, "pruned_count": 8, "active": true} 2024-05-20T20:19:07.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f398c0b2950 msgr2=0x7f398c0b4e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f398c0b2950 0x7f398c0b4e10 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f3998009f40 tx=0x7f3998005f90 comp rx=0 tx=0).stop 2024-05-20T20:19:07.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39a4105420 msgr2=0x7f39a419b860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39a4105420 0x7f39a419b860 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f399002a940 tx=0x7f3990003c10 comp rx=0 tx=0).stop 2024-05-20T20:19:07.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 shutdown_connections 2024-05-20T20:19:07.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f39a4105420 0x7f39a419b860 secure :-1 s=CLOSED pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f399002a940 tx=0x7f3990003c10 comp rx=0 tx=0).stop 2024-05-20T20:19:07.562 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f398c0b2950 0x7f398c0b4e10 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f39a410aa70 0x7f39a419fbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 --2- 172.21.15.28:0/1471144661 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f39a4104a60 0x7f39a419b320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 >> 172.21.15.28:0/1471144661 conn(0x7f39a4100570 msgr2=0x7f39a4100e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:07.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.561+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 shutdown_connections 2024-05-20T20:19:07.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.562+0000 7f39aa800640 1 -- 172.21.15.28:0/1471144661 wait complete. 2024-05-20T20:19:07.582 DEBUG:teuthology.orchestra.run.smithi028:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2024-05-20T20:19:07.618 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:07 smithi028 ceph-mon[31178]: pgmap v421: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 61 KiB/s wr, 38 op/s 2024-05-20T20:19:07.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:07 smithi154 ceph-mon[36314]: pgmap v421: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 61 KiB/s wr, 38 op/s 2024-05-20T20:19:07.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:07 smithi033 ceph-mon[36179]: pgmap v421: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 61 KiB/s wr, 38 op/s 2024-05-20T20:19:07.779 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.779+0000 7f3c8e000640 1 -- 172.21.15.28:0/2613087511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88108870 msgr2=0x7f3c88108c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.779+0000 7f3c8e000640 1 --2- 172.21.15.28:0/2613087511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88108870 0x7f3c88108c50 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f3c7c009890 tx=0x7f3c7c02a380 comp rx=0 tx=0).stop 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.780+0000 7f3c8e000640 1 -- 172.21.15.28:0/2613087511 shutdown_connections 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.780+0000 7f3c8e000640 1 --2- 172.21.15.28:0/2613087511 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c88102860 0x7f3c88102ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.780+0000 7f3c8e000640 1 --2- 172.21.15.28:0/2613087511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c88103220 0x7f3c8810f730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.780+0000 7f3c8e000640 1 --2- 172.21.15.28:0/2613087511 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88108870 0x7f3c88108c50 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.780+0000 7f3c8e000640 1 -- 172.21.15.28:0/2613087511 >> 172.21.15.28:0/2613087511 conn(0x7f3c880fe390 msgr2=0x7f3c881007b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.780+0000 7f3c8e000640 1 -- 172.21.15.28:0/2613087511 shutdown_connections 2024-05-20T20:19:07.780 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.780+0000 7f3c8e000640 1 -- 172.21.15.28:0/2613087511 wait complete. 2024-05-20T20:19:07.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 Processor -- start 2024-05-20T20:19:07.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 -- start start 2024-05-20T20:19:07.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c88102860 0x7f3c8819b310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88103220 0x7f3c8819b850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c88108870 0x7f3c8819fbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f3c88112af0 con 0x7f3c88103220 2024-05-20T20:19:07.782 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f3c88112970 con 0x7f3c88102860 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.781+0000 7f3c8e000640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f3c88112c70 con 0x7f3c88108870 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88103220 0x7f3c8819b850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c8cc00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c88102860 0x7f3c8819b310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88103220 0x7f3c8819b850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52516/0 (socket says 172.21.15.28:52516) 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 -- 172.21.15.28:0/347380803 learned_addr learned my addr 172.21.15.28:0/347380803 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c8d600640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c88108870 0x7f3c8819fbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 -- 172.21.15.28:0/347380803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c88108870 msgr2=0x7f3c8819fbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c88108870 0x7f3c8819fbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 -- 172.21.15.28:0/347380803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c88102860 msgr2=0x7f3c8819b310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c88102860 0x7f3c8819b310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c87e00640 1 -- 172.21.15.28:0/347380803 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3c881a0300 con 0x7f3c88103220 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.782+0000 7f3c8cc00640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c88102860 0x7f3c8819b310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:07.783 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.783+0000 7f3c87e00640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88103220 0x7f3c8819b850 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f3c7400e9c0 tx=0x7f3c7400ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:07.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.783+0000 7f3c85600640 1 -- 172.21.15.28:0/347380803 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c7400cde0 con 0x7f3c88103220 2024-05-20T20:19:07.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.783+0000 7f3c85600640 1 -- 172.21.15.28:0/347380803 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f3c740035f0 con 0x7f3c88103220 2024-05-20T20:19:07.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.783+0000 7f3c85600640 1 -- 172.21.15.28:0/347380803 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c74010690 con 0x7f3c88103220 2024-05-20T20:19:07.784 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.783+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3c881a05f0 con 0x7f3c88103220 2024-05-20T20:19:07.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.783+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f3c881a7e80 con 0x7f3c88103220 2024-05-20T20:19:07.785 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.784+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3c88051030 con 0x7f3c88103220 2024-05-20T20:19:07.787 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.788+0000 7f3c85600640 1 -- 172.21.15.28:0/347380803 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f3c74004000 con 0x7f3c88103220 2024-05-20T20:19:07.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.789+0000 7f3c85600640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c700b29d0 0x7f3c700b4e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:07.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.789+0000 7f3c85600640 1 -- 172.21.15.28:0/347380803 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(95..95 src has 1..95) ==== 8542+0+0 (secure 0 0 0) 0x7f3c740e28f0 con 0x7f3c88103220 2024-05-20T20:19:07.789 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.790+0000 7f3c8cc00640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c700b29d0 0x7f3c700b4e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:07.790 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.791+0000 7f3c8cc00640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c700b29d0 0x7f3c700b4e90 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f3c7c035040 tx=0x7f3c7c030340 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:07.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:07.795+0000 7f3c85600640 1 -- 172.21.15.28:0/347380803 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f3c74009510 con 0x7f3c88103220 2024-05-20T20:19:08.041 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.042+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 --> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) -- 0x7f3c880ff9b0 con 0x7f3c700b29d0 2024-05-20T20:19:08.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.055+0000 7f3c85600640 1 -- 172.21.15.28:0/347380803 <== mgr.25120 v2:172.21.15.28:6800/2022328614 1 ==== mgr_command_reply(tid 0: 0 ) ==== 8+0+10 (secure 0 0 0) 0x7f3c880ff9b0 con 0x7f3c700b29d0 2024-05-20T20:19:08.055 INFO:teuthology.orchestra.run.smithi028.stdout:/ 1m 6m3h 2024-05-20T20:19:08.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.058+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c700b29d0 msgr2=0x7f3c700b4e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:08.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.058+0000 7f3c8e000640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c700b29d0 0x7f3c700b4e90 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f3c7c035040 tx=0x7f3c7c030340 comp rx=0 tx=0).stop 2024-05-20T20:19:08.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.058+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88103220 msgr2=0x7f3c8819b850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:08.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.058+0000 7f3c8e000640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88103220 0x7f3c8819b850 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f3c7400e9c0 tx=0x7f3c7400ee80 comp rx=0 tx=0).stop 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 shutdown_connections 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f3c700b29d0 0x7f3c700b4e90 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3c88108870 0x7f3c8819fbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f3c88103220 0x7f3c8819b850 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 --2- 172.21.15.28:0/347380803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f3c88102860 0x7f3c8819b310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 >> 172.21.15.28:0/347380803 conn(0x7f3c880fe390 msgr2=0x7f3c8810dfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 shutdown_connections 2024-05-20T20:19:08.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:08.059+0000 7f3c8e000640 1 -- 172.21.15.28:0/347380803 wait complete. 2024-05-20T20:19:08.078 DEBUG:teuthology.run_tasks:Unwinding manager fwd_scrub 2024-05-20T20:19:08.088 INFO:tasks.fwd_scrub:joining ForwardScrubbers 2024-05-20T20:19:09.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.048+0000 7f5f47600700 1 -- 172.21.15.28:0/739119442 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f40116690 msgr2=0x7f5f40118a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.049 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.048+0000 7f5f47600700 1 --2- 172.21.15.28:0/739119442 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f40116690 0x7f5f40118a80 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f5f30009990 tx=0x7f5f30009ca0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.050 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.051+0000 7f5f47600700 1 -- 172.21.15.28:0/739119442 shutdown_connections 2024-05-20T20:19:09.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.051+0000 7f5f47600700 1 --2- 172.21.15.28:0/739119442 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f40116690 0x7f5f40118a80 unknown :-1 s=CLOSED pgs=521 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.051+0000 7f5f47600700 1 --2- 172.21.15.28:0/739119442 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5f4010a540 0x7f5f4010a920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.051+0000 7f5f47600700 1 --2- 172.21.15.28:0/739119442 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5f4010ae60 0x7f5f40116150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.051+0000 7f5f47600700 1 -- 172.21.15.28:0/739119442 >> 172.21.15.28:0/739119442 conn(0x7f5f4006e6b0 msgr2=0x7f5f4006eac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:09.051 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.051+0000 7f5f47600700 1 -- 172.21.15.28:0/739119442 shutdown_connections 2024-05-20T20:19:09.055 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.051+0000 7f5f47600700 1 -- 172.21.15.28:0/739119442 wait complete. 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.053+0000 7f5f47600700 1 Processor -- start 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.054+0000 7f5f47600700 1 -- start start 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.055+0000 7f5f47600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f4010a540 0x7f5f401ae6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.055+0000 7f5f47600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5f4010ae60 0x7f5f401aebf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.055+0000 7f5f47600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5f40116690 0x7f5f401a88c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.055+0000 7f5f47600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5f401225a0 con 0x7f5f4010ae60 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.055+0000 7f5f44c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f4010a540 0x7f5f401ae6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.056+0000 7f5f44c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f4010a540 0x7f5f401ae6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:34664/0 (socket says 172.21.15.28:34664) 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.056+0000 7f5f44c00700 1 -- 172.21.15.28:0/1769066803 learned_addr learned my addr 172.21.15.28:0/1769066803 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.056+0000 7f5f45600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5f40116690 0x7f5f401a88c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.056 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.056+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5f40122420 con 0x7f5f40116690 2024-05-20T20:19:09.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.056+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5f40122720 con 0x7f5f4010a540 2024-05-20T20:19:09.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.057+0000 7f5f45600700 1 -- 172.21.15.28:0/1769066803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f4010a540 msgr2=0x7f5f401ae6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.057+0000 7f5f45600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f4010a540 0x7f5f401ae6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.057+0000 7f5f45600700 1 -- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5f4010ae60 msgr2=0x7f5f401aebf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.057+0000 7f5f45600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5f4010ae60 0x7f5f401aebf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.057+0000 7f5f45600700 1 -- 172.21.15.28:0/1769066803 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5f401a9020 con 0x7f5f40116690 2024-05-20T20:19:09.057 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.057+0000 7f5f44c00700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f4010a540 0x7f5f401ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:09.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.057+0000 7f5f45600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5f40116690 0x7f5f401a88c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f5f30009990 tx=0x7f5f300047b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:09.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.058+0000 7f5f3d600700 1 -- 172.21.15.28:0/1769066803 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f30026070 con 0x7f5f40116690 2024-05-20T20:19:09.058 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.059+0000 7f5f3d600700 1 -- 172.21.15.28:0/1769066803 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5f30021410 con 0x7f5f40116690 2024-05-20T20:19:09.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.059+0000 7f5f3d600700 1 -- 172.21.15.28:0/1769066803 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f30017650 con 0x7f5f40116690 2024-05-20T20:19:09.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.059+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5f401a92b0 con 0x7f5f40116690 2024-05-20T20:19:09.059 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.059+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5f401b5580 con 0x7f5f40116690 2024-05-20T20:19:09.068 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.066+0000 7f5f3d600700 1 -- 172.21.15.28:0/1769066803 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f5f30029070 con 0x7f5f40116690 2024-05-20T20:19:09.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.070+0000 7f5f52008b80 1 Processor -- start 2024-05-20T20:19:09.070 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.070+0000 7f5f3d600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5f240b2b80 0x7f5f240b5040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.071+0000 7f5f3d600700 1 -- 172.21.15.28:0/1769066803 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(96..96 src has 1..96) ==== 8586+0+0 (secure 0 0 0) 0x7f5f300e3230 con 0x7f5f40116690 2024-05-20T20:19:09.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.071+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5f240b2b80 0x7f5f240b5040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.071 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.072+0000 7f5f52008b80 1 -- start start 2024-05-20T20:19:09.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.072+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db325290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.072+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 0x5575db31f260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.072 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.072+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 0x5575db31fce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.072+0000 7f5f52008b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5575db27de40 con 0x5575db31f880 2024-05-20T20:19:09.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.072+0000 7f5f52008b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5575db26d9e0 con 0x5575db3257d0 2024-05-20T20:19:09.073 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.072+0000 7f5f52008b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5575db271a20 con 0x5575db324eb0 2024-05-20T20:19:09.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db325290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db325290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:34668/0 (socket says 172.21.15.28:34668) 2024-05-20T20:19:09.074 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 -- 172.21.15.28:0/3630487663 learned_addr learned my addr 172.21.15.28:0/3630487663 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 -- 172.21.15.28:0/3630487663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 msgr2=0x5575db31f260 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 --2- 172.21.15.28:0/3630487663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 0x5575db31f260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 -- 172.21.15.28:0/3630487663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 msgr2=0x5575db31fce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 --2- 172.21.15.28:0/3630487663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 0x5575db31fce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.073+0000 7f5f44c00700 1 -- 172.21.15.28:0/3630487663 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575db236420 con 0x5575db324eb0 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.074+0000 7f5f44c00700 1 --2- 172.21.15.28:0/3630487663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db325290 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f5f38011680 tx=0x7f5f38011990 comp rx=0 tx=0).ready entity=mon.1 client_cookie=735b30da825dad3b server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.074+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5f240b2b80 0x7f5f240b5040 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f5f40074c50 tx=0x7f5f28007570 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.074+0000 7f5f36000700 1 -- 172.21.15.28:0/3630487663 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f38004030 con 0x5575db324eb0 2024-05-20T20:19:09.075 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.074+0000 7f5f36000700 1 -- 172.21.15.28:0/3630487663 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5f38003680 con 0x5575db324eb0 2024-05-20T20:19:09.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.074+0000 7f5f36000700 1 -- 172.21.15.28:0/3630487663 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f3800f7b0 con 0x5575db324eb0 2024-05-20T20:19:09.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.075+0000 7f5f52008b80 1 -- 172.21.15.28:0/3630487663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 msgr2=0x5575db325290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.075+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3630487663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db325290 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7f5f38011680 tx=0x7f5f38011990 comp rx=0 tx=0).stop 2024-05-20T20:19:09.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.076+0000 7f5f52008b80 1 -- 172.21.15.28:0/3630487663 shutdown_connections 2024-05-20T20:19:09.076 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.076+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3630487663 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 0x5575db31fce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.076+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3630487663 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db325290 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.076+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3630487663 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 0x5575db31f260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.076+0000 7f5f52008b80 1 -- 172.21.15.28:0/3630487663 >> 172.21.15.28:0/3630487663 conn(0x5575db24dbe0 msgr2=0x5575db31a010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:09.077 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.077+0000 7f5f52008b80 1 -- 172.21.15.28:0/3630487663 shutdown_connections 2024-05-20T20:19:09.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.077+0000 7f5f52008b80 1 -- 172.21.15.28:0/3630487663 wait complete. 2024-05-20T20:19:09.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.077+0000 7f5f52008b80 1 -- start start 2024-05-20T20:19:09.078 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.078+0000 7f5f52008b80 1 Processor -- start 2024-05-20T20:19:09.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.078+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db31f880 0x5575db32b540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.078+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db324eb0 0x5575db328950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.079 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db3257d0 0x5575db328e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f52008b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5575db173fa0 con 0x5575db3257d0 2024-05-20T20:19:09.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f52008b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5575db20e760 con 0x5575db324eb0 2024-05-20T20:19:09.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f52008b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5575db32c1e0 con 0x5575db31f880 2024-05-20T20:19:09.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f52008b80 10 client.0 _ll_register_callbacks cb 0x5575db217c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:09.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f3fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db324eb0 0x5575db328950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f45600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db3257d0 0x5575db328e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.080 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f3fe00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db324eb0 0x5575db328950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:35326/0 (socket says 172.21.15.28:35326) 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.079+0000 7f5f3fe00700 1 -- 172.21.15.28:0/652377455 learned_addr learned my addr 172.21.15.28:0/652377455 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.080+0000 7f5f3fe00700 1 -- 172.21.15.28:0/652377455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db31f880 msgr2=0x5575db32b540 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.080+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db31f880 0x5575db32b540 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.080+0000 7f5f3fe00700 1 -- 172.21.15.28:0/652377455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db3257d0 msgr2=0x5575db328e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.080+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db3257d0 0x5575db328e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.080+0000 7f5f3fe00700 1 -- 172.21.15.28:0/652377455 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575db1bd8d0 con 0x5575db324eb0 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.080+0000 7f5f45600700 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db3257d0 0x5575db328e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.080+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db324eb0 0x5575db328950 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f5f28016aa0 tx=0x7f5f28004ad0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=62cabdc34db750f2 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:09.081 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.081+0000 7f5f2ea00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:19:09.082 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.081+0000 7f5f2ea00700 1 -- 172.21.15.28:0/652377455 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f28024060 con 0x5575db324eb0 2024-05-20T20:19:09.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.081+0000 7f5f2ea00700 1 -- 172.21.15.28:0/652377455 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(96..96 src has 1..96) ==== 8586+0+0 (secure 0 0 0) 0x7f5f280137c0 con 0x5575db324eb0 2024-05-20T20:19:09.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.081+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5f38003680 con 0x5575db324eb0 2024-05-20T20:19:09.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.082+0000 7f5f2ea00700 1 -- 172.21.15.28:0/652377455 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5f280139c0 con 0x5575db324eb0 2024-05-20T20:19:09.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.082+0000 7f5f52008b80 10 client.28130 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:09.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.082+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5575db23e060 con 0x5575db324eb0 2024-05-20T20:19:09.083 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.083+0000 7f5f2ea00700 1 -- 172.21.15.28:0/652377455 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5f38003680 con 0x5575db324eb0 2024-05-20T20:19:09.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.083+0000 7f5f2ea00700 1 -- 172.21.15.28:0/652377455 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5f28014a70 con 0x5575db324eb0 2024-05-20T20:19:09.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.084+0000 7f5f52008b80 10 client.28130 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:09.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.084+0000 7f5f52008b80 10 client.28130 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:19:09.084 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.084+0000 7f5f52008b80 20 client.28130 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:09.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.084+0000 7f5f52008b80 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db338ef0 0x5575db33b2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.085+0000 7f5f52008b80 4 client.28130 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:19:09.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.085+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5575db27fd90 con 0x5575db338ef0 2024-05-20T20:19:09.085 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.085+0000 7f5f44c00700 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db338ef0 0x5575db33b2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.086+0000 7f5f44c00700 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db338ef0 0x5575db33b2e0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=fff0b79a298cfae server_cookie=e967440d4fb0144e in_seq=0 out_seq=0 2024-05-20T20:19:09.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.086+0000 7f5f2ea00700 10 client.28130 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:19:09.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.088+0000 7f5f2ea00700 1 -- 172.21.15.28:0/652377455 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5575db27fd90 con 0x5575db338ef0 2024-05-20T20:19:09.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.088+0000 7f5f2ea00700 10 client.28130 handle_command_reply: tid=0 2024-05-20T20:19:09.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.088+0000 7f5f52008b80 1 client.28130 shutdown 2024-05-20T20:19:09.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db324eb0 msgr2=0x5575db328950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db324eb0 0x5575db328950 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f5f28016aa0 tx=0x7f5f28004ad0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 shutdown_connections 2024-05-20T20:19:09.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db338ef0 0x5575db33b2e0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db3257d0 0x5575db328e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db31f880 0x5575db32b540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f2ea00700 0 client.28130 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 --2- 172.21.15.28:0/652377455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db324eb0 0x5575db328950 unknown :-1 s=CLOSED pgs=626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.092+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 >> 172.21.15.28:0/652377455 conn(0x5575db24dbe0 msgr2=0x5575db31e250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.093+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 shutdown_connections 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.093+0000 7f5f52008b80 1 -- 172.21.15.28:0/652377455 wait complete. 2024-05-20T20:19:09.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.093+0000 7f5f52008b80 20 client.28130 trim_cache size 0 max 16384 2024-05-20T20:19:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.127+0000 7f5f52008b80 1 Processor -- start 2024-05-20T20:19:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.127+0000 7f5f52008b80 1 -- start start 2024-05-20T20:19:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 0x5575db3352e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db3199e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.127 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 0x5575db319f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f52008b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5f401bcc30 con 0x5575db31f880 2024-05-20T20:19:09.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f52008b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5f401be770 con 0x5575db3257d0 2024-05-20T20:19:09.128 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f52008b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5f401bd300 con 0x5575db324eb0 2024-05-20T20:19:09.130 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f3fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db3199e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f3fe00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db3199e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:34682/0 (socket says 172.21.15.28:34682) 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f3fe00700 1 -- 172.21.15.28:0/3841447822 learned_addr learned my addr 172.21.15.28:0/3841447822 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.128+0000 7f5f45600700 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 0x5575db319f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.129+0000 7f5f3fe00700 1 -- 172.21.15.28:0/3841447822 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 msgr2=0x5575db319f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.129+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 0x5575db319f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.129+0000 7f5f3fe00700 1 -- 172.21.15.28:0/3841447822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 msgr2=0x5575db3352e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.130+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 0x5575db3352e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.130+0000 7f5f3fe00700 1 -- 172.21.15.28:0/3841447822 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5575db33b940 con 0x5575db324eb0 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.130+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db3199e0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f5f28014ec0 tx=0x7f5f28007c60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c390b95d96b3c69e server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.130+0000 7f5f35600700 1 -- 172.21.15.28:0/3841447822 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f28034020 con 0x5575db324eb0 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.131+0000 7f5f35600700 1 -- 172.21.15.28:0/3841447822 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5f28013690 con 0x5575db324eb0 2024-05-20T20:19:09.131 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.131+0000 7f5f35600700 1 -- 172.21.15.28:0/3841447822 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f2801f910 con 0x5575db324eb0 2024-05-20T20:19:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.132+0000 7f5f52008b80 1 -- 172.21.15.28:0/3841447822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 msgr2=0x5575db3199e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.132 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.132+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db3199e0 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f5f28014ec0 tx=0x7f5f28007c60 comp rx=0 tx=0).stop 2024-05-20T20:19:09.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.132+0000 7f5f52008b80 1 -- 172.21.15.28:0/3841447822 shutdown_connections 2024-05-20T20:19:09.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.132+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db3257d0 0x5575db319f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.132+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db31f880 0x5575db3352e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.133+0000 7f5f52008b80 1 --2- 172.21.15.28:0/3841447822 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db324eb0 0x5575db3199e0 secure :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7f5f28014ec0 tx=0x7f5f28007c60 comp rx=0 tx=0).stop 2024-05-20T20:19:09.133 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.133+0000 7f5f52008b80 1 -- 172.21.15.28:0/3841447822 >> 172.21.15.28:0/3841447822 conn(0x5575db24dbe0 msgr2=0x5575db3229c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:09.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.135+0000 7f5f52008b80 1 -- 172.21.15.28:0/3841447822 shutdown_connections 2024-05-20T20:19:09.134 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.135+0000 7f5f52008b80 1 -- 172.21.15.28:0/3841447822 wait complete. 2024-05-20T20:19:09.135 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.136+0000 7f5f52008b80 1 -- start start 2024-05-20T20:19:09.136 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.136+0000 7f5f52008b80 1 Processor -- start 2024-05-20T20:19:09.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.137+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db31f880 0x5575db331d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.137+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db3257d0 0x5575db340530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.137+0000 7f5f52008b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db340bd0 0x5575db3a2a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.137+0000 7f5f52008b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f5f401be0a0 con 0x5575db340bd0 2024-05-20T20:19:09.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.137+0000 7f5f52008b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f5f401bd9d0 con 0x5575db31f880 2024-05-20T20:19:09.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.137+0000 7f5f52008b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f5f401babf0 con 0x5575db3257d0 2024-05-20T20:19:09.137 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.137+0000 7f5f52008b80 10 client.0 _ll_register_callbacks cb 0x5575db217c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f44c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db31f880 0x5575db331d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f44c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db31f880 0x5575db331d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:35340/0 (socket says 172.21.15.28:35340) 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f44c00700 1 -- 172.21.15.28:0/414198213 learned_addr learned my addr 172.21.15.28:0/414198213 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db3257d0 0x5575db340530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f44c00700 1 -- 172.21.15.28:0/414198213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db3257d0 msgr2=0x5575db340530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f44c00700 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db3257d0 0x5575db340530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f44c00700 1 -- 172.21.15.28:0/414198213 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db340bd0 msgr2=0x5575db3a2a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.138+0000 7f5f44c00700 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db340bd0 0x5575db3a2a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.139+0000 7f5f44c00700 1 -- 172.21.15.28:0/414198213 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5575db341350 con 0x5575db31f880 2024-05-20T20:19:09.139 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.139+0000 7f5f44c00700 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db31f880 0x5575db331d80 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f5f38009850 tx=0x7f5f38009490 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4f63811bddd5fe20 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:09.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.139+0000 7f5f2ea00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:19:09.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.140+0000 7f5f2ea00700 1 -- 172.21.15.28:0/414198213 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5f38003680 con 0x5575db31f880 2024-05-20T20:19:09.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.140+0000 7f5f2ea00700 1 -- 172.21.15.28:0/414198213 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(96..96 src has 1..96) ==== 8586+0+0 (secure 0 0 0) 0x7f5f380050c0 con 0x5575db31f880 2024-05-20T20:19:09.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.140+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f5f28013690 con 0x5575db31f880 2024-05-20T20:19:09.140 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.140+0000 7f5f2ea00700 1 -- 172.21.15.28:0/414198213 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f5f380052e0 con 0x5575db31f880 2024-05-20T20:19:09.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.140+0000 7f5f52008b80 10 client.28136 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:09.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.140+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5575db3a4060 con 0x5575db31f880 2024-05-20T20:19:09.141 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.141+0000 7f5f2ea00700 1 -- 172.21.15.28:0/414198213 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f5f28013690 con 0x5575db31f880 2024-05-20T20:19:09.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.141+0000 7f5f2ea00700 1 -- 172.21.15.28:0/414198213 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f5f3800f480 con 0x5575db31f880 2024-05-20T20:19:09.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.142+0000 7f5f52008b80 10 client.28136 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:09.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.142+0000 7f5f52008b80 10 client.28136 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.e 2024-05-20T20:19:09.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.142+0000 7f5f52008b80 20 client.28136 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:09.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.142+0000 7f5f52008b80 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db3a4610 0x5575db3a6a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:09.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.142+0000 7f5f52008b80 4 client.28136 mds_command: new command op to 24482 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:19:09.142 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.142+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 --> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5575db279570 con 0x5575db3a4610 2024-05-20T20:19:09.143 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.143+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db3a4610 0x5575db3a6a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.143+0000 7f5f2ea00700 10 client.28136 ms_handle_connect on v2:172.21.15.33:6834/4076526106 2024-05-20T20:19:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.143+0000 7f5f3fe00700 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db3a4610 0x5575db3a6a30 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=500fd78bbfe24b31 server_cookie=43db018d2f1ae5f5 in_seq=0 out_seq=0 2024-05-20T20:19:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.145+0000 7f5f2ea00700 1 -- 172.21.15.28:0/414198213 <== mds.3 v2:172.21.15.33:6834/4076526106 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5575db279570 con 0x5575db3a4610 2024-05-20T20:19:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.145+0000 7f5f2ea00700 10 client.28136 handle_command_reply: tid=0 2024-05-20T20:19:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.145+0000 7f5f52008b80 1 client.28136 shutdown 2024-05-20T20:19:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.145+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db31f880 msgr2=0x5575db331d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.145 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.145+0000 7f5f52008b80 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db31f880 0x5575db331d80 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f5f38009850 tx=0x7f5f38009490 comp rx=0 tx=0).stop 2024-05-20T20:19:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.145+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 shutdown_connections 2024-05-20T20:19:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.145+0000 7f5f52008b80 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:6834/4076526106,v1:172.21.15.33:6835/4076526106] conn(0x5575db3a4610 0x5575db3a6a30 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.146+0000 7f5f52008b80 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5575db340bd0 0x5575db3a2a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.146+0000 7f5f52008b80 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5575db31f880 0x5575db331d80 unknown :-1 s=CLOSED pgs=627 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.146+0000 7f5f2ea00700 0 client.28136 ms_handle_reset on v2:172.21.15.33:6834/4076526106 2024-05-20T20:19:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.146+0000 7f5f52008b80 1 --2- 172.21.15.28:0/414198213 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5575db3257d0 0x5575db340530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.146 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.146+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 >> 172.21.15.28:0/414198213 conn(0x5575db24dbe0 msgr2=0x5575db3199e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:09.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.147+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 shutdown_connections 2024-05-20T20:19:09.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.148+0000 7f5f52008b80 1 -- 172.21.15.28:0/414198213 wait complete. 2024-05-20T20:19:09.147 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.148+0000 7f5f52008b80 20 client.28136 trim_cache size 0 max 16384 2024-05-20T20:19:09.148 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:19:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5f240b2b80 msgr2=0x7f5f240b5040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5f240b2b80 0x7f5f240b5040 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f5f40074c50 tx=0x7f5f28007570 comp rx=0 tx=0).stop 2024-05-20T20:19:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5f40116690 msgr2=0x7f5f401a88c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5f40116690 0x7f5f401a88c0 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f5f30009990 tx=0x7f5f300047b0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.149 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 shutdown_connections 2024-05-20T20:19:09.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f5f240b2b80 0x7f5f240b5040 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f5f40116690 0x7f5f401a88c0 unknown :-1 s=CLOSED pgs=625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5f4010a540 0x7f5f401ae6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 --2- 172.21.15.28:0/1769066803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f5f4010ae60 0x7f5f401aebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:09.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.149+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 >> 172.21.15.28:0/1769066803 conn(0x7f5f4006e6b0 msgr2=0x7f5f40073500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:09.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.150+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 shutdown_connections 2024-05-20T20:19:09.150 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:09.150+0000 7f5f47600700 1 -- 172.21.15.28:0/1769066803 wait complete. 2024-05-20T20:19:09.617 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:09 smithi028 ceph-mon[31178]: from='client.28115 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:19:09.617 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:09 smithi028 ceph-mon[31178]: from='client.17511 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:19:09.617 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:09 smithi028 ceph-mon[31178]: pgmap v422: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 61 KiB/s wr, 41 op/s 2024-05-20T20:19:09.617 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:09 smithi028 ceph-mon[31178]: osdmap e96: 12 total, 12 up, 12 in 2024-05-20T20:19:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:09 smithi154 ceph-mon[36314]: from='client.28115 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:19:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:09 smithi154 ceph-mon[36314]: from='client.17511 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:19:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:09 smithi154 ceph-mon[36314]: pgmap v422: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 61 KiB/s wr, 41 op/s 2024-05-20T20:19:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:09 smithi154 ceph-mon[36314]: osdmap e96: 12 total, 12 up, 12 in 2024-05-20T20:19:09.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:09 smithi033 ceph-mon[36179]: from='client.28115 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:19:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:09 smithi033 ceph-mon[36179]: from='client.17511 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T20:19:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:09 smithi033 ceph-mon[36179]: pgmap v422: 97 pgs: 97 active+clean; 63 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 61 KiB/s wr, 41 op/s 2024-05-20T20:19:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:09 smithi033 ceph-mon[36179]: osdmap e96: 12 total, 12 up, 12 in 2024-05-20T20:19:10.077 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:2 damage ls 2024-05-20T20:19:10.647 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:11.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:11 smithi028 ceph-mon[31178]: pgmap v424: 97 pgs: 22 active+clean+snaptrim, 39 active+clean+snaptrim_wait, 36 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 57 KiB/s wr, 34 op/s 2024-05-20T20:19:11.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:11 smithi154 ceph-mon[36314]: pgmap v424: 97 pgs: 22 active+clean+snaptrim, 39 active+clean+snaptrim_wait, 36 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 57 KiB/s wr, 34 op/s 2024-05-20T20:19:11.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:11 smithi033 ceph-mon[36179]: pgmap v424: 97 pgs: 22 active+clean+snaptrim, 39 active+clean+snaptrim_wait, 36 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 57 KiB/s wr, 34 op/s 2024-05-20T20:19:12.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.912+0000 7f494da00700 1 -- 172.21.15.28:0/1447862607 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 msgr2=0x7f4948103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.912+0000 7f494da00700 1 --2- 172.21.15.28:0/1447862607 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f4948103bd0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f49380099c0 tx=0x7f4938009cd0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.912+0000 7f494da00700 1 -- 172.21.15.28:0/1447862607 shutdown_connections 2024-05-20T20:19:12.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.913+0000 7f494da00700 1 --2- 172.21.15.28:0/1447862607 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f49481088f0 0x7f4948110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.913 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.913+0000 7f494da00700 1 --2- 172.21.15.28:0/1447862607 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f4948103bd0 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.913+0000 7f494da00700 1 --2- 172.21.15.28:0/1447862607 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f49481041a0 0x7f49481081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.913+0000 7f494da00700 1 -- 172.21.15.28:0/1447862607 >> 172.21.15.28:0/1447862607 conn(0x7f494807a6a0 msgr2=0x7f494807aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:12.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.913+0000 7f494da00700 1 -- 172.21.15.28:0/1447862607 shutdown_connections 2024-05-20T20:19:12.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.913+0000 7f494da00700 1 -- 172.21.15.28:0/1447862607 wait complete. 2024-05-20T20:19:12.914 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.914+0000 7f494da00700 1 Processor -- start 2024-05-20T20:19:12.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f494da00700 1 -- start start 2024-05-20T20:19:12.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f494da00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f494819f7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f494da00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f49481041a0 0x7f494819fd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f494da00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f49481088f0 0x7f4948199980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f494da00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f49481136e0 con 0x7f49481037f0 2024-05-20T20:19:12.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f494da00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f4948113560 con 0x7f49481041a0 2024-05-20T20:19:12.915 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f494da00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f4948113860 con 0x7f49481088f0 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f4946a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f494819f7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f4946a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f494819f7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52584/0 (socket says 172.21.15.28:52584) 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f4946a00700 1 -- 172.21.15.28:0/2411074038 learned_addr learned my addr 172.21.15.28:0/2411074038 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.915+0000 7f493e000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f49481041a0 0x7f494819fd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f4947400700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f49481088f0 0x7f4948199980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f4946a00700 1 -- 172.21.15.28:0/2411074038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f49481088f0 msgr2=0x7f4948199980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f4946a00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f49481088f0 0x7f4948199980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f4946a00700 1 -- 172.21.15.28:0/2411074038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f49481041a0 msgr2=0x7f494819fd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.916 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f4946a00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f49481041a0 0x7f494819fd00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f4946a00700 1 -- 172.21.15.28:0/2411074038 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f494819a0e0 con 0x7f49481037f0 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f493e000700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f49481041a0 0x7f494819fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.916+0000 7f4946a00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f494819f7c0 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f4938006010 tx=0x7f4938004270 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.917+0000 7f493fe00700 1 -- 172.21.15.28:0/2411074038 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4938026070 con 0x7f49481037f0 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.917+0000 7f493fe00700 1 -- 172.21.15.28:0/2411074038 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4938004630 con 0x7f49481037f0 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.917+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f494819a370 con 0x7f49481037f0 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.917+0000 7f493fe00700 1 -- 172.21.15.28:0/2411074038 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f493800fb20 con 0x7f49481037f0 2024-05-20T20:19:12.917 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.917+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f49481a6690 con 0x7f49481037f0 2024-05-20T20:19:12.920 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.920+0000 7f493fe00700 1 -- 172.21.15.28:0/2411074038 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f4938029070 con 0x7f49481037f0 2024-05-20T20:19:12.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 Processor -- start 2024-05-20T20:19:12.921 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 -- start start 2024-05-20T20:19:12.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f148260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f148ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a16f0a6e40 con 0x55a16f148880 2024-05-20T20:19:12.922 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a16f0969e0 con 0x55a16f14deb0 2024-05-20T20:19:12.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f495838bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a16f09aa20 con 0x55a16f14e7d0 2024-05-20T20:19:12.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.922+0000 7f493fe00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f492c0b2a70 0x7f492c0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f148260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f493fe00700 1 -- 172.21.15.28:0/2411074038 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(96..96 src has 1..96) ==== 8586+0+0 (secure 0 0 0) 0x7f49380e2d40 con 0x7f49481037f0 2024-05-20T20:19:12.923 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f148260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:34716/0 (socket says 172.21.15.28:34716) 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4947400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 -- 172.21.15.28:0/2854475455 learned_addr learned my addr 172.21.15.28:0/2854475455 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f493e000700 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f148ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 -- 172.21.15.28:0/2854475455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 msgr2=0x55a16f14e290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4947400700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f492c0b2a70 0x7f492c0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 -- 172.21.15.28:0/2854475455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 msgr2=0x55a16f148ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f148ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f4946a00700 1 -- 172.21.15.28:0/2854475455 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a16f05f420 con 0x55a16f14e7d0 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.923+0000 7f493e000700 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f148ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.924+0000 7f4946a00700 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f148260 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f49380a9fa0 tx=0x7f49380a9960 comp rx=0 tx=0).ready entity=mon.1 client_cookie=65f6cf28455b0a65 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:12.924 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.924+0000 7f4937e00700 1 -- 172.21.15.28:0/2854475455 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49380a9b20 con 0x55a16f14e7d0 2024-05-20T20:19:12.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.924+0000 7f4937e00700 1 -- 172.21.15.28:0/2854475455 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f49380aa710 con 0x55a16f14e7d0 2024-05-20T20:19:12.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.924+0000 7f4937e00700 1 -- 172.21.15.28:0/2854475455 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4938056f00 con 0x55a16f14e7d0 2024-05-20T20:19:12.925 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 -- 172.21.15.28:0/2854475455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 msgr2=0x55a16f148260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f148260 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7f49380a9fa0 tx=0x7f49380a9960 comp rx=0 tx=0).stop 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f4947400700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f492c0b2a70 0x7f492c0b4f30 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f49400085d0 tx=0x7f494000b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 -- 172.21.15.28:0/2854475455 shutdown_connections 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f148ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 --2- 172.21.15.28:0/2854475455 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f148260 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 -- 172.21.15.28:0/2854475455 >> 172.21.15.28:0/2854475455 conn(0x55a16f076be0 msgr2=0x55a16f143010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 -- 172.21.15.28:0/2854475455 shutdown_connections 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.925+0000 7f495838bb80 1 -- 172.21.15.28:0/2854475455 wait complete. 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.926+0000 7f495838bb80 1 -- start start 2024-05-20T20:19:12.926 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.926+0000 7f495838bb80 1 Processor -- start 2024-05-20T20:19:12.927 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f495838bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f154490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f495838bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f151830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f495838bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f151d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f495838bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a16ef9cfa0 con 0x55a16f148880 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f495838bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a16f037760 con 0x55a16f14deb0 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f495838bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a16f155000 con 0x55a16f14e7d0 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f495838bb80 10 client.0 _ll_register_callbacks cb 0x55a16f040c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f4946a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f154490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f493e000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f151830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f4947400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f151d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f493e000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f151830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:35380/0 (socket says 172.21.15.28:35380) 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.927+0000 7f493e000700 1 -- 172.21.15.28:0/1205662875 learned_addr learned my addr 172.21.15.28:0/1205662875 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f493e000700 1 -- 172.21.15.28:0/1205662875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 msgr2=0x55a16f151d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f493e000700 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f151d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f493e000700 1 -- 172.21.15.28:0/1205662875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 msgr2=0x55a16f154490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f493e000700 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f154490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f493e000700 1 -- 172.21.15.28:0/1205662875 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a16efe68d0 con 0x55a16f14deb0 2024-05-20T20:19:12.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f493e000700 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f151830 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f4930012d50 tx=0x7f493000f6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5b36574f2c644fa4 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:12.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f4934c00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:19:12.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f4934c00700 1 -- 172.21.15.28:0/1205662875 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49300120d0 con 0x55a16f14deb0 2024-05-20T20:19:12.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f49380aa710 con 0x55a16f14deb0 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.928+0000 7f4934c00700 1 -- 172.21.15.28:0/1205662875 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(96..96 src has 1..96) ==== 8586+0+0 (secure 0 0 0) 0x7f493000fc00 con 0x55a16f14deb0 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f4934c00700 1 -- 172.21.15.28:0/1205662875 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f4930012270 con 0x55a16f14deb0 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f495838bb80 10 client.28148 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a16f067060 con 0x55a16f14deb0 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f4934c00700 1 -- 172.21.15.28:0/1205662875 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f49380aa710 con 0x55a16f14deb0 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f4934c00700 1 -- 172.21.15.28:0/1205662875 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f4930022030 con 0x55a16f14deb0 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f495838bb80 10 client.28148 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f495838bb80 10 client.28148 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f495838bb80 20 client.28148 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f495838bb80 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1619e0 0x55a16f163e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.929+0000 7f495838bb80 4 client.28148 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.930+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a16f0a8d90 con 0x55a16f1619e0 2024-05-20T20:19:12.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.930+0000 7f4946a00700 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1619e0 0x55a16f163e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.931+0000 7f4946a00700 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1619e0 0x55a16f163e40 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=ac8e7d59cb57c68f server_cookie=a9742f5defd5862e in_seq=0 out_seq=0 2024-05-20T20:19:12.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.931+0000 7f4934c00700 10 client.28148 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.931+0000 7f4934c00700 1 -- 172.21.15.28:0/1205662875 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a16f0a8d90 con 0x55a16f1619e0 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.931+0000 7f4934c00700 10 client.28148 handle_command_reply: tid=0 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.931+0000 7f495838bb80 1 client.28148 shutdown 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 msgr2=0x55a16f151830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f151830 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f4930012d50 tx=0x7f493000f6a0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 shutdown_connections 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f14e7d0 0x55a16f151d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1619e0 0x55a16f163e40 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f148880 0x55a16f154490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f4934c00700 0 client.28148 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:19:12.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 --2- 172.21.15.28:0/1205662875 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f151830 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 >> 172.21.15.28:0/1205662875 conn(0x55a16f076be0 msgr2=0x55a16f147250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:12.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 shutdown_connections 2024-05-20T20:19:12.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 1 -- 172.21.15.28:0/1205662875 wait complete. 2024-05-20T20:19:12.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.932+0000 7f495838bb80 20 client.28148 trim_cache size 0 max 16384 2024-05-20T20:19:12.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.949+0000 7f495838bb80 1 Processor -- start 2024-05-20T20:19:12.949 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f495838bb80 1 -- start start 2024-05-20T20:19:12.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f495838bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f148260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f495838bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f495838bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f150160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f495838bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f49481adda0 con 0x55a16f14e7d0 2024-05-20T20:19:12.950 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f495838bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f49481af8e0 con 0x55a16f14deb0 2024-05-20T20:19:12.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f495838bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f49481ae470 con 0x55a16f148880 2024-05-20T20:19:12.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f4947400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f150160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f4947400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f150160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:52620/0 (socket says 172.21.15.28:52620) 2024-05-20T20:19:12.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f4947400700 1 -- 172.21.15.28:0/211804230 learned_addr learned my addr 172.21.15.28:0/211804230 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:12.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.950+0000 7f4946a00700 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f148260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.951 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f493e000700 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f493e000700 1 -- 172.21.15.28:0/211804230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 msgr2=0x55a16f148260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f493e000700 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f148260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f493e000700 1 -- 172.21.15.28:0/211804230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 msgr2=0x55a16f150160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f493e000700 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f150160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f493e000700 1 -- 172.21.15.28:0/211804230 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a16f13d4c0 con 0x55a16f14deb0 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f4946a00700 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f148260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f493e000700 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f493001c5b0 tx=0x7f4930009f60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b0acef5d107c8a4c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:12.952 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.951+0000 7f4937400700 1 -- 172.21.15.28:0/211804230 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f493001c630 con 0x55a16f14deb0 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f4937400700 1 -- 172.21.15.28:0/211804230 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f493001c7d0 con 0x55a16f14deb0 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f4937400700 1 -- 172.21.15.28:0/211804230 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4930005600 con 0x55a16f14deb0 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 -- 172.21.15.28:0/211804230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 msgr2=0x55a16f14e290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f493001c5b0 tx=0x7f4930009f60 comp rx=0 tx=0).stop 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 -- 172.21.15.28:0/211804230 shutdown_connections 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f150160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f148260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 --2- 172.21.15.28:0/211804230 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f14e290 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 -- 172.21.15.28:0/211804230 >> 172.21.15.28:0/211804230 conn(0x55a16f076be0 msgr2=0x55a16f14b9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.952+0000 7f495838bb80 1 -- 172.21.15.28:0/211804230 shutdown_connections 2024-05-20T20:19:12.953 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.953+0000 7f495838bb80 1 -- 172.21.15.28:0/211804230 wait complete. 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.953+0000 7f495838bb80 1 -- start start 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.953+0000 7f495838bb80 1 Processor -- start 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f495838bb80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f1691b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f495838bb80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f1696f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f495838bb80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f169c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f495838bb80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f49481af210 con 0x55a16f14e7d0 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f495838bb80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f49481aeb40 con 0x55a16f14deb0 2024-05-20T20:19:12.955 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f495838bb80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f49481abd60 con 0x55a16f148880 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f495838bb80 10 client.0 _ll_register_callbacks cb 0x55a16f040c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f1691b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4947400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f169c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f1691b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:34740/0 (socket says 172.21.15.28:34740) 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 -- 172.21.15.28:0/1970936860 learned_addr learned my addr 172.21.15.28:0/1970936860 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f493e000700 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f1696f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 -- 172.21.15.28:0/1970936860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 msgr2=0x55a16f1696f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f1696f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 -- 172.21.15.28:0/1970936860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 msgr2=0x55a16f169c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f169c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.954+0000 7f4946a00700 1 -- 172.21.15.28:0/1970936860 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a16f1d3df0 con 0x55a16f148880 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f4946a00700 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f1691b0 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f49380a7280 tx=0x7f49380a5ed0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3cbff5b44a39254a server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f4934c00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f4934c00700 1 -- 172.21.15.28:0/1970936860 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f49380aa010 con 0x55a16f148880 2024-05-20T20:19:12.956 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f493001c7d0 con 0x55a16f148880 2024-05-20T20:19:12.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f4934c00700 1 -- 172.21.15.28:0/1970936860 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(96..96 src has 1..96) ==== 8586+0+0 (secure 0 0 0) 0x7f493800f460 con 0x55a16f148880 2024-05-20T20:19:12.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f4934c00700 1 -- 172.21.15.28:0/1970936860 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f49380b2de0 con 0x55a16f148880 2024-05-20T20:19:12.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f495838bb80 10 client.28090 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:12.957 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.955+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a16f1d50d0 con 0x55a16f148880 2024-05-20T20:19:12.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f4934c00700 1 -- 172.21.15.28:0/1970936860 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f493001c7d0 con 0x55a16f148880 2024-05-20T20:19:12.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f4934c00700 1 -- 172.21.15.28:0/1970936860 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f49380ca030 con 0x55a16f148880 2024-05-20T20:19:12.958 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f495838bb80 10 client.28090 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f495838bb80 10 client.28090 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.i 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f495838bb80 20 client.28090 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f495838bb80 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1d5640 0x55a16f1d7a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f495838bb80 4 client.28090 mds_command: new command op to 24493 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 --> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55a16f0a2570 con 0x55a16f1d5640 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.956+0000 7f493e000700 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1d5640 0x55a16f1d7a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.957+0000 7f493e000700 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1d5640 0x55a16f1d7a30 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=29e8e83a4c297d0d server_cookie=d26a4e84b8ec29c5 in_seq=0 out_seq=0 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.957+0000 7f4934c00700 10 client.28090 ms_handle_connect on v2:172.21.15.154:6838/1291909351 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.957+0000 7f4934c00700 1 -- 172.21.15.28:0/1970936860 <== mds.2 v2:172.21.15.154:6838/1291909351 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55a16f0a2570 con 0x55a16f1d5640 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.957+0000 7f4934c00700 10 client.28090 handle_command_reply: tid=0 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.957+0000 7f495838bb80 1 client.28090 shutdown 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.957+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 msgr2=0x55a16f1691b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.957+0000 7f495838bb80 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f1691b0 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7f49380a7280 tx=0x7f49380a5ed0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.959 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 shutdown_connections 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:6838/1291909351,v1:172.21.15.154:6839/1291909351] conn(0x55a16f1d5640 0x55a16f1d7a30 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a16f14e7d0 0x55a16f169c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a16f148880 0x55a16f1691b0 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 --2- 172.21.15.28:0/1970936860 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a16f14deb0 0x55a16f1696f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 >> 172.21.15.28:0/1970936860 conn(0x55a16f076be0 msgr2=0x55a16f150120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f4934c00700 0 client.28090 ms_handle_reset on v2:172.21.15.154:6838/1291909351 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 shutdown_connections 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 1 -- 172.21.15.28:0/1970936860 wait complete. 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.958+0000 7f495838bb80 20 client.28090 trim_cache size 0 max 16384 2024-05-20T20:19:12.960 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:19:12.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f492c0b2a70 msgr2=0x7f492c0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f492c0b2a70 0x7f492c0b4f30 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f49400085d0 tx=0x7f494000b040 comp rx=0 tx=0).stop 2024-05-20T20:19:12.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 msgr2=0x7f494819f7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:12.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f494819f7c0 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x7f4938006010 tx=0x7f4938004270 comp rx=0 tx=0).stop 2024-05-20T20:19:12.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 shutdown_connections 2024-05-20T20:19:12.961 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f49481088f0 0x7f4948199980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f49481037f0 0x7f494819f7c0 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f492c0b2a70 0x7f492c0b4f30 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 --2- 172.21.15.28:0/2411074038 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f49481041a0 0x7f494819fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:12.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 >> 172.21.15.28:0/2411074038 conn(0x7f494807a6a0 msgr2=0x7f4948101750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:12.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 shutdown_connections 2024-05-20T20:19:12.962 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:12.959+0000 7f494da00700 1 -- 172.21.15.28:0/2411074038 wait complete. 2024-05-20T20:19:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:13 smithi154 ceph-mon[36314]: pgmap v425: 97 pgs: 22 active+clean+snaptrim, 39 active+clean+snaptrim_wait, 36 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 57 KiB/s wr, 34 op/s 2024-05-20T20:19:13.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:13 smithi033 ceph-mon[36179]: pgmap v425: 97 pgs: 22 active+clean+snaptrim, 39 active+clean+snaptrim_wait, 36 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 57 KiB/s wr, 34 op/s 2024-05-20T20:19:13.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:13 smithi028 ceph-mon[31178]: pgmap v425: 97 pgs: 22 active+clean+snaptrim, 39 active+clean+snaptrim_wait, 36 active+clean; 61 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 57 KiB/s wr, 34 op/s 2024-05-20T20:19:13.907 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:1 damage ls 2024-05-20T20:19:14.384 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:15.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:15 smithi154 ceph-mon[36314]: pgmap v426: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 9.2 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:19:15.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:15 smithi033 ceph-mon[36179]: pgmap v426: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 9.2 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:19:15.784 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:15 smithi028 ceph-mon[31178]: pgmap v426: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 9.2 KiB/s rd, 2.5 KiB/s wr, 13 op/s 2024-05-20T20:19:17.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:16 smithi028 ceph-mon[31178]: osdmap e97: 12 total, 12 up, 12 in 2024-05-20T20:19:17.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:16 smithi028 ceph-mon[31178]: pgmap v428: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.6 KiB/s rd, 3.1 KiB/s wr, 12 op/s 2024-05-20T20:19:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:16 smithi154 ceph-mon[36314]: osdmap e97: 12 total, 12 up, 12 in 2024-05-20T20:19:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:16 smithi154 ceph-mon[36314]: pgmap v428: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.6 KiB/s rd, 3.1 KiB/s wr, 12 op/s 2024-05-20T20:19:17.126 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:16 smithi033 ceph-mon[36179]: osdmap e97: 12 total, 12 up, 12 in 2024-05-20T20:19:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:16 smithi033 ceph-mon[36179]: pgmap v428: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.6 KiB/s rd, 3.1 KiB/s wr, 12 op/s 2024-05-20T20:19:17.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 -- 172.21.15.28:0/1827702116 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810109540 msgr2=0x7fb810109920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 --2- 172.21.15.28:0/1827702116 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810109540 0x7fb810109920 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7fb8040099c0 tx=0x7fb804009cd0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 -- 172.21.15.28:0/1827702116 shutdown_connections 2024-05-20T20:19:17.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 --2- 172.21.15.28:0/1827702116 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103e50 0x7fb810110310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 --2- 172.21.15.28:0/1827702116 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810109540 0x7fb810109920 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 --2- 172.21.15.28:0/1827702116 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb810103490 0x7fb810103910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 -- 172.21.15.28:0/1827702116 >> 172.21.15.28:0/1827702116 conn(0x7fb8100fefc0 msgr2=0x7fb8101013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:17.662 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 -- 172.21.15.28:0/1827702116 shutdown_connections 2024-05-20T20:19:17.663 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.661+0000 7fb817200700 1 -- 172.21.15.28:0/1827702116 wait complete. 2024-05-20T20:19:17.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.662+0000 7fb817200700 1 Processor -- start 2024-05-20T20:19:17.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.662+0000 7fb817200700 1 -- start start 2024-05-20T20:19:17.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb817200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103490 0x7fb81019f6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb817200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810103e50 0x7fb81019fbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb817200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb810109540 0x7fb810199870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb817200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb810113650 con 0x7fb810103e50 2024-05-20T20:19:17.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb817200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101134d0 con 0x7fb810109540 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb817200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101137d0 con 0x7fb810103490 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb815e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103490 0x7fb81019f6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb815400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810103e50 0x7fb81019fbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb815e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103490 0x7fb81019f6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56434/0 (socket says 172.21.15.28:56434) 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb815e00700 1 -- 172.21.15.28:0/3657362354 learned_addr learned my addr 172.21.15.28:0/3657362354 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.663+0000 7fb816800700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb810109540 0x7fb810199870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.664+0000 7fb815400700 1 -- 172.21.15.28:0/3657362354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103490 msgr2=0x7fb81019f6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.664+0000 7fb815400700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103490 0x7fb81019f6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.664+0000 7fb815400700 1 -- 172.21.15.28:0/3657362354 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb810109540 msgr2=0x7fb810199870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.666 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.664+0000 7fb815400700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb810109540 0x7fb810199870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.664+0000 7fb815400700 1 -- 172.21.15.28:0/3657362354 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb810199fd0 con 0x7fb810103e50 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.664+0000 7fb815e00700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103490 0x7fb81019f6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.664+0000 7fb815400700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810103e50 0x7fb81019fbf0 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7fb80000e9f0 tx=0x7fb80000ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.665+0000 7fb80aa00700 1 -- 172.21.15.28:0/3657362354 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb80000c9c0 con 0x7fb810103e50 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.665+0000 7fb80aa00700 1 -- 172.21.15.28:0/3657362354 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb80000cb60 con 0x7fb810103e50 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.665+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb81019a2c0 con 0x7fb810103e50 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.665+0000 7fb80aa00700 1 -- 172.21.15.28:0/3657362354 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb800018680 con 0x7fb810103e50 2024-05-20T20:19:17.667 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.665+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb8101a6580 con 0x7fb810103e50 2024-05-20T20:19:17.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.669+0000 7fb80aa00700 1 -- 172.21.15.28:0/3657362354 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fb800010430 con 0x7fb810103e50 2024-05-20T20:19:17.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.669+0000 7fb8230a1b80 1 Processor -- start 2024-05-20T20:19:17.669 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.670+0000 7fb8230a1b80 1 -- start start 2024-05-20T20:19:17.670 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.670+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4948ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.670+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e4942d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.670+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e49437f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.670+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a6e48a0e40 con 0x55a6e4943390 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.670+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a6e48909e0 con 0x55a6e49487c0 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.670+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a6e4894a20 con 0x55a6e49490e0 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb816800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4948ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb816800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4948ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:55352/0 (socket says 172.21.15.28:55352) 2024-05-20T20:19:17.671 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb816800700 1 -- 172.21.15.28:0/1285248803 learned_addr learned my addr 172.21.15.28:0/1285248803 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:17.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815e00700 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e4942d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.672 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815400700 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e49437f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815e00700 1 -- 172.21.15.28:0/1285248803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 msgr2=0x55a6e4948ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815e00700 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4948ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815e00700 1 -- 172.21.15.28:0/1285248803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 msgr2=0x55a6e49437f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815e00700 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e49437f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815e00700 1 -- 172.21.15.28:0/1285248803 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a6e4859420 con 0x55a6e49490e0 2024-05-20T20:19:17.674 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb816800700 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4948ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.671+0000 7fb815e00700 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e4942d70 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fb8040059d0 tx=0x7fb804005c00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=caecb5b9c96e99c server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.672+0000 7fb80aa00700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb7f80b29f0 0x7fb7f80b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.672+0000 7fb7ff400700 1 -- 172.21.15.28:0/1285248803 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb804028070 con 0x55a6e49490e0 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.672+0000 7fb7ff400700 1 -- 172.21.15.28:0/1285248803 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb804005570 con 0x55a6e49490e0 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.672+0000 7fb7ff400700 1 -- 172.21.15.28:0/1285248803 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb804003bd0 con 0x55a6e49490e0 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.672+0000 7fb80aa00700 1 -- 172.21.15.28:0/3657362354 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7fb8000e2770 con 0x7fb810103e50 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.672+0000 7fb816800700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb7f80b29f0 0x7fb7f80b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.675 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 -- 172.21.15.28:0/1285248803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 msgr2=0x55a6e4942d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb816800700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb7f80b29f0 0x7fb7f80b4eb0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7fb80c007610 tx=0x7fb80c00b040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e4942d70 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fb8040059d0 tx=0x7fb804005c00 comp rx=0 tx=0).stop 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 -- 172.21.15.28:0/1285248803 shutdown_connections 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e49437f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4948ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/1285248803 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e4942d70 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 -- 172.21.15.28:0/1285248803 >> 172.21.15.28:0/1285248803 conn(0x55a6e4870be0 msgr2=0x55a6e493d810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 -- 172.21.15.28:0/1285248803 shutdown_connections 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.673+0000 7fb8230a1b80 1 -- 172.21.15.28:0/1285248803 wait complete. 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.674+0000 7fb8230a1b80 1 -- start start 2024-05-20T20:19:17.676 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.674+0000 7fb8230a1b80 1 Processor -- start 2024-05-20T20:19:17.680 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.674+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e494ee10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.674+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e494c1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x55a6e4796fa0 con 0x55a6e4943390 2024-05-20T20:19:17.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x55a6e4831760 con 0x55a6e49487c0 2024-05-20T20:19:17.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x55a6e494fba0 con 0x55a6e49490e0 2024-05-20T20:19:17.681 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb8230a1b80 10 client.0 _ll_register_callbacks cb 0x55a6e483ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:17.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb815e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e494ee10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.682 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb815e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e494ee10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53114/0 (socket says 172.21.15.28:53114) 2024-05-20T20:19:17.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb815e00700 1 -- 172.21.15.28:0/2916205181 learned_addr learned my addr 172.21.15.28:0/2916205181 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:17.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb815400700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e494c1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb816800700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb815e00700 1 -- 172.21.15.28:0/2916205181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 msgr2=0x55a6e494c6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.683 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb815e00700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c6f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.675+0000 7fb815e00700 1 -- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 msgr2=0x55a6e494c1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb815e00700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e494c1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb815e00700 1 -- 172.21.15.28:0/2916205181 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a6e47e08d0 con 0x55a6e4943390 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb815400700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e494c1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb815e00700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e494ee10 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7fb8040050d0 tx=0x7fb80400b6c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=18aa223a30b22f5d server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb7f3e00700 10 client.0 ms_handle_connect on v2:172.21.15.28:3300/0 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb7f3e00700 1 -- 172.21.15.28:0/2916205181 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb804031060 con 0x55a6e4943390 2024-05-20T20:19:17.684 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7fb804005570 con 0x55a6e4943390 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.676+0000 7fb7f3e00700 1 -- 172.21.15.28:0/2916205181 <== mon.0 v2:172.21.15.28:3300/0 2 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7fb8040037c0 con 0x55a6e4943390 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb7f3e00700 1 -- 172.21.15.28:0/2916205181 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb804028040 con 0x55a6e4943390 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb8230a1b80 10 client.17544 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a6e4861060 con 0x55a6e4943390 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb7f3e00700 1 -- 172.21.15.28:0/2916205181 <== mon.0 v2:172.21.15.28:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb804005570 con 0x55a6e4943390 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb7f3e00700 1 -- 172.21.15.28:0/2916205181 <== mon.0 v2:172.21.15.28:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb804023c30 con 0x55a6e4943390 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb8230a1b80 10 client.17544 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb8230a1b80 10 client.17544 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.677+0000 7fb8230a1b80 20 client.17544 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:17.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.678+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e495c590 0x55a6e495e980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.678+0000 7fb8230a1b80 4 client.17544 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:19:17.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.678+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x55a6e48a2d90 con 0x55a6e495c590 2024-05-20T20:19:17.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.678+0000 7fb815400700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e495c590 0x55a6e495e980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.679+0000 7fb815400700 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e495c590 0x55a6e495e980 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f5cfa2d6ce5ce56d server_cookie=ff635151a556918 in_seq=0 out_seq=0 2024-05-20T20:19:17.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.679+0000 7fb7f3e00700 10 client.17544 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:19:17.686 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.679+0000 7fb7f3e00700 1 -- 172.21.15.28:0/2916205181 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x55a6e48a2d90 con 0x55a6e495c590 2024-05-20T20:19:17.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.679+0000 7fb7f3e00700 10 client.17544 handle_command_reply: tid=0 2024-05-20T20:19:17.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 client.17544 shutdown 2024-05-20T20:19:17.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 msgr2=0x55a6e494ee10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e494ee10 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7fb8040050d0 tx=0x7fb80400b6c0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 shutdown_connections 2024-05-20T20:19:17.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e495c590 0x55a6e495e980 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e494ee10 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/2916205181 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e494c1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 >> 172.21.15.28:0/2916205181 conn(0x55a6e4870be0 msgr2=0x55a6e493cfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:17.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb7f3e00700 0 client.17544 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:19:17.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 shutdown_connections 2024-05-20T20:19:17.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 1 -- 172.21.15.28:0/2916205181 wait complete. 2024-05-20T20:19:17.688 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.680+0000 7fb8230a1b80 20 client.17544 trim_cache size 0 max 16384 2024-05-20T20:19:17.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 Processor -- start 2024-05-20T20:19:17.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 -- start start 2024-05-20T20:19:17.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e4942d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4949e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.700 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101ab960 con 0x55a6e4943390 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101ad4a0 con 0x55a6e49487c0 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.697+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101ac030 con 0x55a6e49490e0 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e4942d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e4942d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53116/0 (socket says 172.21.15.28:53116) 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4949e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815e00700 1 -- 172.21.15.28:0/3909015028 learned_addr learned my addr 172.21.15.28:0/3909015028 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb816800700 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.701 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815400700 1 -- 172.21.15.28:0/3909015028 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 msgr2=0x55a6e494c600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815400700 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815400700 1 -- 172.21.15.28:0/3909015028 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 msgr2=0x55a6e4942d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815400700 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e4942d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815400700 1 -- 172.21.15.28:0/3909015028 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55a6e4948df0 con 0x55a6e49487c0 2024-05-20T20:19:17.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb816800700 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:19:17.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.698+0000 7fb815400700 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4949e10 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7fb8000dfeb0 tx=0x7fb8000e4c80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3fe496c4713ae723 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:17.702 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.699+0000 7fb7fea00700 1 -- 172.21.15.28:0/3909015028 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8000b2040 con 0x55a6e49487c0 2024-05-20T20:19:17.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.699+0000 7fb7fea00700 1 -- 172.21.15.28:0/3909015028 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fb8000b21e0 con 0x55a6e49487c0 2024-05-20T20:19:17.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.699+0000 7fb7fea00700 1 -- 172.21.15.28:0/3909015028 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb8000b28a0 con 0x55a6e49487c0 2024-05-20T20:19:17.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.699+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3909015028 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 msgr2=0x55a6e4949e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.699+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4949e10 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7fb8000dfeb0 tx=0x7fb8000e4c80 comp rx=0 tx=0).stop 2024-05-20T20:19:17.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3909015028 shutdown_connections 2024-05-20T20:19:17.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e49490e0 0x55a6e494c600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.703 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e4943390 0x55a6e4942d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3909015028 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e4949e10 unknown :-1 s=CLOSED pgs=630 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3909015028 >> 172.21.15.28:0/3909015028 conn(0x55a6e4870be0 msgr2=0x55a6e494aaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:17.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3909015028 shutdown_connections 2024-05-20T20:19:17.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3909015028 wait complete. 2024-05-20T20:19:17.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 -- start start 2024-05-20T20:19:17.704 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.700+0000 7fb8230a1b80 1 Processor -- start 2024-05-20T20:19:17.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e4943390 0x55a6e49bf130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.708 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e49bf670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb8230a1b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e49490e0 0x55a6e495eb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101acdd0 con 0x55a6e49490e0 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101ac700 con 0x55a6e49487c0 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb8230a1b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fb8101a9940 con 0x55a6e4943390 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb8230a1b80 10 client.0 _ll_register_callbacks cb 0x55a6e483ac70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb815400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e49bf670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb815e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e4943390 0x55a6e49bf130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb816800700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e49490e0 0x55a6e495eb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.709 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb815400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e49bf670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:55374/0 (socket says 172.21.15.28:55374) 2024-05-20T20:19:17.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.701+0000 7fb815400700 1 -- 172.21.15.28:0/3023505749 learned_addr learned my addr 172.21.15.28:0/3023505749 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:17.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb815e00700 1 -- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 msgr2=0x55a6e49bf670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb815e00700 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e49bf670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb815e00700 1 -- 172.21.15.28:0/3023505749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e49490e0 msgr2=0x55a6e495eb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:17.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb815e00700 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e49490e0 0x55a6e495eb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb815e00700 1 -- 172.21.15.28:0/3023505749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55a6e495f250 con 0x55a6e4943390 2024-05-20T20:19:17.710 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb815400700 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e49bf670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb815e00700 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e4943390 0x55a6e49bf130 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7fb804006010 tx=0x7fb80400fc70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2eded6c694525c41 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb7f3e00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb7f3e00700 1 -- 172.21.15.28:0/3023505749 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb804031040 con 0x55a6e4943390 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb7f3e00700 1 -- 172.21.15.28:0/3023505749 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7fb804003bd0 con 0x55a6e4943390 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.702+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x55a6e489c570 con 0x55a6e4943390 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb7f3e00700 1 -- 172.21.15.28:0/3023505749 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7fb804044050 con 0x55a6e4943390 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 10 client.28108 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:17.711 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x55a6e49c6900 con 0x55a6e4943390 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb7f3e00700 1 -- 172.21.15.28:0/3023505749 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x55a6e489c570 con 0x55a6e4943390 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb7f3e00700 1 -- 172.21.15.28:0/3023505749 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7fb804023910 con 0x55a6e4943390 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 10 client.28108 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 10 client.28108 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.h 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 20 client.28108 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e49c6ef0 0x55a6e49c9310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 4 client.28108 mds_command: new command op to 24500 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:19:17.712 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.703+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 --> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] -- command(tid 0: {"prefix": "damage ls"}) -- 0x55a6e4713250 con 0x55a6e49c6ef0 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.704+0000 7fb815400700 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e49c6ef0 0x55a6e49c9310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.704+0000 7fb815400700 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e49c6ef0 0x55a6e49c9310 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=439866310659e919 server_cookie=1cb809727a0b424 in_seq=0 out_seq=0 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.704+0000 7fb7f3e00700 10 client.28108 ms_handle_connect on v2:172.21.15.33:6836/4183908883 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb7f3e00700 1 -- 172.21.15.28:0/3023505749 <== mds.1 v2:172.21.15.33:6836/4183908883 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x55a6e4713250 con 0x55a6e49c6ef0 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb7f3e00700 10 client.28108 handle_command_reply: tid=0 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 client.28108 shutdown 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e4943390 msgr2=0x55a6e49bf130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e4943390 0x55a6e49bf130 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7fb804006010 tx=0x7fb80400fc70 comp rx=0 tx=0).stop 2024-05-20T20:19:17.713 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 shutdown_connections 2024-05-20T20:19:17.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:6836/4183908883,v1:172.21.15.33:6837/4183908883] conn(0x55a6e49c6ef0 0x55a6e49c9310 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x55a6e49490e0 0x55a6e495eb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x55a6e4943390 0x55a6e49bf130 unknown :-1 s=CLOSED pgs=527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb7f3e00700 0 client.28108 ms_handle_reset on v2:172.21.15.33:6836/4183908883 2024-05-20T20:19:17.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 --2- 172.21.15.28:0/3023505749 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x55a6e49487c0 0x55a6e49bf670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 >> 172.21.15.28:0/3023505749 conn(0x55a6e4870be0 msgr2=0x55a6e4949c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:17.714 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 shutdown_connections 2024-05-20T20:19:17.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 1 -- 172.21.15.28:0/3023505749 wait complete. 2024-05-20T20:19:17.715 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.705+0000 7fb8230a1b80 20 client.28108 trim_cache size 0 max 16384 2024-05-20T20:19:17.715 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:19:17.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb7f80b29f0 msgr2=0x7fb7f80b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb7f80b29f0 0x7fb7f80b4eb0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7fb80c007610 tx=0x7fb80c00b040 comp rx=0 tx=0).stop 2024-05-20T20:19:17.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810103e50 msgr2=0x7fb81019fbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:17.716 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810103e50 0x7fb81019fbf0 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7fb80000e9f0 tx=0x7fb80000ed00 comp rx=0 tx=0).stop 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 shutdown_connections 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fb7f80b29f0 0x7fb7f80b4eb0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fb810109540 0x7fb810199870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb810103490 0x7fb81019f6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 --2- 172.21.15.28:0/3657362354 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fb810103e50 0x7fb81019fbf0 unknown :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.706+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 >> 172.21.15.28:0/3657362354 conn(0x7fb8100fefc0 msgr2=0x7fb8101005c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.707+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 shutdown_connections 2024-05-20T20:19:17.717 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:17.707+0000 7fb817200700 1 -- 172.21.15.28:0/3657362354 wait complete. 2024-05-20T20:19:18.544 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph tell mds.1:0 damage ls 2024-05-20T20:19:19.041 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:19.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:19 smithi028 ceph-mon[31178]: pgmap v429: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 2.5 KiB/s wr, 11 op/s 2024-05-20T20:19:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:19 smithi154 ceph-mon[36314]: pgmap v429: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 2.5 KiB/s wr, 11 op/s 2024-05-20T20:19:19.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:19 smithi033 ceph-mon[36179]: pgmap v429: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 2.5 KiB/s wr, 11 op/s 2024-05-20T20:19:21.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:21 smithi028 ceph-mon[31178]: pgmap v430: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 2.4 KiB/s wr, 10 op/s 2024-05-20T20:19:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:21 smithi033 ceph-mon[36179]: pgmap v430: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 2.4 KiB/s wr, 10 op/s 2024-05-20T20:19:21.628 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:21 smithi154 ceph-mon[36314]: pgmap v430: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 2.4 KiB/s wr, 10 op/s 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.575+0000 7f2359400700 1 -- 172.21.15.28:0/3593720983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406b250 msgr2=0x7f235406b630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.575+0000 7f2359400700 1 --2- 172.21.15.28:0/3593720983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406b250 0x7f235406b630 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7f2344009990 tx=0x7f2344009ca0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.576+0000 7f2359400700 1 -- 172.21.15.28:0/3593720983 shutdown_connections 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.576+0000 7f2359400700 1 --2- 172.21.15.28:0/3593720983 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f235410fff0 0x7f23541123e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.576+0000 7f2359400700 1 --2- 172.21.15.28:0/3593720983 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406b250 0x7f235406b630 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.576+0000 7f2359400700 1 --2- 172.21.15.28:0/3593720983 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f235406bb70 0x7f235410fab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.576+0000 7f2359400700 1 -- 172.21.15.28:0/3593720983 >> 172.21.15.28:0/3593720983 conn(0x7f23540793f0 msgr2=0x7f2354079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.576+0000 7f2359400700 1 -- 172.21.15.28:0/3593720983 shutdown_connections 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.576+0000 7f2359400700 1 -- 172.21.15.28:0/3593720983 wait complete. 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.577+0000 7f2359400700 1 Processor -- start 2024-05-20T20:19:22.577 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.577+0000 7f2359400700 1 -- start start 2024-05-20T20:19:22.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2359400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f235406b250 0x7f235419be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2359400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406bb70 0x7f235419c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2359400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f235410fff0 0x7f23541a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2359400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f23541158f0 con 0x7f235410fff0 2024-05-20T20:19:22.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2359400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f2354115770 con 0x7f235406b250 2024-05-20T20:19:22.579 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2359400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f2354115a70 con 0x7f235406bb70 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2353200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f235410fff0 0x7f23541a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2351e00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406bb70 0x7f235419c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2353200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f235410fff0 0x7f23541a06a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53158/0 (socket says 172.21.15.28:53158) 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2353200700 1 -- 172.21.15.28:0/2087247941 learned_addr learned my addr 172.21.15.28:0/2087247941 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.578+0000 7f2352800700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f235406b250 0x7f235419be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2352800700 1 -- 172.21.15.28:0/2087247941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406bb70 msgr2=0x7f235419c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2352800700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406bb70 0x7f235419c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2352800700 1 -- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f235410fff0 msgr2=0x7f23541a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2352800700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f235410fff0 0x7f23541a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2352800700 1 -- 172.21.15.28:0/2087247941 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f23541a0de0 con 0x7f235406b250 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2351e00700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406bb70 0x7f235419c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2352800700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f235406b250 0x7f235419be20 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7f23440059d0 tx=0x7f23440047b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:22.580 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f234b400700 1 -- 172.21.15.28:0/2087247941 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2344027070 con 0x7f235406b250 2024-05-20T20:19:22.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f23541a1070 con 0x7f235406b250 2024-05-20T20:19:22.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f234b400700 1 -- 172.21.15.28:0/2087247941 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f2344022470 con 0x7f235406b250 2024-05-20T20:19:22.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.579+0000 7f234b400700 1 -- 172.21.15.28:0/2087247941 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2344017650 con 0x7f235406b250 2024-05-20T20:19:22.581 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.580+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f23541a8950 con 0x7f235406b250 2024-05-20T20:19:22.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.583+0000 7f234b400700 1 -- 172.21.15.28:0/2087247941 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f234402a070 con 0x7f235406b250 2024-05-20T20:19:22.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.584+0000 7f2363e20b80 1 Processor -- start 2024-05-20T20:19:22.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.584+0000 7f2363e20b80 1 -- start start 2024-05-20T20:19:22.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b5490eb0 0x5607b5491290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b548b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b548bce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2363e20b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5607b53e9e40 con 0x5607b54917d0 2024-05-20T20:19:22.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2363e20b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5607b53d99e0 con 0x5607b548b880 2024-05-20T20:19:22.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2363e20b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5607b53dda20 con 0x5607b5490eb0 2024-05-20T20:19:22.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f234b400700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23380b2a70 0x7f23380b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f234b400700 1 -- 172.21.15.28:0/2087247941 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7f23440e3210 con 0x7f235406b250 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2351e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b548bce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2351e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b548bce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:55404/0 (socket says 172.21.15.28:55404) 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2351e00700 1 -- 172.21.15.28:0/1288943731 learned_addr learned my addr 172.21.15.28:0/1288943731 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2352800700 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b548b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2353200700 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b5490eb0 0x5607b5491290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.585+0000 7f2353200700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23380b2a70 0x7f23380b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.586+0000 7f2351e00700 1 -- 172.21.15.28:0/1288943731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b5490eb0 msgr2=0x5607b5491290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.586+0000 7f2351e00700 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b5490eb0 0x5607b5491290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.586+0000 7f2351e00700 1 -- 172.21.15.28:0/1288943731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 msgr2=0x5607b548b260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.586+0000 7f2351e00700 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b548b260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.586+0000 7f2351e00700 1 -- 172.21.15.28:0/1288943731 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5607b53a2420 con 0x5607b548b880 2024-05-20T20:19:22.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.586+0000 7f2351e00700 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b548bce0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f23400115f0 tx=0x7f2340011900 comp rx=0 tx=0).ready entity=mon.2 client_cookie=20703dd5d016c4c9 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:22.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.587+0000 7f2353200700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23380b2a70 0x7f23380b4f30 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f234c009d90 tx=0x7f234c009d20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:22.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.587+0000 7f233fe00700 1 -- 172.21.15.28:0/1288943731 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2340004230 con 0x5607b548b880 2024-05-20T20:19:22.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.587+0000 7f233fe00700 1 -- 172.21.15.28:0/1288943731 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23400043d0 con 0x5607b548b880 2024-05-20T20:19:22.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.587+0000 7f233fe00700 1 -- 172.21.15.28:0/1288943731 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23400198e0 con 0x5607b548b880 2024-05-20T20:19:22.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 -- 172.21.15.28:0/1288943731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 msgr2=0x5607b548bce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b548bce0 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f23400115f0 tx=0x7f2340011900 comp rx=0 tx=0).stop 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 -- 172.21.15.28:0/1288943731 shutdown_connections 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b548bce0 unknown :-1 s=CLOSED pgs=632 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b5490eb0 0x5607b5491290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1288943731 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b548b260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 -- 172.21.15.28:0/1288943731 >> 172.21.15.28:0/1288943731 conn(0x5607b53b9be0 msgr2=0x5607b5486010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 -- 172.21.15.28:0/1288943731 shutdown_connections 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 -- 172.21.15.28:0/1288943731 wait complete. 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.588+0000 7f2363e20b80 1 -- start start 2024-05-20T20:19:22.589 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 1 Processor -- start 2024-05-20T20:19:22.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b54974c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b5494860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5494da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x5607b52dffa0 con 0x5607b54917d0 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x5607b537a760 con 0x5607b5490eb0 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x5607b5498240 con 0x5607b548b880 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.589+0000 7f2363e20b80 10 client.0 _ll_register_callbacks cb 0x5607b5383c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b54974c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b54974c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56500/0 (socket says 172.21.15.28:56500) 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 -- 172.21.15.28:0/1379355805 learned_addr learned my addr 172.21.15.28:0/1379355805 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2351e00700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b5494860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2353200700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5494da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 -- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 msgr2=0x5607b5494860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b5494860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 -- 172.21.15.28:0/1379355805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 msgr2=0x5607b5494da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5494da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 -- 172.21.15.28:0/1379355805 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5607b53298d0 con 0x5607b548b880 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2353200700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5494da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f2352800700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b54974c0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f2344005cb0 tx=0x7f23440aa350 comp rx=0 tx=0).ready entity=mon.1 client_cookie=92302fbbea5d61d7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.590+0000 7f233cc00700 10 client.0 ms_handle_connect on v2:172.21.15.154:3300/0 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.591+0000 7f233cc00700 1 -- 172.21.15.28:0/1379355805 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23440095d0 con 0x5607b548b880 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.591+0000 7f233cc00700 1 -- 172.21.15.28:0/1379355805 <== mon.1 v2:172.21.15.154:3300/0 2 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7f23440e0b70 con 0x5607b548b880 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.591+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f23400043d0 con 0x5607b548b880 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.591+0000 7f233cc00700 1 -- 172.21.15.28:0/1379355805 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f23440e0d70 con 0x5607b548b880 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.591+0000 7f2363e20b80 10 client.28120 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.591+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5607b53aa060 con 0x5607b548b880 2024-05-20T20:19:22.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.591+0000 7f233cc00700 1 -- 172.21.15.28:0/1379355805 <== mon.1 v2:172.21.15.154:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23400043d0 con 0x5607b548b880 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f233cc00700 1 -- 172.21.15.28:0/1379355805 <== mon.1 v2:172.21.15.154:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f23440aa770 con 0x5607b548b880 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f2363e20b80 10 client.28120 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f2363e20b80 10 client.28120 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f2363e20b80 20 client.28120 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b54a4cd0 0x5607b54a70e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f2363e20b80 4 client.28120 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "get_command_descriptions"}) -- 0x5607b53ebd90 con 0x5607b54a4cd0 2024-05-20T20:19:22.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.592+0000 7f2351e00700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b54a4cd0 0x5607b54a70e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.593+0000 7f2351e00700 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b54a4cd0 0x5607b54a70e0 crc :-1 s=READY pgs=275 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5c7a42cccb879bb7 server_cookie=9f5a9b556db01637 in_seq=0 out_seq=0 2024-05-20T20:19:22.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.593+0000 7f233cc00700 10 client.28120 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:22.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f233cc00700 1 -- 172.21.15.28:0/1379355805 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+24041 (crc 0 0 0) 0x5607b53ebd90 con 0x5607b54a4cd0 2024-05-20T20:19:22.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f233cc00700 10 client.28120 handle_command_reply: tid=0 2024-05-20T20:19:22.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 client.28120 shutdown 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 msgr2=0x5607b54974c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b54974c0 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f2344005cb0 tx=0x7f23440aa350 comp rx=0 tx=0).stop 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 shutdown_connections 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b54a4cd0 0x5607b54a70e0 crc :-1 s=READY pgs=275 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5494da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b54974c0 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 --2- 172.21.15.28:0/1379355805 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b5494860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f233cc00700 0 client.28120 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 >> 172.21.15.28:0/1379355805 conn(0x5607b53b9be0 msgr2=0x5607b548a250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 shutdown_connections 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 1 -- 172.21.15.28:0/1379355805 wait complete. 2024-05-20T20:19:22.596 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.594+0000 7f2363e20b80 20 client.28120 trim_cache size 0 max 16384 2024-05-20T20:19:22.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 Processor -- start 2024-05-20T20:19:22.611 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 -- start start 2024-05-20T20:19:22.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b548b260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b54a70c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5493150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f23541b0020 con 0x5607b54917d0 2024-05-20T20:19:22.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f23541b1b60 con 0x5607b5490eb0 2024-05-20T20:19:22.612 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.612+0000 7f2363e20b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f23541b06f0 con 0x5607b548b880 2024-05-20T20:19:22.613 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b548b260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2351e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b54a70c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b548b260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56508/0 (socket says 172.21.15.28:56508) 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 -- 172.21.15.28:0/267436008 learned_addr learned my addr 172.21.15.28:0/267436008 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2353200700 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5493150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 -- 172.21.15.28:0/267436008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 msgr2=0x5607b54a70c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b54a70c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 -- 172.21.15.28:0/267436008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 msgr2=0x5607b5493150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5493150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.613+0000 7f2352800700 1 -- 172.21.15.28:0/267436008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x5607b54a7760 con 0x5607b548b880 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.614+0000 7f2352800700 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b548b260 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f23440a7780 tx=0x7f2344005fd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2c0891341cc8457f server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.614+0000 7f233f400700 1 -- 172.21.15.28:0/267436008 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23440a9e60 con 0x5607b548b880 2024-05-20T20:19:22.614 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.614+0000 7f233f400700 1 -- 172.21.15.28:0/267436008 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23440aa020 con 0x5607b548b880 2024-05-20T20:19:22.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.614+0000 7f233f400700 1 -- 172.21.15.28:0/267436008 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23440e0b70 con 0x5607b548b880 2024-05-20T20:19:22.615 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 -- 172.21.15.28:0/267436008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 msgr2=0x5607b548b260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b548b260 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f23440a7780 tx=0x7f2344005fd0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 -- 172.21.15.28:0/267436008 shutdown_connections 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b54917d0 0x5607b5493150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b548b880 0x5607b548b260 unknown :-1 s=CLOSED pgs=530 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 --2- 172.21.15.28:0/267436008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b5490eb0 0x5607b54a70c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 -- 172.21.15.28:0/267436008 >> 172.21.15.28:0/267436008 conn(0x5607b53b9be0 msgr2=0x5607b548e9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 -- 172.21.15.28:0/267436008 shutdown_connections 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 -- 172.21.15.28:0/267436008 wait complete. 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.615+0000 7f2363e20b80 1 -- start start 2024-05-20T20:19:22.616 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 1 Processor -- start 2024-05-20T20:19:22.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b549d6b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.617 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b5490eb0 0x5607b549dbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b54917d0 0x5607b54abd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f23541b1490 con 0x5607b5490eb0 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f23541b0dc0 con 0x5607b548b880 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f23541adfe0 con 0x5607b54917d0 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.616+0000 7f2363e20b80 10 client.0 _ll_register_callbacks cb 0x5607b5383c70 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2351e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b5490eb0 0x5607b549dbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2352800700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b549d6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2351e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b5490eb0 0x5607b549dbf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53192/0 (socket says 172.21.15.28:53192) 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2353200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b54917d0 0x5607b54abd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2351e00700 1 -- 172.21.15.28:0/934886106 learned_addr learned my addr 172.21.15.28:0/934886106 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2352800700 1 -- 172.21.15.28:0/934886106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b54917d0 msgr2=0x5607b54abd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2352800700 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b54917d0 0x5607b54abd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.618 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2352800700 1 -- 172.21.15.28:0/934886106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b5490eb0 msgr2=0x5607b549dbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2352800700 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b5490eb0 0x5607b549dbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2352800700 1 -- 172.21.15.28:0/934886106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x5607b54ac5f0 con 0x5607b548b880 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f2352800700 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b549d6b0 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f23440aa660 tx=0x7f2344022ce0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ef3acecebed73ab7 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f233cc00700 10 client.0 ms_handle_connect on v2:172.21.15.33:3300/0 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.617+0000 7f233cc00700 1 -- 172.21.15.28:0/934886106 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f23440a9e60 con 0x5607b548b880 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f233cc00700 1 -- 172.21.15.28:0/934886106 <== mon.2 v2:172.21.15.33:3300/0 2 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7f23440b3060 con 0x5607b548b880 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_get_version(what=fsmap handle=1) -- 0x7f23440aa020 con 0x5607b548b880 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f233cc00700 1 -- 172.21.15.28:0/934886106 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_get_version_reply(handle=1 version=29) ==== 24+0+0 (secure 0 0 0) 0x7f23440095d0 con 0x5607b548b880 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 10 client.28190 fetch_fsmap learned FSMap version 29 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,fsmap=29,monmap=4+}) -- 0x5607b54ac5f0 con 0x5607b548b880 2024-05-20T20:19:22.619 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f233cc00700 1 -- 172.21.15.28:0/934886106 <== mon.2 v2:172.21.15.33:3300/0 4 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f23440aa020 con 0x5607b548b880 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f233cc00700 1 -- 172.21.15.28:0/934886106 <== mon.2 v2:172.21.15.33:3300/0 5 ==== fsmap(e 29) ==== 8354+0+0 (secure 0 0 0) 0x7f23440a5e80 con 0x5607b548b880 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 10 client.28190 fetch_fsmap finished waiting for FSMap version 29 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 10 client.28190 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 20 client.28190 populate_metadata read hostname 'smithi028' 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b550f600 0x5607b5511a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 4 client.28190 mds_command: new command op to 24542 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.618+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 --> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] -- command(tid 0: {"prefix": "damage ls"}) -- 0x5607b53e5570 con 0x5607b550f600 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.619+0000 7f2351e00700 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b550f600 0x5607b5511a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.619+0000 7f2351e00700 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b550f600 0x5607b5511a20 crc :-1 s=READY pgs=276 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9992544bc571ab0a server_cookie=593519abf57dd1d2 in_seq=0 out_seq=0 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.619+0000 7f233cc00700 10 client.28190 ms_handle_connect on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.619+0000 7f233cc00700 1 -- 172.21.15.28:0/934886106 <== mds.0 v2:172.21.15.33:6838/2646117588 1 ==== command_reply(tid 0: 0 ) ==== 8+0+3 (crc 0 0 0) 0x5607b53e5570 con 0x5607b550f600 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.619+0000 7f233cc00700 10 client.28190 handle_command_reply: tid=0 2024-05-20T20:19:22.621 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.619+0000 7f2363e20b80 1 client.28190 shutdown 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 msgr2=0x5607b549d6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b549d6b0 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f23440aa660 tx=0x7f2344022ce0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 shutdown_connections 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:6838/2646117588,v1:172.21.15.33:6839/2646117588] conn(0x5607b550f600 0x5607b5511a20 crc :-1 s=READY pgs=276 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x5607b54917d0 0x5607b54abd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x5607b548b880 0x5607b549d6b0 unknown :-1 s=CLOSED pgs=633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 --2- 172.21.15.28:0/934886106 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x5607b5490eb0 0x5607b549dbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f233cc00700 0 client.28190 ms_handle_reset on v2:172.21.15.33:6838/2646117588 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 >> 172.21.15.28:0/934886106 conn(0x5607b53b9be0 msgr2=0x5607b5493090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 shutdown_connections 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 1 -- 172.21.15.28:0/934886106 wait complete. 2024-05-20T20:19:22.622 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.620+0000 7f2363e20b80 20 client.28190 trim_cache size 0 max 16384 2024-05-20T20:19:22.623 INFO:teuthology.orchestra.run.smithi028.stdout:[] 2024-05-20T20:19:22.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23380b2a70 msgr2=0x7f23380b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23380b2a70 0x7f23380b4f30 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f234c009d90 tx=0x7f234c009d20 comp rx=0 tx=0).stop 2024-05-20T20:19:22.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f235406b250 msgr2=0x7f235419be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:22.623 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f235406b250 0x7f235419be20 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7f23440059d0 tx=0x7f23440047b0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 shutdown_connections 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f235410fff0 0x7f23541a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f23380b2a70 0x7f23380b4f30 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f235406b250 0x7f235419be20 unknown :-1 s=CLOSED pgs=631 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 --2- 172.21.15.28:0/2087247941 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f235406bb70 0x7f235419c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 >> 172.21.15.28:0/2087247941 conn(0x7f23540793f0 msgr2=0x7f23541114e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 shutdown_connections 2024-05-20T20:19:22.624 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:22.621+0000 7f2359400700 1 -- 172.21.15.28:0/2087247941 wait complete. 2024-05-20T20:19:23.304 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:23 smithi028 ceph-mon[31178]: pgmap v431: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 2.4 KiB/s wr, 10 op/s 2024-05-20T20:19:23.515 ERROR:tasks.fwd_scrub.fs.[cephfs]:exception: Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_ceph-c_4ebd1d5361d457738957dc0455bba90102296634/qa/tasks/fwd_scrub.py", line 35, in _run self.do_scrub() File "/home/teuthworker/src/git.ceph.com_ceph-c_4ebd1d5361d457738957dc0455bba90102296634/qa/tasks/fwd_scrub.py", line 49, in do_scrub self.sleep_unless_stopped(self.sleep_between_iterations) File "/home/teuthworker/src/git.ceph.com_ceph-c_4ebd1d5361d457738957dc0455bba90102296634/qa/tasks/thrasher.py", line 47, in sleep_unless_stopped raise self.Stopped() tasks.thrasher.ThrasherGreenlet.Stopped 2024-05-20T20:19:23.515 INFO:tasks.fwd_scrub:done joining 2024-05-20T20:19:23.516 DEBUG:teuthology.run_tasks:Unwinding manager check-counter 2024-05-20T20:19:23.527 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-05-20T20:19:23.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:23 smithi154 ceph-mon[36314]: pgmap v431: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 2.4 KiB/s wr, 10 op/s 2024-05-20T20:19:23.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:23 smithi033 ceph-mon[36179]: pgmap v431: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 2.4 KiB/s wr, 10 op/s 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.746+0000 7fe611400640 1 -- 172.21.15.28:0/2736801051 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 msgr2=0x7fe60c06a580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.746+0000 7fe611400640 1 --2- 172.21.15.28:0/2736801051 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 0x7fe60c06a580 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7fe5f4009900 tx=0x7fe5f402a3f0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.747+0000 7fe611400640 1 -- 172.21.15.28:0/2736801051 shutdown_connections 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.747+0000 7fe611400640 1 --2- 172.21.15.28:0/2736801051 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe60c10f0b0 0x7fe60c1114c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.747+0000 7fe611400640 1 --2- 172.21.15.28:0/2736801051 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 0x7fe60c06a580 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.747+0000 7fe611400640 1 --2- 172.21.15.28:0/2736801051 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c103750 0x7fe60c069be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.747+0000 7fe611400640 1 -- 172.21.15.28:0/2736801051 >> 172.21.15.28:0/2736801051 conn(0x7fe60c077290 msgr2=0x7fe60c0776a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.747+0000 7fe611400640 1 -- 172.21.15.28:0/2736801051 shutdown_connections 2024-05-20T20:19:23.747 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.747+0000 7fe611400640 1 -- 172.21.15.28:0/2736801051 wait complete. 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.748+0000 7fe611400640 1 Processor -- start 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.748+0000 7fe611400640 1 -- start start 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe611400640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 0x7fe60c19b2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe611400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe60c103750 0x7fe60c19b7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe611400640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c10f0b0 0x7fe60c19fb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe611400640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fe60c114d20 con 0x7fe60c10f0b0 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe611400640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fe60c114ba0 con 0x7fe60c103750 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe611400640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fe60c114ea0 con 0x7fe60c06a120 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe610a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c10f0b0 0x7fe60c19fb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe60b400640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe60c103750 0x7fe60c19b7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe610a00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c10f0b0 0x7fe60c19fb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53212/0 (socket says 172.21.15.28:53212) 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe610a00640 1 -- 172.21.15.28:0/3511830063 learned_addr learned my addr 172.21.15.28:0/3511830063 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:23.749 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.749+0000 7fe60be00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 0x7fe60c19b2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:23.750 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe610a00640 1 -- 172.21.15.28:0/3511830063 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 msgr2=0x7fe60c19b2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe610a00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 0x7fe60c19b2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe610a00640 1 -- 172.21.15.28:0/3511830063 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe60c103750 msgr2=0x7fe60c19b7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe610a00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe60c103750 0x7fe60c19b7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe610a00640 1 -- 172.21.15.28:0/3511830063 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe60c1a0290 con 0x7fe60c10f0b0 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe60be00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 0x7fe60c19b2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe60b400640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe60c103750 0x7fe60c19b7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe610a00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c10f0b0 0x7fe60c19fb60 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7fe60400e970 tx=0x7fe60400ed70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe608c00640 1 -- 172.21.15.28:0/3511830063 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe60400ccc0 con 0x7fe60c10f0b0 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe608c00640 1 -- 172.21.15.28:0/3511830063 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fe60400ce60 con 0x7fe60c10f0b0 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe60c1a0580 con 0x7fe60c10f0b0 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe608c00640 1 -- 172.21.15.28:0/3511830063 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe604010430 con 0x7fe60c10f0b0 2024-05-20T20:19:23.751 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.750+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe60c1a7e10 con 0x7fe60c10f0b0 2024-05-20T20:19:23.752 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.752+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe60c069790 con 0x7fe60c10f0b0 2024-05-20T20:19:23.755 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.756+0000 7fe608c00640 1 -- 172.21.15.28:0/3511830063 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fe604004f70 con 0x7fe60c10f0b0 2024-05-20T20:19:23.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.757+0000 7fe608c00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe5f00b2960 0x7fe5f00b4e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:23.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.758+0000 7fe608c00640 1 -- 172.21.15.28:0/3511830063 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7fe6040e2bd0 con 0x7fe60c10f0b0 2024-05-20T20:19:23.757 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.758+0000 7fe60be00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe5f00b2960 0x7fe5f00b4e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:23.758 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.758+0000 7fe60be00640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe5f00b2960 0x7fe5f00b4e20 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fe5fc0099a0 tx=0x7fe5fc009500 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:23.759 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.760+0000 7fe608c00640 1 -- 172.21.15.28:0/3511830063 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fe604054df0 con 0x7fe60c10f0b0 2024-05-20T20:19:23.979 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.980+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fe60c19c5f0 con 0x7fe60c10f0b0 2024-05-20T20:19:23.982 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.983+0000 7fe608c00640 1 -- 172.21.15.28:0/3511830063 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v97) ==== 74+0+23038 (secure 0 0 0) 0x7fe60c19c5f0 con 0x7fe60c10f0b0 2024-05-20T20:19:23.983 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:19:23.983 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":97,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:19:15.740066+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"96","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":23,"snap_epoch":96,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6800/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/2759010565":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/1496067961":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1975849601":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/19609679":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":10,"length":1},{"begin":23,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe5f00b2960 msgr2=0x7fe5f00b4e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe5f00b2960 0x7fe5f00b4e20 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fe5fc0099a0 tx=0x7fe5fc009500 comp rx=0 tx=0).stop 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c10f0b0 msgr2=0x7fe60c19fb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c10f0b0 0x7fe60c19fb60 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7fe60400e970 tx=0x7fe60400ed70 comp rx=0 tx=0).stop 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 shutdown_connections 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fe60c103750 0x7fe60c19b7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fe5f00b2960 0x7fe5f00b4e20 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fe60c10f0b0 0x7fe60c19fb60 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.986 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 --2- 172.21.15.28:0/3511830063 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe60c06a120 0x7fe60c19b2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:23.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 >> 172.21.15.28:0/3511830063 conn(0x7fe60c077290 msgr2=0x7fe60c100a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:23.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 shutdown_connections 2024-05-20T20:19:23.987 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:23.986+0000 7fe611400640 1 -- 172.21.15.28:0/3511830063 wait complete. 2024-05-20T20:19:24.007 INFO:tasks.check_counter.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-20T19:55:47.376785+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-05-20T20:01:17.448722+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '78', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-05-20T20:01:22.420379+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '96', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 23, 'snap_epoch': 96, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_type': 'Fair distribution', 'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-05-20T20:19:24.007 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-05-20T20:19:24.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 -- 172.21.15.28:0/303379769 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 msgr2=0x7f487c111ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.203 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 --2- 172.21.15.28:0/303379769 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c111ba0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7f4874009860 tx=0x7f487402a340 comp rx=0 tx=0).stop 2024-05-20T20:19:24.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 -- 172.21.15.28:0/303379769 shutdown_connections 2024-05-20T20:19:24.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 --2- 172.21.15.28:0/303379769 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f487c1058b0 0x7f487c1098f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 --2- 172.21.15.28:0/303379769 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c111ba0 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 --2- 172.21.15.28:0/303379769 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 0x7f487c1052e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 -- 172.21.15.28:0/303379769 >> 172.21.15.28:0/303379769 conn(0x7f487c1007e0 msgr2=0x7f487c102c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:24.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 -- 172.21.15.28:0/303379769 shutdown_connections 2024-05-20T20:19:24.204 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.203+0000 7f4884000640 1 -- 172.21.15.28:0/303379769 wait complete. 2024-05-20T20:19:24.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.205+0000 7f4884000640 1 Processor -- start 2024-05-20T20:19:24.205 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.205+0000 7f4884000640 1 -- start start 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4884000640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 0x7f487c19b4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4884000640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c19ba10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4884000640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f487c19fd90 0x7f487c1a0240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4884000640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f487c114f30 con 0x7f487c109fd0 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4884000640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f487c114db0 con 0x7f487c19fd90 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4884000640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f487c1150b0 con 0x7f487c104f00 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4880c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c19ba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4881600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 0x7f487c19b4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4880c00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c19ba10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53226/0 (socket says 172.21.15.28:53226) 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4881600640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 0x7f487c19b4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56554/0 (socket says 172.21.15.28:56554) 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4880c00640 1 -- 172.21.15.28:0/1125765150 learned_addr learned my addr 172.21.15.28:0/1125765150 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:24.206 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.206+0000 7f4882000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f487c19fd90 0x7f487c1a0240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.207 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4881600640 1 -- 172.21.15.28:0/1125765150 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f487c19fd90 msgr2=0x7f487c1a0240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4881600640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f487c19fd90 0x7f487c1a0240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4881600640 1 -- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 msgr2=0x7f487c19ba10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4881600640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c19ba10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4881600640 1 -- 172.21.15.28:0/1125765150 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f487c1a0880 con 0x7f487c104f00 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4880c00640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c19ba10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4881600640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 0x7f487c19b4d0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f4870009fd0 tx=0x7f487000c880 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f487a000640 1 -- 172.21.15.28:0/1125765150 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4870003970 con 0x7f487c104f00 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f487c1a0b70 con 0x7f487c104f00 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.208+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f487c1a8360 con 0x7f487c104f00 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.207+0000 7f487a000640 1 -- 172.21.15.28:0/1125765150 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f4870004020 con 0x7f487c104f00 2024-05-20T20:19:24.208 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.208+0000 7f487a000640 1 -- 172.21.15.28:0/1125765150 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f487000b9b0 con 0x7f487c104f00 2024-05-20T20:19:24.209 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.209+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f485c005290 con 0x7f487c104f00 2024-05-20T20:19:24.211 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.211+0000 7f487a000640 1 -- 172.21.15.28:0/1125765150 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f487000bb50 con 0x7f487c104f00 2024-05-20T20:19:24.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.214+0000 7f487a000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f48640b2960 0x7f48640b4e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.213 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.214+0000 7f487a000640 1 -- 172.21.15.28:0/1125765150 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7f48700e2d50 con 0x7f487c104f00 2024-05-20T20:19:24.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.214+0000 7f4880c00640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f48640b2960 0x7f48640b4e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.214 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.215+0000 7f487a000640 1 -- 172.21.15.28:0/1125765150 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f4870024020 con 0x7f487c104f00 2024-05-20T20:19:24.215 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.216+0000 7f4880c00640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f48640b2960 0x7f48640b4e20 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f487c103f10 tx=0x7f4868005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:24.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.427+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) -- 0x7f485c005850 con 0x7f487c104f00 2024-05-20T20:19:24.427 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.428+0000 7f487a000640 1 -- 172.21.15.28:0/1125765150 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v97) ==== 93+0+10 (secure 0 0 0) 0x7f48700aa6c0 con 0x7f487c104f00 2024-05-20T20:19:24.428 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 1 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f48640b2960 msgr2=0x7f48640b4e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f48640b2960 0x7f48640b4e20 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f487c103f10 tx=0x7f4868005f90 comp rx=0 tx=0).stop 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 msgr2=0x7f487c19b4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 0x7f487c19b4d0 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f4870009fd0 tx=0x7f487000c880 comp rx=0 tx=0).stop 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 shutdown_connections 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f487c19fd90 0x7f487c1a0240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f48640b2960 0x7f48640b4e20 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f487c109fd0 0x7f487c19ba10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 --2- 172.21.15.28:0/1125765150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f487c104f00 0x7f487c19b4d0 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 >> 172.21.15.28:0/1125765150 conn(0x7f487c1007e0 msgr2=0x7f487c101af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:24.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 shutdown_connections 2024-05-20T20:19:24.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.431+0000 7f4884000640 1 -- 172.21.15.28:0/1125765150 wait complete. 2024-05-20T20:19:24.457 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-05-20T20:19:24.553 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:24 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/3511830063' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:19:24.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:24 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/3511830063' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:19:24.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:24 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/3511830063' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:19:24.649 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 -- 172.21.15.28:0/1916042976 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c0103b20 msgr2=0x7f12c0103f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1916042976 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c0103b20 0x7f12c0103f00 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7f12b0009ca0 tx=0x7f12b002a950 comp rx=0 tx=0).stop 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 -- 172.21.15.28:0/1916042976 shutdown_connections 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1916042976 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0104440 0x7f12c010c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1916042976 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f12c010cf40 0x7f12c010f3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1916042976 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c0103b20 0x7f12c0103f00 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 -- 172.21.15.28:0/1916042976 >> 172.21.15.28:0/1916042976 conn(0x7f12c00fd340 msgr2=0x7f12c00ff760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 -- 172.21.15.28:0/1916042976 shutdown_connections 2024-05-20T20:19:24.650 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.650+0000 7f12c5e00640 1 -- 172.21.15.28:0/1916042976 wait complete. 2024-05-20T20:19:24.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.651+0000 7f12c5e00640 1 Processor -- start 2024-05-20T20:19:24.651 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.651+0000 7f12c5e00640 1 -- start start 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c5e00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0103b20 0x7f12c019b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c5e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f12c0104440 0x7f12c019b920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c5e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c010cf40 0x7f12c019fca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c5e00640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f12c0112c40 con 0x7f12c0104440 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c5e00640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f12c0112ac0 con 0x7f12c0103b20 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c5e00640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f12c0112dc0 con 0x7f12c010cf40 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12bfe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f12c0104440 0x7f12c019b920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c4a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0103b20 0x7f12c019b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c4a00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0103b20 0x7f12c019b3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:55476/0 (socket says 172.21.15.28:55476) 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12bfe00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f12c0104440 0x7f12c019b920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53240/0 (socket says 172.21.15.28:53240) 2024-05-20T20:19:24.652 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c4a00640 1 -- 172.21.15.28:0/1911062518 learned_addr learned my addr 172.21.15.28:0/1911062518 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:24.653 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.652+0000 7f12c5400640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c010cf40 0x7f12c019fca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.653+0000 7f12c4a00640 1 -- 172.21.15.28:0/1911062518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c010cf40 msgr2=0x7f12c019fca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.653+0000 7f12c4a00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c010cf40 0x7f12c019fca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.653+0000 7f12c4a00640 1 -- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f12c0104440 msgr2=0x7f12c019b920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.653+0000 7f12c4a00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f12c0104440 0x7f12c019b920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.653+0000 7f12c4a00640 1 -- 172.21.15.28:0/1911062518 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f12c01a03d0 con 0x7f12c0103b20 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.653+0000 7f12c5400640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c010cf40 0x7f12c019fca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.653+0000 7f12c4a00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0103b20 0x7f12c019b3e0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f12b0005e30 tx=0x7f12b0003d60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.654+0000 7f12bd600640 1 -- 172.21.15.28:0/1911062518 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12b0034040 con 0x7f12c0103b20 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.654+0000 7f12bd600640 1 -- 172.21.15.28:0/1911062518 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f12b0030850 con 0x7f12c0103b20 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.654+0000 7f12bd600640 1 -- 172.21.15.28:0/1911062518 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12b00045e0 con 0x7f12c0103b20 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.654+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f12c01a0660 con 0x7f12c0103b20 2024-05-20T20:19:24.654 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.654+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f12c01a7f50 con 0x7f12c0103b20 2024-05-20T20:19:24.655 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.656+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f12c0069be0 con 0x7f12c0103b20 2024-05-20T20:19:24.659 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.659+0000 7f12bd600640 1 -- 172.21.15.28:0/1911062518 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f12b0037020 con 0x7f12c0103b20 2024-05-20T20:19:24.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.661+0000 7f12bd600640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f12a40b29e0 0x7f12a40b4ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:24.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.661+0000 7f12bd600640 1 -- 172.21.15.28:0/1911062518 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7f12b0103300 con 0x7f12c0103b20 2024-05-20T20:19:24.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.661+0000 7f12bfe00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f12a40b29e0 0x7f12a40b4ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:24.661 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.662+0000 7f12bfe00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f12a40b29e0 0x7f12a40b4ea0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f12c00ff780 tx=0x7f12ac005f90 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:24.665 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.665+0000 7f12bd600640 1 -- 172.21.15.28:0/1911062518 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f12b00cad80 con 0x7f12c0103b20 2024-05-20T20:19:24.878 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.878+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) -- 0x7f12c01a8120 con 0x7f12c0103b20 2024-05-20T20:19:24.879 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.878+0000 7f12bd600640 1 -- 172.21.15.28:0/1911062518 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v97) ==== 104+0+11 (secure 0 0 0) 0x7f12b00ca680 con 0x7f12c0103b20 2024-05-20T20:19:24.879 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 32 2024-05-20T20:19:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f12a40b29e0 msgr2=0x7f12a40b4ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f12a40b29e0 0x7f12a40b4ea0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f12c00ff780 tx=0x7f12ac005f90 comp rx=0 tx=0).stop 2024-05-20T20:19:24.881 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0103b20 msgr2=0x7f12c019b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0103b20 0x7f12c019b3e0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f12b0005e30 tx=0x7f12b0003d60 comp rx=0 tx=0).stop 2024-05-20T20:19:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 shutdown_connections 2024-05-20T20:19:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f12a40b29e0 0x7f12a40b4ea0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12c010cf40 0x7f12c019fca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.882 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f12c0104440 0x7f12c019b920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 --2- 172.21.15.28:0/1911062518 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f12c0103b20 0x7f12c019b3e0 unknown :-1 s=CLOSED pgs=634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 >> 172.21.15.28:0/1911062518 conn(0x7f12c00fd340 msgr2=0x7f12c01028d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.881+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 shutdown_connections 2024-05-20T20:19:24.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:24.882+0000 7f12c5e00640 1 -- 172.21.15.28:0/1911062518 wait complete. 2024-05-20T20:19:24.901 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-05-20T20:19:25.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 -- 172.21.15.28:0/3989260574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 msgr2=0x7f01b8102d70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 --2- 172.21.15.28:0/3989260574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 0x7f01b8102d70 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f01a4009900 tx=0x7f01a402a3f0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 -- 172.21.15.28:0/3989260574 shutdown_connections 2024-05-20T20:19:25.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 --2- 172.21.15.28:0/3989260574 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 0x7f01b8102d70 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 --2- 172.21.15.28:0/3989260574 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01b81032b0 0x7f01b810f7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 --2- 172.21.15.28:0/3989260574 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01b8108900 0x7f01b8108ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.089 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 -- 172.21.15.28:0/3989260574 >> 172.21.15.28:0/3989260574 conn(0x7f01b80fe420 msgr2=0x7f01b8100840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:25.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 -- 172.21.15.28:0/3989260574 shutdown_connections 2024-05-20T20:19:25.090 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.088+0000 7f01be200640 1 -- 172.21.15.28:0/3989260574 wait complete. 2024-05-20T20:19:25.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.089+0000 7f01be200640 1 Processor -- start 2024-05-20T20:19:25.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.089+0000 7f01be200640 1 -- start start 2024-05-20T20:19:25.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01be200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 0x7f01b8074d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01be200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01b81032b0 0x7f01b8075290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.091 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01be200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01b8108900 0x7f01b80757d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01be200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f01b8112bc0 con 0x7f01b81032b0 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01be200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f01b8112a40 con 0x7f01b81028f0 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01be200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f01b8112d40 con 0x7f01b8108900 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01b7e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01b81032b0 0x7f01b8075290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bce00640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 0x7f01b8074d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01b7e00640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01b81032b0 0x7f01b8075290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:53258/0 (socket says 172.21.15.28:53258) 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01b7e00640 1 -- 172.21.15.28:0/2679046073 learned_addr learned my addr 172.21.15.28:0/2679046073 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bd800640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01b8108900 0x7f01b80757d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bce00640 1 -- 172.21.15.28:0/2679046073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01b8108900 msgr2=0x7f01b80757d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bce00640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01b8108900 0x7f01b80757d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bce00640 1 -- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01b81032b0 msgr2=0x7f01b8075290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bce00640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01b81032b0 0x7f01b8075290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bce00640 1 -- 172.21.15.28:0/2679046073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f01b81a75c0 con 0x7f01b81028f0 2024-05-20T20:19:25.092 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.090+0000 7f01bd800640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01b8108900 0x7f01b80757d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:19:25.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.091+0000 7f01bce00640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 0x7f01b8074d50 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f01ac00e9f0 tx=0x7f01ac00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:25.093 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.091+0000 7f01b5600640 1 -- 172.21.15.28:0/2679046073 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01ac00ccc0 con 0x7f01b81028f0 2024-05-20T20:19:25.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.091+0000 7f01b5600640 1 -- 172.21.15.28:0/2679046073 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f01ac0108e0 con 0x7f01b81028f0 2024-05-20T20:19:25.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.091+0000 7f01b5600640 1 -- 172.21.15.28:0/2679046073 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f01ac018880 con 0x7f01b81028f0 2024-05-20T20:19:25.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.091+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f01b81a78b0 con 0x7f01b81028f0 2024-05-20T20:19:25.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.092+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f01b81a7db0 con 0x7f01b81028f0 2024-05-20T20:19:25.094 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.093+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f01b8069790 con 0x7f01b81028f0 2024-05-20T20:19:25.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.095+0000 7f01b5600640 1 -- 172.21.15.28:0/2679046073 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f01ac010430 con 0x7f01b81028f0 2024-05-20T20:19:25.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.096+0000 7f01b5600640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a00b2990 0x7f01a00b4e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.097+0000 7f01b5600640 1 -- 172.21.15.28:0/2679046073 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7f01ac0e2e10 con 0x7f01b81028f0 2024-05-20T20:19:25.097 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.097+0000 7f01b7e00640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a00b2990 0x7f01a00b4e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.097+0000 7f01b7e00640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a00b2990 0x7f01a00b4e50 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f01a4005ec0 tx=0x7f01a4005a20 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:25.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.099+0000 7f01b5600640 1 -- 172.21.15.28:0/2679046073 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f01ac0aa780 con 0x7f01b81028f0 2024-05-20T20:19:25.309 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.309+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) -- 0x7f01b810ae80 con 0x7f01b81028f0 2024-05-20T20:19:25.310 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.310+0000 7f01b5600640 1 -- 172.21.15.28:0/2679046073 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v97) ==== 100+0+11 (secure 0 0 0) 0x7f01ac0aa080 con 0x7f01b81028f0 2024-05-20T20:19:25.310 INFO:teuthology.orchestra.run.smithi028.stdout:pg_num: 64 2024-05-20T20:19:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a00b2990 msgr2=0x7f01a00b4e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a00b2990 0x7f01a00b4e50 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f01a4005ec0 tx=0x7f01a4005a20 comp rx=0 tx=0).stop 2024-05-20T20:19:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 msgr2=0x7f01b8074d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.313 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 0x7f01b8074d50 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f01ac00e9f0 tx=0x7f01ac00eeb0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 shutdown_connections 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f01a00b2990 0x7f01a00b4e50 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f01b8108900 0x7f01b80757d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f01b81032b0 0x7f01b8075290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 --2- 172.21.15.28:0/2679046073 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f01b81028f0 0x7f01b8074d50 unknown :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.312+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 >> 172.21.15.28:0/2679046073 conn(0x7f01b80fe420 msgr2=0x7f01b80fec40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.313+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 shutdown_connections 2024-05-20T20:19:25.314 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.313+0000 7f01be200640 1 -- 172.21.15.28:0/2679046073 wait complete. 2024-05-20T20:19:25.333 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json-pretty 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.524+0000 7fca62200640 1 -- 172.21.15.28:0/1006924450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 msgr2=0x7fca5c10f700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.524+0000 7fca62200640 1 --2- 172.21.15.28:0/1006924450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 0x7fca5c10f700 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7fca54009860 tx=0x7fca5402a340 comp rx=0 tx=0).stop 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.525+0000 7fca62200640 1 -- 172.21.15.28:0/1006924450 shutdown_connections 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.525+0000 7fca62200640 1 --2- 172.21.15.28:0/1006924450 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 0x7fca5c10f700 unknown :-1 s=CLOSED pgs=637 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.525+0000 7fca62200640 1 --2- 172.21.15.28:0/1006924450 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca5c102830 0x7fca5c102cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.525+0000 7fca62200640 1 --2- 172.21.15.28:0/1006924450 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c108840 0x7fca5c108c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.525+0000 7fca62200640 1 -- 172.21.15.28:0/1006924450 >> 172.21.15.28:0/1006924450 conn(0x7fca5c0fe380 msgr2=0x7fca5c1007a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.525+0000 7fca62200640 1 -- 172.21.15.28:0/1006924450 shutdown_connections 2024-05-20T20:19:25.526 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.525+0000 7fca62200640 1 -- 172.21.15.28:0/1006924450 wait complete. 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.526+0000 7fca62200640 1 Processor -- start 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.526+0000 7fca62200640 1 -- start start 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca62200640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c102830 0x7fca5c19b200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca62200640 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 0x7fca5c19b740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca62200640 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca5c108840 0x7fca5c19fac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca62200640 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fca5c112b00 con 0x7fca5c108840 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca62200640 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fca5c112980 con 0x7fca5c1031f0 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca62200640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fca5c112c80 con 0x7fca5c102830 2024-05-20T20:19:25.528 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c102830 0x7fca5c19b200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c102830 0x7fca5c19b200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.28:56598/0 (socket says 172.21.15.28:56598) 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 -- 172.21.15.28:0/336399207 learned_addr learned my addr 172.21.15.28:0/336399207 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca61800640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca5c108840 0x7fca5c19fac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca5be00640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 0x7fca5c19b740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 -- 172.21.15.28:0/336399207 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 msgr2=0x7fca5c19b740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 0x7fca5c19b740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 -- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca5c108840 msgr2=0x7fca5c19fac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca5c108840 0x7fca5c19fac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.527+0000 7fca60e00640 1 -- 172.21.15.28:0/336399207 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca5c1a01f0 con 0x7fca5c102830 2024-05-20T20:19:25.529 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.528+0000 7fca60e00640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c102830 0x7fca5c19b200 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7fca4c00eb90 tx=0x7fca4c00c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:25.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.528+0000 7fca59600640 1 -- 172.21.15.28:0/336399207 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca4c004d20 con 0x7fca5c102830 2024-05-20T20:19:25.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.528+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca5c1a04e0 con 0x7fca5c102830 2024-05-20T20:19:25.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.528+0000 7fca59600640 1 -- 172.21.15.28:0/336399207 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fca4c00ee90 con 0x7fca5c102830 2024-05-20T20:19:25.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.528+0000 7fca59600640 1 -- 172.21.15.28:0/336399207 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca4c010670 con 0x7fca5c102830 2024-05-20T20:19:25.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.528+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca5c1a7d70 con 0x7fca5c102830 2024-05-20T20:19:25.530 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.529+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fca5c069be0 con 0x7fca5c102830 2024-05-20T20:19:25.532 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.532+0000 7fca59600640 1 -- 172.21.15.28:0/336399207 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fca4c010830 con 0x7fca5c102830 2024-05-20T20:19:25.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.533+0000 7fca59600640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fca400b29e0 0x7fca400b4ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:19:25.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.533+0000 7fca59600640 1 -- 172.21.15.28:0/336399207 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(97..97 src has 1..97) ==== 8542+0+0 (secure 0 0 0) 0x7fca4c0e2dc0 con 0x7fca5c102830 2024-05-20T20:19:25.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.533+0000 7fca5be00640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fca400b29e0 0x7fca400b4ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:19:25.534 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.534+0000 7fca5be00640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fca400b29e0 0x7fca400b4ea0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fca5c076ce0 tx=0x7fca48009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:19:25.535 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.535+0000 7fca59600640 1 -- 172.21.15.28:0/336399207 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fca4c054df0 con 0x7fca5c102830 2024-05-20T20:19:25.553 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:25 smithi028 ceph-mon[31178]: pgmap v432: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.3 KiB/s wr, 10 op/s 2024-05-20T20:19:25.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:25 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1125765150' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:19:25.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:25 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1911062518' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-05-20T20:19:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:25 smithi154 ceph-mon[36314]: pgmap v432: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.3 KiB/s wr, 10 op/s 2024-05-20T20:19:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:25 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1125765150' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:19:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:25 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1911062518' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-05-20T20:19:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:25 smithi033 ceph-mon[36179]: pgmap v432: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 1.3 KiB/s wr, 10 op/s 2024-05-20T20:19:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:25 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1125765150' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T20:19:25.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:25 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1911062518' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-05-20T20:19:25.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.793+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json-pretty"} v 0) -- 0x7fca5c19c550 con 0x7fca5c102830 2024-05-20T20:19:25.805 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.804+0000 7fca59600640 1 -- 172.21.15.28:0/336399207 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json-pretty"}]=0 v27) ==== 81+0+816174 (secure 0 0 0) 0x7fca5c19c550 con 0x7fca5c102830 2024-05-20T20:19:25.818 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:19:25.818 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "epoch": 27, 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "active_gid": 25120, 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "active_name": "x", 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "active_addrs": { 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "addrvec": [ 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v2", 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:6800", 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 2022328614 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.819 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v1", 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:6801", 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 2022328614 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: "active_addr": "172.21.15.28:6801/2022328614", 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: "active_change": "2024-05-20T20:05:37.589333+0000", 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: "active_mgr_features": 4540701547738038271, 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: "available": true, 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: "standbys": [ 2024-05-20T20:19:25.820 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "gid": 25111, 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "z", 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr_features": 4540701547738038271, 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "available_modules": [ 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "alerts", 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.821 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How frequently to reexamine health status", 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.822 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.823 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:25.824 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.825 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.826 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.827 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.828 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_destination": { 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_destination", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Email address to send alerts to", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.829 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_from_name": { 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_from_name", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "Ceph", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Email From: name", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.830 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.831 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.831 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.831 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_host": { 2024-05-20T20:19:25.831 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_host", 2024-05-20T20:19:25.832 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.832 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.832 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.832 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.832 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.832 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP server", 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_password": { 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_password", 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.833 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Password to authenticate with", 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_port": { 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_port", 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.834 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "465", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP port", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_sender": { 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_sender", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.835 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP envelope sender", 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_ssl": { 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_ssl", 2024-05-20T20:19:25.836 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use SSL to connect to SMTP server", 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_user": { 2024-05-20T20:19:25.837 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_user", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "User to authenticate as", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.838 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.839 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "balancer", 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "active": { 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "active", 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.840 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically balance PGs across cluster", 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "begin_time": { 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "begin_time", 2024-05-20T20:19:25.841 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0000", 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "beginning time of day to automatically balance", 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "begin_weekday": { 2024-05-20T20:19:25.842 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "begin_weekday", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "6", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.843 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_max_iterations": { 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_max_iterations", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "25", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "250", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.844 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_metrics": { 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_metrics", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "pgs,objects,bytes", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.845 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_step": { 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_step", 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0.5", 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0.001", 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "0.999", 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "aggressiveness of optimization", 2024-05-20T20:19:25.846 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "end_time": { 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "end_time", 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "2359", 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.847 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "ending time of day to automatically balance", 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "end_weekday": { 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "end_weekday", 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.848 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "6", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.849 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.850 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.851 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.852 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.853 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "min_score": { 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "min_score", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:25.854 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": { 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mode", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.855 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "upmap", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "crush-compat", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "none", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "read", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap-read" 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Balancer mode", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.856 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "pool_ids": { 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pool_ids", 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-05-20T20:19:25.857 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.858 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.859 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap_max_deviation": { 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "upmap_max_deviation", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.860 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "deviation below which no optimization is attempted", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap_max_optimizations": { 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "upmap_max_optimizations", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.861 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cephadm", 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:25.862 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_down_multiplier": { 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_down_multiplier", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3.0", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.863 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_refresh_rate": { 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_refresh_rate", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "20", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.864 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_starting_port": { 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_starting_port", 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "4721", 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-05-20T20:19:25.865 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_ptrace": { 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_ptrace", 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.866 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "autotune_interval": { 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "autotune_interval", 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:25.867 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to autotune daemon memory", 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "autotune_memory_target_ratio": { 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "autotune_memory_target_ratio", 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.868 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0.7", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm_log_destination": { 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cephadm_log_destination", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.869 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "file", 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "file,syslog", 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "syslog" 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.870 INFO:teuthology.orchestra.run.smithi028.stdout: "cgroups_split": { 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cgroups_split", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.871 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "config_checks_enabled": { 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_checks_enabled", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.872 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "config_dashboard": { 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_dashboard", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.873 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_alertmanager": { 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_alertmanager", 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:25.874 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_base": { 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_base", 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/ceph", 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.875 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Container image name, without the tag", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_elasticsearch": { 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_elasticsearch", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.876 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "elasticsearch container image", 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_grafana": { 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_grafana", 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/grafana:9.4.12", 2024-05-20T20:19:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_haproxy": { 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_haproxy", 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.878 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "HAproxy container image", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_agent": { 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_agent", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.879 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger agent container image", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_collector": { 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_collector", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.880 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger collector container image", 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_query": { 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_query", 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.881 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger query container image", 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_keepalived": { 2024-05-20T20:19:25.882 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_keepalived", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Keepalived container image", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.883 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_loki": { 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_loki", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/grafana/loki:3.0.0", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Loki container image", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.884 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_node_exporter": { 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_node_exporter", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.885 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_nvmeof": { 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_nvmeof", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/nvmeof:1.2.5", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Nvme-of container image", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.886 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_prometheus": { 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_prometheus", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.887 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_promtail": { 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_promtail", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/grafana/promtail:3.0.0", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Promtail container image", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.888 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_samba": { 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_samba", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/samba.org/samba-server:devbuilds-centos-amd64", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Samba/SMB container image", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.889 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_snmp_gateway": { 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_snmp_gateway", 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SNMP Gateway container image", 2024-05-20T20:19:25.890 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "container_init": { 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_init", 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.891 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Run podman/docker with `--init`", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "daemon_cache_timeout": { 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "daemon_cache_timeout", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.892 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "default_cephadm_command_timeout": { 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "default_cephadm_command_timeout", 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "900", 2024-05-20T20:19:25.893 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "default_registry": { 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "default_registry", 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.894 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io", 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "device_cache_timeout": { 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_cache_timeout", 2024-05-20T20:19:25.895 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1800", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache device inventory", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "device_enhanced_scan": { 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_enhanced_scan", 2024-05-20T20:19:25.896 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use libstoragemgmt during device scans", 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "facts_cache_timeout": { 2024-05-20T20:19:25.897 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "facts_cache_timeout", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache host facts data", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.898 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "grafana_dashboards_path": { 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "grafana_dashboards_path", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/etc/grafana/dashboards/ceph-dashboard/", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "location of dashboards to include in grafana deployments", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.899 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "host_check_interval": { 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "host_check_interval", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to perform a host check", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.900 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "hw_monitoring": { 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "hw_monitoring", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.901 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "inventory_list_all": { 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "inventory_list_all", 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.902 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.903 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.904 INFO:teuthology.orchestra.run.smithi028.stdout: "log_refresh_metadata": { 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_refresh_metadata", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.905 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.906 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.907 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.908 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "manage_etc_ceph_ceph_conf": { 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-05-20T20:19:25.909 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-05-20T20:19:25.910 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "*", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "max_count_per_host": { 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_count_per_host", 2024-05-20T20:19:25.911 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "max number of daemons per service per host", 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "max_osd_draining_count": { 2024-05-20T20:19:25.912 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_osd_draining_count", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "migration_current": { 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "migration_current", 2024-05-20T20:19:25.913 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "internal - do not modify", 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": { 2024-05-20T20:19:25.914 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mode", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "root", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm-package", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "root" 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "mode for remote execution of cephadm", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.915 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "oob_default_addr": { 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "oob_default_addr", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "169.254.1.1", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Default address for RedFish API (oob management).", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.916 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "prometheus_alerts_path": { 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prometheus_alerts_path", 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.917 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_insecure": { 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_insecure", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.918 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_password": { 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_password", 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.919 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_url": { 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_url", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.920 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_username": { 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_username", 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.921 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "secure_monitoring_stack": { 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "secure_monitoring_stack", 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.922 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "service_discovery_port": { 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "service_discovery_port", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.923 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8765", 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "cephadm service discovery port", 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.924 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "ssh_config_file": { 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssh_config_file", 2024-05-20T20:19:25.925 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "use_agent": { 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "use_agent", 2024-05-20T20:19:25.926 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "use_repo_digest": { 2024-05-20T20:19:25.927 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "use_repo_digest", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_failed_host_check": { 2024-05-20T20:19:25.928 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_failed_host_check", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if the host check fails", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.929 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_stray_daemons": { 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_stray_daemons", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.930 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_stray_hosts": { 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_stray_hosts", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.931 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crash", 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.932 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.933 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.934 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.935 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:25.936 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.937 INFO:teuthology.orchestra.run.smithi028.stdout: "retain_interval": { 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "retain_interval", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "31536000", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long to retain crashes before pruning them", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.938 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.939 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_recent_interval": { 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_recent_interval", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1209600", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "time interval in which to warn about recent crashes", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.940 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "dashboard", 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.941 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "ALERTMANAGER_API_HOST": { 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ALERTMANAGER_API_HOST", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.942 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.943 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "AUDIT_API_ENABLED": { 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "AUDIT_API_ENABLED", 2024-05-20T20:19:25.944 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-05-20T20:19:25.945 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "ENABLE_BROWSABLE_API": { 2024-05-20T20:19:25.946 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ENABLE_BROWSABLE_API", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.947 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.948 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.949 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.950 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.951 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_NFS": { 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.952 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_RBD": { 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.953 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_RGW": { 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.954 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.955 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_PASSWORD": { 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_PASSWORD", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.956 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.957 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_URL": { 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_URL", 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.958 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_USERNAME": { 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_USERNAME", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.959 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.960 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.961 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-05-20T20:19:25.962 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "ISSUE_TRACKER_API_KEY": { 2024-05-20T20:19:25.963 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "MANAGED_BY_CLUSTERS": { 2024-05-20T20:19:25.964 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "MANAGED_BY_CLUSTERS", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "[]", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "MULTICLUSTER_CONFIG": { 2024-05-20T20:19:25.965 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "MULTICLUSTER_CONFIG", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "{}", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.966 INFO:teuthology.orchestra.run.smithi028.stdout: "PROMETHEUS_API_HOST": { 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PROMETHEUS_API_HOST", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.967 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.968 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.969 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.970 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.971 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.972 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.973 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.974 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.975 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_ENABLED": { 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_ENABLED", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.976 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-05-20T20:19:25.977 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:25.978 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8", 2024-05-20T20:19:25.979 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "REST_REQUESTS_TIMEOUT": { 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.980 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "45", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_ACCESS_KEY": { 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_ACCESS_KEY", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.981 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.982 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_SECRET_KEY": { 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_SECRET_KEY", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.983 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_SSL_VERIFY": { 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_SSL_VERIFY", 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.984 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "UNSAFE_TLS_v1_2": { 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "UNSAFE_TLS_v1_2", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.985 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-05-20T20:19:25.986 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.987 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.988 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.989 INFO:teuthology.orchestra.run.smithi028.stdout: "cross_origin_url": { 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cross_origin_url", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.990 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "crt_file": { 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crt_file", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.991 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "debug": { 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "debug", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable/disable debug options", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.992 INFO:teuthology.orchestra.run.smithi028.stdout: "jwt_token_ttl": { 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "jwt_token_ttl", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "28800", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.993 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "key_file": { 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "key_file", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.994 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.995 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.996 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.997 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:25.998 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:25.999 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "motd": { 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "motd", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.000 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "The message of the day", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "redirect_resolve_ip_addr": { 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "redirect_resolve_ip_addr", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.001 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "::", 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.002 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.003 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8080", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.004 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl": { 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.005 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl_server_port": { 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl_server_port", 2024-05-20T20:19:26.006 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8443", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_behaviour": { 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_behaviour", 2024-05-20T20:19:26.007 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "redirect", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "redirect" 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.008 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_error_status_code": { 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_error_status_code", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "500", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "400", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "599", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.009 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "url_prefix": { 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "url_prefix", 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.010 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "devicehealth", 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "enable_monitoring": { 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enable_monitoring", 2024-05-20T20:19:26.011 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "monitor device health metrics", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.012 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.013 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.014 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.015 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.016 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.017 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "mark_out_threshold": { 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mark_out_threshold", 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "2419200", 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-05-20T20:19:26.018 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "pool_name": { 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pool_name", 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "device_health_metrics", 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.019 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "name of pool in which to store device health metrics", 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "retention_period": { 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "retention_period", 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15552000", 2024-05-20T20:19:26.020 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long to retain device health metrics", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "scrape_frequency": { 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "scrape_frequency", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "86400", 2024-05-20T20:19:26.021 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to scrape device health metrics", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "self_heal": { 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "self_heal", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.022 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.023 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to wake up and check device health", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.024 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_threshold": { 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_threshold", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "7257600", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-05-20T20:19:26.025 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "diskprediction_local", 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.026 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.027 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.028 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.029 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.030 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "predict_interval": { 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "predict_interval", 2024-05-20T20:19:26.031 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "86400", 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "predictor_model": { 2024-05-20T20:19:26.032 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "predictor_model", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "prophetstor", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.033 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.034 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.035 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "influx", 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": false, 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "influxdb python module not found", 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "batch_size": { 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "batch_size", 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.036 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5000", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "database": { 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "database", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.037 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "ceph", 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "hostname": { 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "hostname", 2024-05-20T20:19:26.038 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB server hostname", 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.039 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "30", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "5", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.040 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.041 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.042 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.043 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.044 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.045 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "password": { 2024-05-20T20:19:26.046 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "password", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "password of InfluxDB server user", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.047 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "port": { 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "port", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8086", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB server port", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.048 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.049 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl": { 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "false", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.050 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "threads": { 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "threads", 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "32", 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-05-20T20:19:26.051 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "username": { 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "username", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "username of InfluxDB server user", 2024-05-20T20:19:26.052 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "verify_ssl": { 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "verify_ssl", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "true", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-05-20T20:19:26.053 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "insights", 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.054 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.055 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.056 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.057 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.058 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.059 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.060 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "iostat", 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.061 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.062 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.063 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.064 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.065 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.066 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.067 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "k8sevents", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_event_retention_days": { 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ceph_event_retention_days", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "7", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.068 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Days to hold ceph event information within local cache", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "config_check_secs": { 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_check_secs", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "10", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.069 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.070 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.071 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.072 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.073 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.074 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.075 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.076 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "localpool", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "failure_domain": { 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "failure_domain", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "host", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.077 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "failure domain for any created local pool", 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.078 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.079 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.080 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.081 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.082 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "min_size": { 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "min_size", 2024-05-20T20:19:26.083 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default min_size for any created local pool", 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "num_rep": { 2024-05-20T20:19:26.084 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "num_rep", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default replica count for any created local pool", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_num": { 2024-05-20T20:19:26.085 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pg_num", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "128", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default pg_num for any created local pool", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.086 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "prefix": { 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prefix", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "name prefix for any created local pool", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.087 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.088 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "subtree": { 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "subtree", 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "rack", 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.089 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "CRUSH level for which to create a local pool", 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mds_autoscaler", 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.090 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.091 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.092 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.093 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.094 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.095 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.096 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mirroring", 2024-05-20T20:19:26.097 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.098 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.099 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.100 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.101 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.102 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.103 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.104 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "nfs", 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.105 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.106 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.107 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.108 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.109 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.110 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.111 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.111 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.111 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.197 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "orchestrator", 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "fail_fs": { 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "fail_fs", 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.198 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.199 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.200 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.201 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.202 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.203 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator": { 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "orchestrator", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.204 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm", 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "rook", 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "test_orchestrator" 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Orchestrator backend", 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.205 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.206 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "osd_perf_query", 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.207 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.208 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.209 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.210 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.211 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.212 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "osd_support", 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.213 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.214 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.215 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.216 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.217 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.218 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.219 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pg_autoscaler", 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.220 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.221 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.222 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.223 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.224 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.225 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.226 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "threshold": { 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "threshold", 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.227 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3.0", 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1.0", 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "scaling threshold", 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.228 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "progress", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_pg_recovery_event": { 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_pg_recovery_event", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.229 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow the module to show pg recovery progress", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "enabled": { 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enabled", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.230 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.231 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.232 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.233 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.234 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.235 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "max_completed_events": { 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_completed_events", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "50", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "number of past completed events to remember", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.236 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long the module is going to sleep", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.237 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.238 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prometheus", 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "cache": { 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cache", 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.239 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "exclude_perf_counters": { 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "exclude_perf_counters", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.240 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.241 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.242 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.243 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.244 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.245 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_stats_pools": { 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_stats_pools", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.246 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_stats_pools_refresh_interval": { 2024-05-20T20:19:26.247 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "300", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "scrape_interval": { 2024-05-20T20:19:26.248 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "scrape_interval", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15.0", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.249 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "::", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.250 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "9283", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.251 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.252 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "stale_cache_strategy": { 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "stale_cache_strategy", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "log", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.253 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_behaviour": { 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_behaviour", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "default", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "default", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "error" 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.254 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_error_status_code": { 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_error_status_code", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "500", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "400", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "599", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.255 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_support", 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.256 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.257 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.258 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.259 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.260 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.261 INFO:teuthology.orchestra.run.smithi028.stdout: "max_concurrent_snap_create": { 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_concurrent_snap_create", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.262 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "mirror_snapshot_schedule": { 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mirror_snapshot_schedule", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.263 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.264 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "trash_purge_schedule": { 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "trash_purge_schedule", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.265 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "restful", 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "enable_auth": { 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enable_auth", 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.266 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "key_file": { 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "key_file", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.267 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.268 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.269 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.270 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.271 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.272 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.273 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.274 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.275 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rgw", 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.276 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.277 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.278 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.279 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.280 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.281 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.282 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rook", 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.283 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.284 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.285 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.286 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.287 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.288 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "storage_class": { 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "storage_class", 2024-05-20T20:19:26.289 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "local", 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.290 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "selftest", 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.291 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.292 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.293 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.294 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.295 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "roption1": { 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "roption1", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.296 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "roption2": { 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "roption2", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "xyz", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.297 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption1": { 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption1", 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.298 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption2": { 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption2", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.299 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption3": { 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption3", 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.300 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption4": { 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption4", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.301 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption5": { 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption5", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.302 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption6": { 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption6", 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.303 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption7": { 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption7", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.304 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "42", 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.305 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "testkey": { 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testkey", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.306 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "testlkey": { 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testlkey", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.307 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "testnewline": { 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testnewline", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.308 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smb", 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.309 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.310 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.311 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.312 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.313 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.314 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.315 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "update_orchestration": { 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "update_orchestration", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically update orchestration when smb resources are changed", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.316 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snap_schedule", 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_m_granularity": { 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_m_granularity", 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.317 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow minute scheduled snapshots", 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "dump_on_update": { 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "dump_on_update", 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.318 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "dump database to debug log on update", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.319 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.320 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.321 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.322 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.323 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.324 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "stats", 2024-05-20T20:19:26.325 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.326 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.330 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.331 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.332 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.333 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.334 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.335 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "status", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.336 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.337 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.338 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.339 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.340 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.341 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "telegraf", 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "address": { 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "address", 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.342 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.343 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.344 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.345 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.346 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.347 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.348 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.349 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "telemetry", 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.350 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_basic": { 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_basic", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share basic cluster information (size, version)", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.351 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_crash": { 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_crash", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.352 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_device": { 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_device", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.353 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_ident": { 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_ident", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.354 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_perf": { 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_perf", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share various performance metrics of a cluster", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.355 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "contact": { 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "contact", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.356 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "description": { 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "description", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.357 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "device_url": { 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_url", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.358 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "enabled": { 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enabled", 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.359 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "24", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "8", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.360 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "last_opt_revision": { 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "last_opt_revision", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.361 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "leaderboard": { 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "leaderboard", 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.362 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "leaderboard_description": { 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "leaderboard_description", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.363 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.364 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.365 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.366 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.367 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.368 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "organization": { 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "organization", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.369 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "proxy": { 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "proxy", 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.370 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.371 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "url": { 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "url", 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-05-20T20:19:26.372 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "test_orchestrator", 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.373 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.374 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.375 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.376 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.377 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.378 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.379 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "volumes", 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.380 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.381 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.382 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.383 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.384 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "max_concurrent_clones": { 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_concurrent_clones", 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.385 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "4", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Number of asynchronous cloner threads", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "periodic_async_work": { 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "periodic_async_work", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.386 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Periodically check for async work", 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "snapshot_clone_delay": { 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snapshot_clone_delay", 2024-05-20T20:19:26.387 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "snapshot_clone_no_wait": { 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snapshot_clone_no_wait", 2024-05-20T20:19:26.388 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.389 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.390 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "discovery_interval": { 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "discovery_interval", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "100", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.391 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "identifier": { 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "identifier", 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.392 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.393 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.394 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.395 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.396 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.397 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.398 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.399 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_host": { 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_host", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.400 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_port": { 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_port", 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10051", 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.401 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_sender": { 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_sender", 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-05-20T20:19:26.402 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.403 INFO:teuthology.orchestra.run.smithi028.stdout: "gid": 25133, 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "y", 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "mgr_features": 4540701547738038271, 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "available_modules": [ 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "alerts", 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.404 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How frequently to reexamine health status", 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.405 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.406 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.407 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.408 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.409 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_destination": { 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_destination", 2024-05-20T20:19:26.410 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Email address to send alerts to", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_from_name": { 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_from_name", 2024-05-20T20:19:26.411 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "Ceph", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Email From: name", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_host": { 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_host", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.412 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP server", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_password": { 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_password", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.413 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Password to authenticate with", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_port": { 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_port", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.414 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "465", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP port", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_sender": { 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_sender", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.415 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP envelope sender", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_ssl": { 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_ssl", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.416 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use SSL to connect to SMTP server", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_user": { 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_user", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.417 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "User to authenticate as", 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.418 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.419 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "balancer", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "active": { 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "active", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.420 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically balance PGs across cluster", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "begin_time": { 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "begin_time", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0000", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.421 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "beginning time of day to automatically balance", 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "begin_weekday": { 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "begin_weekday", 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.422 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "6", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_max_iterations": { 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_max_iterations", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "25", 2024-05-20T20:19:26.423 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "250", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_metrics": { 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_metrics", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "pgs,objects,bytes", 2024-05-20T20:19:26.424 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_step": { 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_step", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0.5", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0.001", 2024-05-20T20:19:26.425 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "0.999", 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "aggressiveness of optimization", 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "end_time": { 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "end_time", 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.426 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "2359", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "ending time of day to automatically balance", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "end_weekday": { 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "end_weekday", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.427 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "6", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.428 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.429 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.430 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.431 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.432 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "min_score": { 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "min_score", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.433 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": { 2024-05-20T20:19:26.434 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mode", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "upmap", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "crush-compat", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "none", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "read", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap", 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap-read" 2024-05-20T20:19:26.435 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Balancer mode", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "pool_ids": { 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pool_ids", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.436 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.437 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.438 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap_max_deviation": { 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "upmap_max_deviation", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.439 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "deviation below which no optimization is attempted", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap_max_optimizations": { 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "upmap_max_optimizations", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.440 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cephadm", 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_down_multiplier": { 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_down_multiplier", 2024-05-20T20:19:26.441 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3.0", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_refresh_rate": { 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_refresh_rate", 2024-05-20T20:19:26.442 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "20", 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_starting_port": { 2024-05-20T20:19:26.443 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_starting_port", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "4721", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_ptrace": { 2024-05-20T20:19:26.444 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_ptrace", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "autotune_interval": { 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "autotune_interval", 2024-05-20T20:19:26.445 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to autotune daemon memory", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "autotune_memory_target_ratio": { 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "autotune_memory_target_ratio", 2024-05-20T20:19:26.446 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0.7", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm_log_destination": { 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cephadm_log_destination", 2024-05-20T20:19:26.447 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "file", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "file,syslog", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "syslog" 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.448 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "cgroups_split": { 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cgroups_split", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.449 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "config_checks_enabled": { 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_checks_enabled", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.450 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "config_dashboard": { 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_dashboard", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.451 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_alertmanager": { 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_alertmanager", 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.452 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_base": { 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_base", 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/ceph", 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.453 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Container image name, without the tag", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_elasticsearch": { 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_elasticsearch", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.454 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "elasticsearch container image", 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_grafana": { 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_grafana", 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/grafana:9.4.12", 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.455 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_haproxy": { 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_haproxy", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.456 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "HAproxy container image", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_agent": { 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_agent", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.457 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger agent container image", 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_collector": { 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_collector", 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-05-20T20:19:26.458 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger collector container image", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_query": { 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_query", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-05-20T20:19:26.459 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger query container image", 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_keepalived": { 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_keepalived", 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.460 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Keepalived container image", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_loki": { 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_loki", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.461 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/grafana/loki:3.0.0", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Loki container image", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_node_exporter": { 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_node_exporter", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.462 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_nvmeof": { 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_nvmeof", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.463 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/nvmeof:1.2.5", 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Nvme-of container image", 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_prometheus": { 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_prometheus", 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.464 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_promtail": { 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_promtail", 2024-05-20T20:19:26.465 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/grafana/promtail:3.0.0", 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Promtail container image", 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_samba": { 2024-05-20T20:19:26.466 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_samba", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/samba.org/samba-server:devbuilds-centos-amd64", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Samba/SMB container image", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.467 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_snmp_gateway": { 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_snmp_gateway", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SNMP Gateway container image", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.468 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "container_init": { 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_init", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Run podman/docker with `--init`", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.469 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "daemon_cache_timeout": { 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "daemon_cache_timeout", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.470 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "default_cephadm_command_timeout": { 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "default_cephadm_command_timeout", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "900", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.471 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "default_registry": { 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "default_registry", 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io", 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-05-20T20:19:26.472 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "device_cache_timeout": { 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_cache_timeout", 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1800", 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.473 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache device inventory", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "device_enhanced_scan": { 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_enhanced_scan", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.474 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use libstoragemgmt during device scans", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "facts_cache_timeout": { 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "facts_cache_timeout", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.475 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache host facts data", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "grafana_dashboards_path": { 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "grafana_dashboards_path", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/etc/grafana/dashboards/ceph-dashboard/", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.476 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "location of dashboards to include in grafana deployments", 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "host_check_interval": { 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "host_check_interval", 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.477 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to perform a host check", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "hw_monitoring": { 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "hw_monitoring", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.478 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "inventory_list_all": { 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "inventory_list_all", 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.479 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.480 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.481 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "log_refresh_metadata": { 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_refresh_metadata", 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-05-20T20:19:26.482 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-05-20T20:19:26.483 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.484 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.485 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "manage_etc_ceph_ceph_conf": { 2024-05-20T20:19:26.486 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-05-20T20:19:26.487 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "*", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "max_count_per_host": { 2024-05-20T20:19:26.488 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_count_per_host", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "max number of daemons per service per host", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "max_osd_draining_count": { 2024-05-20T20:19:26.489 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_osd_draining_count", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.490 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "migration_current": { 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "migration_current", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "internal - do not modify", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.491 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": { 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mode", 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "root", 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm-package", 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "root" 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.492 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "mode for remote execution of cephadm", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "oob_default_addr": { 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "oob_default_addr", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "169.254.1.1", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.493 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Default address for RedFish API (oob management).", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "prometheus_alerts_path": { 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prometheus_alerts_path", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.494 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_insecure": { 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_insecure", 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.495 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_password": { 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_password", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.496 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_url": { 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_url", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.497 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_username": { 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_username", 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.498 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "secure_monitoring_stack": { 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "secure_monitoring_stack", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.499 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "service_discovery_port": { 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "service_discovery_port", 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8765", 2024-05-20T20:19:26.500 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "cephadm service discovery port", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.501 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "ssh_config_file": { 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssh_config_file", 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.502 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "use_agent": { 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "use_agent", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.503 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "use_repo_digest": { 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "use_repo_digest", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.504 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_failed_host_check": { 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_failed_host_check", 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.505 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if the host check fails", 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_stray_daemons": { 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_stray_daemons", 2024-05-20T20:19:26.506 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.507 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_stray_hosts": { 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_stray_hosts", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.508 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crash", 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.509 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.510 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.511 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.512 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.513 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "retain_interval": { 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "retain_interval", 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "31536000", 2024-05-20T20:19:26.514 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long to retain crashes before pruning them", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.515 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_recent_interval": { 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_recent_interval", 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.516 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1209600", 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "time interval in which to warn about recent crashes", 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "dashboard", 2024-05-20T20:19:26.517 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.518 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "ALERTMANAGER_API_HOST": { 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ALERTMANAGER_API_HOST", 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.519 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.520 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "AUDIT_API_ENABLED": { 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "AUDIT_API_ENABLED", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.521 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.522 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "ENABLE_BROWSABLE_API": { 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ENABLE_BROWSABLE_API", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.523 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.524 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.525 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.526 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.527 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_NFS": { 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.528 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_RBD": { 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.529 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_RGW": { 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.530 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.531 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_PASSWORD": { 2024-05-20T20:19:26.534 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_PASSWORD", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-05-20T20:19:26.535 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.536 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_URL": { 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_URL", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.537 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_USERNAME": { 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_USERNAME", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.538 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.539 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.540 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.541 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "ISSUE_TRACKER_API_KEY": { 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.542 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "MANAGED_BY_CLUSTERS": { 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "MANAGED_BY_CLUSTERS", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "[]", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.543 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "MULTICLUSTER_CONFIG": { 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "MULTICLUSTER_CONFIG", 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "{}", 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.544 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "PROMETHEUS_API_HOST": { 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PROMETHEUS_API_HOST", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.545 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.546 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.547 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.548 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.549 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.550 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.551 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.552 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.553 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_ENABLED": { 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_ENABLED", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.554 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.555 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.556 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.557 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8", 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "REST_REQUESTS_TIMEOUT": { 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-05-20T20:19:26.558 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "45", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_ACCESS_KEY": { 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_ACCESS_KEY", 2024-05-20T20:19:26.559 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-05-20T20:19:26.560 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_SECRET_KEY": { 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_SECRET_KEY", 2024-05-20T20:19:26.561 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_SSL_VERIFY": { 2024-05-20T20:19:26.562 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_SSL_VERIFY", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.563 INFO:teuthology.orchestra.run.smithi028.stdout: "UNSAFE_TLS_v1_2": { 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "UNSAFE_TLS_v1_2", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.564 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.565 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.566 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.567 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "cross_origin_url": { 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cross_origin_url", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.568 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "crt_file": { 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crt_file", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.569 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "debug": { 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "debug", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable/disable debug options", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.570 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "jwt_token_ttl": { 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "jwt_token_ttl", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "28800", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.571 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "key_file": { 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "key_file", 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.572 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.573 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.574 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.575 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.576 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.577 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "motd": { 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "motd", 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.578 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "The message of the day", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "redirect_resolve_ip_addr": { 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "redirect_resolve_ip_addr", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.579 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "::", 2024-05-20T20:19:26.580 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8080", 2024-05-20T20:19:26.581 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.582 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl": { 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.583 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl_server_port": { 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl_server_port", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.584 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8443", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_behaviour": { 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_behaviour", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.585 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "redirect", 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "redirect" 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_error_status_code": { 2024-05-20T20:19:26.586 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_error_status_code", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "500", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "400", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "599", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "url_prefix": { 2024-05-20T20:19:26.587 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "url_prefix", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.588 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "devicehealth", 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "enable_monitoring": { 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enable_monitoring", 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.589 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "monitor device health metrics", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.590 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.591 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.592 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.593 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.594 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "mark_out_threshold": { 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mark_out_threshold", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.595 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "2419200", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "pool_name": { 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pool_name", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.596 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "device_health_metrics", 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "name of pool in which to store device health metrics", 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "retention_period": { 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "retention_period", 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.597 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15552000", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long to retain device health metrics", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "scrape_frequency": { 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "scrape_frequency", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.598 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "86400", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to scrape device health metrics", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "self_heal": { 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "self_heal", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.599 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.600 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to wake up and check device health", 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.601 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.602 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_threshold": { 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_threshold", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "7257600", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-05-20T20:19:26.603 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "diskprediction_local", 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.604 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.605 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.606 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.607 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.608 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.609 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "predict_interval": { 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "predict_interval", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "86400", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.610 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "predictor_model": { 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "predictor_model", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "prophetstor", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.611 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.612 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.613 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "influx", 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": false, 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "influxdb python module not found", 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "batch_size": { 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "batch_size", 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.614 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5000", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "database": { 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "database", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.615 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "ceph", 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "hostname": { 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "hostname", 2024-05-20T20:19:26.616 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB server hostname", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.617 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "30", 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "5", 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.618 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.619 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.620 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.621 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.622 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.623 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "password": { 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "password", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "password of InfluxDB server user", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.624 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "port": { 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "port", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8086", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB server port", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.625 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.626 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl": { 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "false", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-05-20T20:19:26.627 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "threads": { 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "threads", 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "32", 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.628 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "username": { 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "username", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.629 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "username of InfluxDB server user", 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "verify_ssl": { 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "verify_ssl", 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "true", 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.630 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "insights", 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.631 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.632 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.633 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.634 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.635 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.636 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.637 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "iostat", 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.638 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.639 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.640 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.641 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.642 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.643 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "k8sevents", 2024-05-20T20:19:26.644 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_event_retention_days": { 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ceph_event_retention_days", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "7", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Days to hold ceph event information within local cache", 2024-05-20T20:19:26.645 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "config_check_secs": { 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_check_secs", 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "10", 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.646 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.647 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.648 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.649 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.650 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.651 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.652 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.653 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "localpool", 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "failure_domain": { 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "failure_domain", 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "host", 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.654 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "failure domain for any created local pool", 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.655 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.656 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.657 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.658 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.659 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "min_size": { 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "min_size", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.660 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default min_size for any created local pool", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "num_rep": { 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "num_rep", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.661 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default replica count for any created local pool", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_num": { 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pg_num", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.662 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "128", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default pg_num for any created local pool", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "prefix": { 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prefix", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.663 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "name prefix for any created local pool", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.664 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "subtree": { 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "subtree", 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.665 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "rack", 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "CRUSH level for which to create a local pool", 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.666 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mds_autoscaler", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.667 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.668 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.669 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.670 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.671 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.672 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mirroring", 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.673 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.674 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.675 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.676 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.677 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.678 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.679 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "nfs", 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.680 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.681 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.682 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.683 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.684 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.685 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.686 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "orchestrator", 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "fail_fs": { 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "fail_fs", 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.687 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.688 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.689 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.690 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.691 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.692 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator": { 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "orchestrator", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.693 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm", 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "rook", 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "test_orchestrator" 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Orchestrator backend", 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.694 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.695 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "osd_perf_query", 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.696 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.697 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.698 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.699 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.700 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.701 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "osd_support", 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.702 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.703 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.704 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.705 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.706 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.707 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.708 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pg_autoscaler", 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.709 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.710 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.711 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.712 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.713 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.714 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.715 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "threshold": { 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "threshold", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.716 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3.0", 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1.0", 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "scaling threshold", 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "progress", 2024-05-20T20:19:26.717 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_pg_recovery_event": { 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_pg_recovery_event", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow the module to show pg recovery progress", 2024-05-20T20:19:26.718 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "enabled": { 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enabled", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.719 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.720 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.721 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.722 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.723 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.724 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "max_completed_events": { 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_completed_events", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "50", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "number of past completed events to remember", 2024-05-20T20:19:26.725 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long the module is going to sleep", 2024-05-20T20:19:26.726 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.727 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prometheus", 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "cache": { 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cache", 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.728 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "exclude_perf_counters": { 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "exclude_perf_counters", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.729 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.730 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.731 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.732 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.733 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.734 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.735 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_stats_pools": { 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_stats_pools", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.736 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_stats_pools_refresh_interval": { 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "300", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.737 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "scrape_interval": { 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "scrape_interval", 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15.0", 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.738 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "::", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.739 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.740 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.740 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.740 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:26.740 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:26.740 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.740 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "9283", 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.742 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.743 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "stale_cache_strategy": { 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "stale_cache_strategy", 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "log", 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.744 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_behaviour": { 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_behaviour", 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "default", 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.745 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "default", 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "error" 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_error_status_code": { 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_error_status_code", 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.746 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "500", 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "400", 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "599", 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.747 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_support", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.748 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.749 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.750 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.751 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.752 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "max_concurrent_snap_create": { 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_concurrent_snap_create", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.753 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "mirror_snapshot_schedule": { 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mirror_snapshot_schedule", 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.754 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.755 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "trash_purge_schedule": { 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "trash_purge_schedule", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.756 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "restful", 2024-05-20T20:19:26.757 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "enable_auth": { 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enable_auth", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.758 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "key_file": { 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "key_file", 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.759 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.760 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.761 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.762 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.763 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.764 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.765 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.766 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.767 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rgw", 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.768 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.769 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.770 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.771 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.772 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.773 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.774 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rook", 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.775 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.776 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.777 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.778 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.779 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.780 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.781 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "storage_class": { 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "storage_class", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "local", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.782 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "selftest", 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.783 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.784 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.785 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.786 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.787 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "roption1": { 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "roption1", 2024-05-20T20:19:26.788 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "roption2": { 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "roption2", 2024-05-20T20:19:26.789 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "xyz", 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption1": { 2024-05-20T20:19:26.790 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption1", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.791 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption2": { 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption2", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.792 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption3": { 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption3", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.793 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption4": { 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption4", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.794 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption5": { 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption5", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.795 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption6": { 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption6", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.796 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption7": { 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption7", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "42", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.797 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.798 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "testkey": { 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testkey", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.799 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "testlkey": { 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testlkey", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.800 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "testnewline": { 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testnewline", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.801 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smb", 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.802 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.803 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.804 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.805 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.806 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.807 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "update_orchestration": { 2024-05-20T20:19:26.808 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "update_orchestration", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically update orchestration when smb resources are changed", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.809 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snap_schedule", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_m_granularity": { 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_m_granularity", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.810 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow minute scheduled snapshots", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "dump_on_update": { 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "dump_on_update", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.811 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "dump database to debug log on update", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.812 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.813 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.814 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.815 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.816 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.817 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "stats", 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.818 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.819 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.820 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.821 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.822 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.823 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.824 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "status", 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.825 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.826 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.827 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.828 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.829 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.830 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.831 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "telegraf", 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "address": { 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "address", 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-05-20T20:19:26.832 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.833 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.834 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.835 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.836 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.837 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.838 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.839 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.840 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "telemetry", 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_basic": { 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_basic", 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.841 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share basic cluster information (size, version)", 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_crash": { 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_crash", 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.842 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_device": { 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_device", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.843 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_ident": { 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_ident", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.844 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_perf": { 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_perf", 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.845 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share various performance metrics of a cluster", 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "contact": { 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "contact", 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.846 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "description": { 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "description", 2024-05-20T20:19:26.847 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "device_url": { 2024-05-20T20:19:26.848 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_url", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.849 INFO:teuthology.orchestra.run.smithi028.stdout: "enabled": { 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enabled", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.850 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "24", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "8", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.851 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "last_opt_revision": { 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "last_opt_revision", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.852 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "leaderboard": { 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "leaderboard", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.853 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "leaderboard_description": { 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "leaderboard_description", 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.854 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.855 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.856 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.857 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.858 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.859 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "organization": { 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "organization", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.860 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "proxy": { 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "proxy", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.861 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.862 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "url": { 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "url", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.863 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "test_orchestrator", 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.864 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.865 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.866 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.867 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.868 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.869 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.870 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "volumes", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.871 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.872 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.873 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.874 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.875 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "max_concurrent_clones": { 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_concurrent_clones", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "4", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.876 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Number of asynchronous cloner threads", 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "periodic_async_work": { 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "periodic_async_work", 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.877 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Periodically check for async work", 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "snapshot_clone_delay": { 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snapshot_clone_delay", 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.878 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "snapshot_clone_no_wait": { 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snapshot_clone_no_wait", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.879 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.880 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix", 2024-05-20T20:19:26.881 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "discovery_interval": { 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "discovery_interval", 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "100", 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.882 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "identifier": { 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "identifier", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.883 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.884 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.885 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.886 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.887 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.888 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.889 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.890 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_host": { 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_host", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.891 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_port": { 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_port", 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.892 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10051", 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_sender": { 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_sender", 2024-05-20T20:19:26.893 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.894 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "modules": [ 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm", 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "dashboard", 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "iostat", 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "nfs", 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "restful", 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "snap_schedule" 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: "available_modules": [ 2024-05-20T20:19:26.895 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "alerts", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.896 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How frequently to reexamine health status", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.897 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.898 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.899 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.900 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.901 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_destination": { 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_destination", 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.902 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Email address to send alerts to", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_from_name": { 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_from_name", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.903 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "Ceph", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Email From: name", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_host": { 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_host", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.904 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP server", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_password": { 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_password", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.905 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Password to authenticate with", 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_port": { 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_port", 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.906 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "465", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP port", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_sender": { 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_sender", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.907 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SMTP envelope sender", 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_ssl": { 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_ssl", 2024-05-20T20:19:26.908 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use SSL to connect to SMTP server", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "smtp_user": { 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smtp_user", 2024-05-20T20:19:26.909 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "User to authenticate as", 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.910 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.911 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "balancer", 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "active": { 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "active", 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.912 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically balance PGs across cluster", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "begin_time": { 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "begin_time", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0000", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.913 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "beginning time of day to automatically balance", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "begin_weekday": { 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "begin_weekday", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0", 2024-05-20T20:19:26.914 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "6", 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_max_iterations": { 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_max_iterations", 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "25", 2024-05-20T20:19:26.915 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "250", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_metrics": { 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_metrics", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "pgs,objects,bytes", 2024-05-20T20:19:26.916 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "crush_compat_step": { 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crush_compat_step", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0.5", 2024-05-20T20:19:26.917 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0.001", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "0.999", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "aggressiveness of optimization", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "end_time": { 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "end_time", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "2359", 2024-05-20T20:19:26.918 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "ending time of day to automatically balance", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "end_weekday": { 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "end_weekday", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.919 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "0", 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "6", 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.920 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.921 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.922 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.923 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.924 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "min_score": { 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "min_score", 2024-05-20T20:19:26.925 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": { 2024-05-20T20:19:26.926 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mode", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "upmap", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "crush-compat", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "none", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "read", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap", 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap-read" 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.927 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Balancer mode", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "pool_ids": { 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pool_ids", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.928 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.929 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.930 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap_max_deviation": { 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "upmap_max_deviation", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:26.931 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "deviation below which no optimization is attempted", 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "upmap_max_optimizations": { 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "upmap_max_optimizations", 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.932 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cephadm", 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:26.933 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_down_multiplier": { 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_down_multiplier", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3.0", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.934 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_refresh_rate": { 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_refresh_rate", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "20", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.935 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "agent_starting_port": { 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "agent_starting_port", 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "4721", 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-05-20T20:19:26.936 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_ptrace": { 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_ptrace", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-05-20T20:19:26.937 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "autotune_interval": { 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "autotune_interval", 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.938 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to autotune daemon memory", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "autotune_memory_target_ratio": { 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "autotune_memory_target_ratio", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0.7", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.939 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm_log_destination": { 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cephadm_log_destination", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.940 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "file", 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "file,syslog", 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "syslog" 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Destination for cephadm command's persistent logging", 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "cgroups_split": { 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cgroups_split", 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.941 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "config_checks_enabled": { 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_checks_enabled", 2024-05-20T20:19:26.942 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "config_dashboard": { 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_dashboard", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.943 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.944 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.944 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.944 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_alertmanager": { 2024-05-20T20:19:26.946 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_alertmanager", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.947 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_base": { 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_base", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/ceph", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Container image name, without the tag", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.948 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_elasticsearch": { 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_elasticsearch", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "elasticsearch container image", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.949 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_grafana": { 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_grafana", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/grafana:9.4.12", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.950 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_haproxy": { 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_haproxy", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "HAproxy container image", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.951 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_agent": { 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_agent", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger agent container image", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.952 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_collector": { 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_collector", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger collector container image", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.953 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_jaeger_query": { 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_jaeger_query", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Jaeger query container image", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.954 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_keepalived": { 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_keepalived", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Keepalived container image", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.955 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_loki": { 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_loki", 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/grafana/loki:3.0.0", 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Loki container image", 2024-05-20T20:19:26.956 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_node_exporter": { 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_node_exporter", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.957 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_nvmeof": { 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_nvmeof", 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/ceph/nvmeof:1.2.5", 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.958 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Nvme-of container image", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_prometheus": { 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_prometheus", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.959 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Prometheus container image", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_promtail": { 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_promtail", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/grafana/promtail:3.0.0", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.960 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Promtail container image", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_samba": { 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_samba", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "quay.io/samba.org/samba-server:devbuilds-centos-amd64", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Samba/SMB container image", 2024-05-20T20:19:26.961 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "container_image_snmp_gateway": { 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_image_snmp_gateway", 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.962 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "SNMP Gateway container image", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "container_init": { 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "container_init", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.963 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Run podman/docker with `--init`", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "daemon_cache_timeout": { 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "daemon_cache_timeout", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.964 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "default_cephadm_command_timeout": { 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "default_cephadm_command_timeout", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "900", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.965 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "default_registry": { 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "default_registry", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "docker.io", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.966 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "device_cache_timeout": { 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_cache_timeout", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1800", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.967 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache device inventory", 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "device_enhanced_scan": { 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_enhanced_scan", 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.968 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use libstoragemgmt during device scans", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "facts_cache_timeout": { 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "facts_cache_timeout", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.969 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "seconds to cache host facts data", 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "grafana_dashboards_path": { 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "grafana_dashboards_path", 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/etc/grafana/dashboards/ceph-dashboard/", 2024-05-20T20:19:26.970 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "location of dashboards to include in grafana deployments", 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "host_check_interval": { 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "host_check_interval", 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.971 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to perform a host check", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "hw_monitoring": { 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "hw_monitoring", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.972 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "inventory_list_all": { 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "inventory_list_all", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.973 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.974 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.975 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "log_refresh_metadata": { 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_refresh_metadata", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.976 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.977 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:26.978 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:26.979 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "manage_etc_ceph_ceph_conf": { 2024-05-20T20:19:26.980 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.981 INFO:teuthology.orchestra.run.smithi028.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "*", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.982 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "max_count_per_host": { 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_count_per_host", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "max number of daemons per service per host", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.983 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "max_osd_draining_count": { 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_osd_draining_count", 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.984 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "migration_current": { 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "migration_current", 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.985 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "internal - do not modify", 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "mode": { 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mode", 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "root", 2024-05-20T20:19:26.986 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm-package", 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "root" 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "mode for remote execution of cephadm", 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "oob_default_addr": { 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "oob_default_addr", 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.987 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "169.254.1.1", 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Default address for RedFish API (oob management).", 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "prometheus_alerts_path": { 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prometheus_alerts_path", 2024-05-20T20:19:26.988 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_insecure": { 2024-05-20T20:19:26.989 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_insecure", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.990 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_password": { 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_password", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.991 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_url": { 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_url", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.992 INFO:teuthology.orchestra.run.smithi028.stdout: "registry_username": { 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "registry_username", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.993 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "secure_monitoring_stack": { 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "secure_monitoring_stack", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.994 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "service_discovery_port": { 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "service_discovery_port", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8765", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "cephadm service discovery port", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.995 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.996 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "ssh_config_file": { 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssh_config_file", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.997 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "use_agent": { 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "use_agent", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:26.998 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "use_repo_digest": { 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "use_repo_digest", 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-05-20T20:19:26.999 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_failed_host_check": { 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_failed_host_check", 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.000 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if the host check fails", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_stray_daemons": { 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_stray_daemons", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.001 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_on_stray_hosts": { 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_on_stray_hosts", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.002 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crash", 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.003 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.004 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.005 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.006 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.007 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.008 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "retain_interval": { 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "retain_interval", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "31536000", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long to retain crashes before pruning them", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.009 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.010 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_recent_interval": { 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_recent_interval", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1209600", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "time interval in which to warn about recent crashes", 2024-05-20T20:19:27.011 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "dashboard", 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.012 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "ALERTMANAGER_API_HOST": { 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ALERTMANAGER_API_HOST", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.013 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.014 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "AUDIT_API_ENABLED": { 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "AUDIT_API_ENABLED", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.015 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.016 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "ENABLE_BROWSABLE_API": { 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ENABLE_BROWSABLE_API", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.017 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-05-20T20:19:27.018 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-05-20T20:19:27.019 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-05-20T20:19:27.020 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-05-20T20:19:27.021 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_NFS": { 2024-05-20T20:19:27.022 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.023 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_RBD": { 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "FEATURE_TOGGLE_RGW": { 2024-05-20T20:19:27.024 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.025 INFO:teuthology.orchestra.run.smithi028.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.026 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_PASSWORD": { 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_PASSWORD", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.027 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.028 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_URL": { 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_URL", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.029 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_API_USERNAME": { 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_API_USERNAME", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.030 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.031 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.032 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.033 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "ISSUE_TRACKER_API_KEY": { 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.034 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "MANAGED_BY_CLUSTERS": { 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "MANAGED_BY_CLUSTERS", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "[]", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.035 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "MULTICLUSTER_CONFIG": { 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "MULTICLUSTER_CONFIG", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "{}", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.036 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "PROMETHEUS_API_HOST": { 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PROMETHEUS_API_HOST", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.037 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.038 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.039 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.040 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.041 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.042 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.043 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.044 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.045 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_ENABLED": { 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_ENABLED", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.046 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.047 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.048 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8", 2024-05-20T20:19:27.049 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "REST_REQUESTS_TIMEOUT": { 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.050 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "45", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_ACCESS_KEY": { 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_ACCESS_KEY", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.051 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "admin", 2024-05-20T20:19:27.052 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_SECRET_KEY": { 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_SECRET_KEY", 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.053 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "RGW_API_SSL_VERIFY": { 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "RGW_API_SSL_VERIFY", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.054 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "UNSAFE_TLS_v1_2": { 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "UNSAFE_TLS_v1_2", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.055 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.056 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:27.057 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.058 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "cross_origin_url": { 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cross_origin_url", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.059 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "crt_file": { 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "crt_file", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.060 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "debug": { 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "debug", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.061 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Enable/disable debug options", 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "jwt_token_ttl": { 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "jwt_token_ttl", 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "28800", 2024-05-20T20:19:27.062 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "key_file": { 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "key_file", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.063 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.064 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.065 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.066 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.067 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.068 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "motd": { 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "motd", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.069 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "The message of the day", 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "redirect_resolve_ip_addr": { 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "redirect_resolve_ip_addr", 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.070 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "::", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.071 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8080", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.072 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.073 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl": { 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl", 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.074 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl_server_port": { 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl_server_port", 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8443", 2024-05-20T20:19:27.075 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_behaviour": { 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_behaviour", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "redirect", 2024-05-20T20:19:27.076 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "redirect" 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_error_status_code": { 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_error_status_code", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.077 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "500", 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "400", 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "599", 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "url_prefix": { 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "url_prefix", 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.078 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.079 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "devicehealth", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "enable_monitoring": { 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enable_monitoring", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "monitor device health metrics", 2024-05-20T20:19:27.080 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.081 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.082 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.083 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.084 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.085 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "mark_out_threshold": { 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mark_out_threshold", 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "2419200", 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.086 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "pool_name": { 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pool_name", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "device_health_metrics", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.087 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "name of pool in which to store device health metrics", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "retention_period": { 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "retention_period", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15552000", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.088 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long to retain device health metrics", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "scrape_frequency": { 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "scrape_frequency", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "86400", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.089 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to scrape device health metrics", 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "self_heal": { 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "self_heal", 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.090 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:27.091 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how frequently to wake up and check device health", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.092 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "warn_threshold": { 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "warn_threshold", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "7257600", 2024-05-20T20:19:27.093 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "diskprediction_local", 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.094 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.095 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.096 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.097 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.098 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.099 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "predict_interval": { 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "predict_interval", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "86400", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.100 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "predictor_model": { 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "predictor_model", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "prophetstor", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.101 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "600", 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.102 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.103 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "influx", 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": false, 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "influxdb python module not found", 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "batch_size": { 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "batch_size", 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.104 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5000", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "database": { 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "database", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.105 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "ceph", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "hostname": { 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "hostname", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.106 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB server hostname", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.107 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "30", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "5", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.108 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.109 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.110 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.111 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.112 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "password": { 2024-05-20T20:19:27.113 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "password", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "password of InfluxDB server user", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.114 INFO:teuthology.orchestra.run.smithi028.stdout: "port": { 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "port", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "8086", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "InfluxDB server port", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.115 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.116 INFO:teuthology.orchestra.run.smithi028.stdout: "ssl": { 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ssl", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "false", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.117 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "threads": { 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "threads", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "32", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.118 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "username": { 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "username", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "username of InfluxDB server user", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.119 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "verify_ssl": { 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "verify_ssl", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "true", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.120 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "insights", 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.121 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.122 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.123 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.124 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.125 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.126 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.127 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "iostat", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.128 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.129 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.130 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.131 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.132 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.133 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "k8sevents", 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_event_retention_days": { 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "ceph_event_retention_days", 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.134 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "7", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Days to hold ceph event information within local cache", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "config_check_secs": { 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "config_check_secs", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.135 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "10", 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.136 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.137 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.138 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.139 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.140 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.141 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.142 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "localpool", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "failure_domain": { 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "failure_domain", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "host", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.143 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "failure domain for any created local pool", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.144 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.145 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.146 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.147 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.151 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.151 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.151 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.151 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.152 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "min_size": { 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "min_size", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default min_size for any created local pool", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.153 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "num_rep": { 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "num_rep", 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3", 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default replica count for any created local pool", 2024-05-20T20:19:27.154 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_num": { 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pg_num", 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "128", 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.155 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "default pg_num for any created local pool", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "prefix": { 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prefix", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.156 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "name prefix for any created local pool", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.157 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "subtree": { 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "subtree", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "rack", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.158 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "CRUSH level for which to create a local pool", 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mds_autoscaler", 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.159 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.160 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.161 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.162 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.163 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.164 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.165 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mirroring", 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.166 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.167 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.168 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.169 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.170 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.171 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "nfs", 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.172 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.173 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.174 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.175 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.176 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.177 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.178 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "orchestrator", 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "fail_fs": { 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "fail_fs", 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.179 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.180 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.181 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.182 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.183 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.184 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator": { 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "orchestrator", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.185 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "cephadm", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "rook", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "test_orchestrator" 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Orchestrator backend", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.186 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.187 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "osd_perf_query", 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.188 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.189 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.190 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.191 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.192 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.193 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "osd_support", 2024-05-20T20:19:27.194 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.195 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.196 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.197 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.198 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.199 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.200 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "pg_autoscaler", 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.201 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.202 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.203 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.204 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.205 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.206 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.207 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "threshold": { 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "threshold", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:27.208 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "3.0", 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1.0", 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "scaling threshold", 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.209 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "progress", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_pg_recovery_event": { 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_pg_recovery_event", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.210 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow the module to show pg recovery progress", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "enabled": { 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enabled", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.211 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.212 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.213 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.214 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.215 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.216 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "max_completed_events": { 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_completed_events", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "50", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "number of past completed events to remember", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.217 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "sleep_interval": { 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sleep_interval", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "5", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "how long the module is going to sleep", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.218 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.219 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "prometheus", 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "cache": { 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "cache", 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.220 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "exclude_perf_counters": { 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "exclude_perf_counters", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.221 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.222 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.223 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.224 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.225 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.226 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_stats_pools": { 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_stats_pools", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.227 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_stats_pools_refresh_interval": { 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.228 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "300", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "scrape_interval": { 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "scrape_interval", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.229 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15.0", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.230 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "::", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.231 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "9283", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.232 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "stale_cache_strategy": { 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "stale_cache_strategy", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.233 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "log", 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_behaviour": { 2024-05-20T20:19:27.234 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_behaviour", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "default", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "default", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "error" 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.235 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "standby_error_status_code": { 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "standby_error_status_code", 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "500", 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "400", 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "599", 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.236 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_support", 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.237 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.238 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.239 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.240 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.241 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.242 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "max_concurrent_snap_create": { 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_concurrent_snap_create", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.243 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "mirror_snapshot_schedule": { 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "mirror_snapshot_schedule", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.244 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.245 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "trash_purge_schedule": { 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "trash_purge_schedule", 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.246 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "restful", 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "enable_auth": { 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enable_auth", 2024-05-20T20:19:27.247 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "key_file": { 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "key_file", 2024-05-20T20:19:27.248 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.249 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.250 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.251 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.252 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.253 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.254 INFO:teuthology.orchestra.run.smithi028.stdout: "server_addr": { 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_addr", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.255 INFO:teuthology.orchestra.run.smithi028.stdout: "server_port": { 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "server_port", 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.256 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.257 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:26 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2679046073' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-05-20T20:19:27.257 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:26 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/336399207' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-05-20T20:19:27.258 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:26 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2679046073' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-05-20T20:19:27.258 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:26 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/336399207' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-05-20T20:19:27.258 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.258 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.258 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.258 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.259 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rgw", 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.260 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.261 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.262 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.263 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.264 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.265 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.266 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rook", 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.267 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.268 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.269 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.270 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.271 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.272 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "storage_class": { 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "storage_class", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.273 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "local", 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "selftest", 2024-05-20T20:19:27.274 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.275 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.276 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.277 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.278 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.279 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "roption1": { 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "roption1", 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.280 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "roption2": { 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "roption2", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "xyz", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.281 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption1": { 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption1", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.282 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption2": { 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption2", 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.283 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption3": { 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption3", 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "float", 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.284 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption4": { 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption4", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.285 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption5": { 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption5", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.286 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption6": { 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption6", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.287 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "rwoption7": { 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rwoption7", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.288 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "1", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "42", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.289 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "testkey": { 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testkey", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.290 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "testlkey": { 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testlkey", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.291 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "testnewline": { 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "testnewline", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.292 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "smb", 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.293 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.294 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.295 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.296 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.297 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.298 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.299 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "update_orchestration": { 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "update_orchestration", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "automatically update orchestration when smb resources are changed", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.300 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snap_schedule", 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "allow_m_granularity": { 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "allow_m_granularity", 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.301 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "allow minute scheduled snapshots", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "dump_on_update": { 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "dump_on_update", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.302 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "dump database to debug log on update", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.303 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.304 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.305 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.306 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.307 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.308 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.309 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "stats", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.310 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.311 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.312 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.313 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.314 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.315 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "status", 2024-05-20T20:19:27.316 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.317 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.318 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.319 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.320 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.321 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.322 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "telegraf", 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "address": { 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "address", 2024-05-20T20:19:27.323 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:27.324 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "15", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.325 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.326 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.327 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.328 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.329 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.330 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.331 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "telemetry", 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_basic": { 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_basic", 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.332 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share basic cluster information (size, version)", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_crash": { 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_crash", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.333 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_device": { 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_device", 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.334 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_ident": { 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_ident", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.335 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "channel_perf": { 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "channel_perf", 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.336 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Share various performance metrics of a cluster", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "contact": { 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "contact", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.337 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "description": { 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "description", 2024-05-20T20:19:27.338 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "device_url": { 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "device_url", 2024-05-20T20:19:27.339 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "enabled": { 2024-05-20T20:19:27.340 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "enabled", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:27.341 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "24", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "8", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.342 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "last_opt_revision": { 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "last_opt_revision", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "1", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.343 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "leaderboard": { 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "leaderboard", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.344 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "leaderboard_description": { 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "leaderboard_description", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.345 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.346 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.347 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.348 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.349 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.350 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "organization": { 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "organization", 2024-05-20T20:19:27.354 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "proxy": { 2024-05-20T20:19:27.355 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "proxy", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.356 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "url": { 2024-05-20T20:19:27.357 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "url", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.358 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "test_orchestrator", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.359 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.360 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.361 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.362 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.363 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.364 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "volumes", 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.365 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.366 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.367 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.368 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.369 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.370 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "max_concurrent_clones": { 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "max_concurrent_clones", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "4", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Number of asynchronous cloner threads", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.371 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "periodic_async_work": { 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "periodic_async_work", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Periodically check for async work", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.372 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "snapshot_clone_delay": { 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snapshot_clone_delay", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.373 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "snapshot_clone_no_wait": { 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snapshot_clone_no_wait", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "True", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.374 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.375 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix", 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "can_run": true, 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "error_string": "", 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "module_options": { 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "discovery_interval": { 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "discovery_interval", 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "uint", 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.376 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "100", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "identifier": { 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "identifier", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.377 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "interval": { 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "interval", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "secs", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.378 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "60", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "log_level": { 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_level", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.379 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.380 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster": { 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.381 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_cluster_level": { 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_cluster_level", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "info", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [ 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "critical", 2024-05-20T20:19:27.382 INFO:teuthology.orchestra.run.smithi028.stdout: "debug", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "error", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "info", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "warning" 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "log_to_file": { 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "log_to_file", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "bool", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.383 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "False", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "sqlite3_killpoint": { 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "sqlite3_killpoint", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "dev", 2024-05-20T20:19:27.384 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 1, 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "0", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_host": { 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_host", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.385 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "", 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_port": { 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_port", 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "int", 2024-05-20T20:19:27.386 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "10051", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "zabbix_sender": { 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "zabbix_sender", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "str", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "level": "advanced", 2024-05-20T20:19:27.387 INFO:teuthology.orchestra.run.smithi028.stdout: "flags": 0, 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "min": "", 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "max": "", 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "enum_allowed": [], 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "desc": "", 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "long_desc": "", 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "tags": [], 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "see_also": [] 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "services": { 2024-05-20T20:19:27.388 INFO:teuthology.orchestra.run.smithi028.stdout: "dashboard": "https://172.21.15.28:8443/" 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "always_on_modules": { 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "octopus": [ 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "balancer", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "devicehealth", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_autoscaler", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "progress", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_support", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "status", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "telemetry", 2024-05-20T20:19:27.389 INFO:teuthology.orchestra.run.smithi028.stdout: "volumes" 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "pacific": [ 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "balancer", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "devicehealth", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_autoscaler", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "progress", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_support", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "status", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "telemetry", 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: "volumes" 2024-05-20T20:19:27.390 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "quincy": [ 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "balancer", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "devicehealth", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_autoscaler", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "progress", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_support", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "status", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "telemetry", 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "volumes" 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "reef": [ 2024-05-20T20:19:27.391 INFO:teuthology.orchestra.run.smithi028.stdout: "balancer", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "devicehealth", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_autoscaler", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "progress", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_support", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "status", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "telemetry", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "volumes" 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: ], 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "squid": [ 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "balancer", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "crash", 2024-05-20T20:19:27.392 INFO:teuthology.orchestra.run.smithi028.stdout: "devicehealth", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "orchestrator", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "pg_autoscaler", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "progress", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "rbd_support", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "status", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "telemetry", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "volumes" 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "last_failure_osd_epoch": 81, 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "active_clients": [ 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "devicehealth", 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: "addrvec": [ 2024-05-20T20:19:27.393 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v2", 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:0", 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 3549236304 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "libcephsqlite", 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "addrvec": [ 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v2", 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:0", 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 3603390838 2024-05-20T20:19:27.394 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "rbd_support", 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: "addrvec": [ 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v2", 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:0", 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 1657479752 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.395 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snap_schedule", 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "addrvec": [ 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v2", 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:0", 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 2569093583 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "snap_schedule", 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "addrvec": [ 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v2", 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:0", 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 485303750 2024-05-20T20:19:27.396 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: "name": "volumes", 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: "addrvec": [ 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: { 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: "type": "v2", 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: "addr": "172.21.15.28:0", 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: "nonce": 2286776261 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout: ] 2024-05-20T20:19:27.397 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:19:27.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fca400b29e0 msgr2=0x7fca400b4ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fca400b29e0 0x7fca400b4ea0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fca5c076ce0 tx=0x7fca48009260 comp rx=0 tx=0).stop 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c102830 msgr2=0x7fca5c19b200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c102830 0x7fca5c19b200 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7fca4c00eb90 tx=0x7fca4c00c6a0 comp rx=0 tx=0).stop 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 shutdown_connections 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fca5c1031f0 0x7fca5c19b740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fca400b29e0 0x7fca400b4ea0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fca5c108840 0x7fca5c19fac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 --2- 172.21.15.28:0/336399207 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fca5c102830 0x7fca5c19b200 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.820+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 >> 172.21.15.28:0/336399207 conn(0x7fca5c0fe380 msgr2=0x7fca5c10e0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.821+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 shutdown_connections 2024-05-20T20:19:27.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:19:25.821+0000 7fca62200640 1 -- 172.21.15.28:0/336399207 wait complete. 2024-05-20T20:19:27.451 DEBUG:tasks.check_counter:Getting stats from a 2024-05-20T20:19:27.451 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.a perf dump 2024-05-20T20:19:27.451 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:26 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2679046073' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-05-20T20:19:27.452 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:26 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/336399207' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-05-20T20:19:27.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:27 smithi154 ceph-mon[36314]: pgmap v433: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 1.2 KiB/s wr, 9 op/s 2024-05-20T20:19:27.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:27 smithi033 ceph-mon[36179]: pgmap v433: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 1.2 KiB/s wr, 9 op/s 2024-05-20T20:19:27.803 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:27 smithi028 ceph-mon[31178]: pgmap v433: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 1.2 KiB/s wr, 9 op/s 2024-05-20T20:19:27.930 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:29.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:29 smithi154 ceph-mon[36314]: pgmap v434: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 9 op/s 2024-05-20T20:19:29.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:29 smithi033 ceph-mon[36179]: pgmap v434: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 9 op/s 2024-05-20T20:19:29.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:29 smithi028 ceph-mon[31178]: pgmap v434: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 12 KiB/s wr, 9 op/s 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 1390, 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 1376, 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 2281344, 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 522784, 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 5, 2024-05-20T20:19:30.625 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 1, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.468017194, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.207236480, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.159164474, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.102478503, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 1375, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.420343199, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000305704 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.626 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 1926528, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 186976 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 1892, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 2114, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 485658, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 1156957, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 7, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.366512346, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.111022150, 2024-05-20T20:19:30.627 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.082837034, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.143666880, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 2112, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.380971153, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000180384 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 3312, 2024-05-20T20:19:30.628 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 557152 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 4, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 6, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 1629, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 1284, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 8, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.008414353, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.000094375, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.007693101, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.000107867, 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:30.629 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 4, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.003112219, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000778054 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 0, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 0 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "cct": { 2024-05-20T20:19:30.630 INFO:teuthology.orchestra.run.smithi028.stdout: "total_workers": 1, 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "unhealthy_workers": 0 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "finisher-MDSRank": { 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "queue_len": 0, 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "complete_latency": { 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 4801, 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.255780814, 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000053276 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "finisher-PurgeQueue": { 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "queue_len": 0, 2024-05-20T20:19:30.631 INFO:teuthology.orchestra.run.smithi028.stdout: "complete_latency": { 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "request": 0, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "reply": 0, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "reply_latency": { 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.632 INFO:teuthology.orchestra.run.smithi028.stdout: "slow_reply": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "forward": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_fetch_complete": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_fetch_keys": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_commit": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_split": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_merge": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_top": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_bottom": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_pin_tail": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_pinned": 0, 2024-05-20T20:19:30.633 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_expired": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_with_caps": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "caps": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "subtrees": 1, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_hit": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_forward": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_discover": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_remote_ino": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_lock": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "load_cent": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "q": 0, 2024-05-20T20:19:30.634 INFO:teuthology.orchestra.run.smithi028.stdout: "exported": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "exported_inodes": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "imported": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "imported_inodes": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "openino_dir_fetch": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "openino_peer_discover": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "root_rfiles": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "root_rbytes": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "root_rsnaps": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_set_tag": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:19:30.635 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_file_inodes": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_revoke": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_caps": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_cap_release": 0, 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: "process_request_cap_release": 0 2024-05-20T20:19:30.636 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_cache": { 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_update": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_update_receipt": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_try_discover": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_send_discover": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_handle_discover": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "num_strays": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "num_strays_delayed": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_created": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_enqueued": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_reintegrated": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_migrated": 0, 2024-05-20T20:19:30.637 INFO:teuthology.orchestra.run.smithi028.stdout: "num_recovering_processing": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "recovery_started": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "recovery_completed": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_quiesce_path": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_quiesce_inode": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_exportdir": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_flush": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_fragstats": 0, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_inodestats": 0 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.638 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_log": { 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "evlrg": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "evadd": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "evex": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "evtrm": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "ev": 5, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "evexg": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "evexd": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "segadd": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "segex": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "segtrm": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "seg": 2, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "segmjr": 2, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "segexg": 0, 2024-05-20T20:19:30.639 INFO:teuthology.orchestra.run.smithi028.stdout: "segexd": 0, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "expos": 4194304, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "wrpos": 4195734, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "rdpos": 4195593, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "jlat": { 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "replayed": 5 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_mem": { 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "ino": 3, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "ino+": 3, 2024-05-20T20:19:30.640 INFO:teuthology.orchestra.run.smithi028.stdout: "ino-": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "dir": 1, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "dir+": 1, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "dir-": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "dn": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "dn+": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "dn-": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "cap": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "cap+": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "cap-": 0, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "rss": 49408, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "heap": 215416 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_server": { 2024-05-20T20:19:30.641 INFO:teuthology.orchestra.run.smithi028.stdout: "dispatch_client_request": 0, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "dispatch_server_request": 0, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_request": 0, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_session": 0, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_peer_request": 0, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "req_create_latency": { 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "req_getattr_latency": { 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.642 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "req_getfilelock_latency": { 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "req_link_latency": { 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookup_latency": { 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.643 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookuphash_latency": { 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupino_latency": { 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupname_latency": { 2024-05-20T20:19:30.644 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupparent_latency": { 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupsnap_latency": { 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lssnap_latency": { 2024-05-20T20:19:30.645 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.646 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.646 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.646 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.646 INFO:teuthology.orchestra.run.smithi028.stdout: "req_mkdir_latency": { 2024-05-20T20:19:30.646 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "req_mknod_latency": { 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "req_mksnap_latency": { 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.647 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "req_open_latency": { 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "req_readdir_latency": { 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rename_latency": { 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.648 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "req_renamesnap_latency": { 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "req_snapdiff_latency": { 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rmdir_latency": { 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.649 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rmsnap_latency": { 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rmxattr_latency": { 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.650 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setattr_latency": { 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setdirlayout_latency": { 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setfilelock_latency": { 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.651 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setlayout_latency": { 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setxattr_latency": { 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "req_symlink_latency": { 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.652 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "req_unlink_latency": { 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "req_getvxattr_latency": { 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.653 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_sessions": { 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "session_count": 1, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "session_add": 2, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "session_remove": 1, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "sessions_open": 2, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "sessions_stale": 0, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "total_load": 0, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "average_load": 0, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "avg_session_uptime": 3740, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "mdthresh_evicted": 0 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "mempool": { 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:30.654 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:30.655 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:30.656 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_items": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_bytes": 198951, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_items": 55, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_bytes": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_items": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:30.657 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_bytes": 8128, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_items": 169, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_items": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_bytes": 8512, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_items": 13, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:30.658 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_items": 0, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_items": 0 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "objecter": { 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "op_active": 0, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "op_laggy": 0, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "op_send": 2946, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "op_send_bytes": 21, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "op_resend": 3, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "op_reply": 2946, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "op_latency": { 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 2946, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 7.905782884, 2024-05-20T20:19:30.659 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.002683565 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "op_inflight": 0, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "oplen_avg": { 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 2946, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 2947 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "op": 2946, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "op_r": 2946, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "op_w": 0, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "op_rmw": 0, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "op_pg": 0, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_stat": 1881, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_create": 0, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_read": 1063, 2024-05-20T20:19:30.660 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_write": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_writefull": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_writesame": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_append": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_zero": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_truncate": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_delete": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_mapext": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_sparse_read": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_clonerange": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_getxattr": 1, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_setxattr": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_rmxattr": 0, 2024-05-20T20:19:30.661 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_call": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_watch": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_notify": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_pgls": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_other": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_active": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_send": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_resend": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_ping": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "poolop_active": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "poolop_send": 0, 2024-05-20T20:19:30.662 INFO:teuthology.orchestra.run.smithi028.stdout: "poolop_resend": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "poolstat_active": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "poolstat_send": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "poolstat_resend": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "statfs_active": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "statfs_send": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "statfs_resend": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "command_active": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "command_send": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "command_resend": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "map_epoch": 97, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "map_full": 0, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "map_inc": 19, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_sessions": 6, 2024-05-20T20:19:30.663 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_session_open": 9, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_session_close": 3, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_laggy": 0, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_wr": 0, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_rd": 2, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_del": 0 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "oft": { 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_objs": 0, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_updates": 0, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_removes": 0 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "osd-slow-ops": { 2024-05-20T20:19:30.664 INFO:teuthology.orchestra.run.smithi028.stdout: "slow_ops_count": 0 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "purge_queue": { 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing_ops": 0, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing": 0, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing_high_water": 0, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executed_ops": 0, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executed": 0, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_item_in_journal": 0 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 104857600, 2024-05-20T20:19:30.665 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 3282, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 2453150, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 3282, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 3282, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 2453150, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.666 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-objecter_bytes": { 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 104857600, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 2946, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 1435, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 2946, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 1435, 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:30.667 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-objecter_ops": { 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 1024, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 2946, 2024-05-20T20:19:30.668 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 2946, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 2946, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 2946, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 3758096384, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:30.669 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-write_buf_throttle-0x557633f54780": { 2024-05-20T20:19:30.670 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 3758096384, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:30.671 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:30.672 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.672 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:30.672 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:19:31.552 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:31 smithi028 ceph-mon[31178]: pgmap v435: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.3 KiB/s rd, 157 KiB/s wr, 16 op/s 2024-05-20T20:19:31.606 INFO:tasks.check_counter:Daemon mds.a mds.exported=0 2024-05-20T20:19:31.607 INFO:tasks.check_counter:Daemon mds.a mds.imported=0 2024-05-20T20:19:31.607 INFO:tasks.check_counter:Daemon mds.a mds.root_rsnaps=0 2024-05-20T20:19:31.607 INFO:tasks.check_counter:Daemon mds.a mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:19:31.607 INFO:tasks.check_counter:Daemon mds.a mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:19:31.607 DEBUG:tasks.check_counter:Getting stats from d 2024-05-20T20:19:31.607 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.d perf dump 2024-05-20T20:19:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:31 smithi154 ceph-mon[36314]: pgmap v435: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.3 KiB/s rd, 157 KiB/s wr, 16 op/s 2024-05-20T20:19:31.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:31 smithi033 ceph-mon[36179]: pgmap v435: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.3 KiB/s rd, 157 KiB/s wr, 16 op/s 2024-05-20T20:19:32.073 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:33.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:33 smithi028 ceph-mon[31178]: pgmap v436: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 157 KiB/s wr, 12 op/s 2024-05-20T20:19:33.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:33 smithi154 ceph-mon[36314]: pgmap v436: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 157 KiB/s wr, 12 op/s 2024-05-20T20:19:33.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:33 smithi033 ceph-mon[36179]: pgmap v436: 97 pgs: 97 active+clean; 55 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 157 KiB/s wr, 12 op/s 2024-05-20T20:19:34.577 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 319, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 305, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 1890160, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 187728, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 4, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 0, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.155259199, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.053755876, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.117050314, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.012583455, 2024-05-20T20:19:34.578 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 305, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.120527970, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000395173 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 1890160, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 187728 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:34.579 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 2, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 232, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 320, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 198224, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 5, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.040683420, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.030490504, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.004367276, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 229, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.074441524, 2024-05-20T20:19:34.580 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000325072 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 320, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 198224 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 0, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 2, 2024-05-20T20:19:34.581 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 0, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 0, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 2, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 1, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.001123057, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.000000000, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.000852427, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:34.582 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 0, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 0 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "cct": { 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "total_workers": 0, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "unhealthy_workers": 0 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "mempool": { 2024-05-20T20:19:34.583 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:34.584 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:34.585 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_items": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_bytes": 148754, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_items": 31, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_bytes": 0, 2024-05-20T20:19:34.586 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_items": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_bytes": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_items": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_items": 0, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_bytes": 928, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_items": 2, 2024-05-20T20:19:34.587 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_items": 0, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_items": 0 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 104857600, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 317, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 1842696, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:34.588 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 317, 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 317, 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 1842696, 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:34.589 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:19:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:35 smithi028 ceph-mon[31178]: pgmap v437: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.1 KiB/s rd, 158 KiB/s wr, 230 op/s 2024-05-20T20:19:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:35 smithi154 ceph-mon[36314]: pgmap v437: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.1 KiB/s rd, 158 KiB/s wr, 230 op/s 2024-05-20T20:19:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:35 smithi033 ceph-mon[36179]: pgmap v437: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.1 KiB/s rd, 158 KiB/s wr, 230 op/s 2024-05-20T20:19:35.661 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.d 2024-05-20T20:19:35.662 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.d 2024-05-20T20:19:35.662 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.d 2024-05-20T20:19:35.662 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.d 2024-05-20T20:19:35.662 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.d 2024-05-20T20:19:35.662 DEBUG:tasks.check_counter:Getting stats from g 2024-05-20T20:19:35.663 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.g perf dump 2024-05-20T20:19:36.145 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:37.512 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:37 smithi028 ceph-mon[31178]: pgmap v438: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 157 KiB/s wr, 225 op/s 2024-05-20T20:19:37.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:37 smithi154 ceph-mon[36314]: pgmap v438: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 157 KiB/s wr, 225 op/s 2024-05-20T20:19:37.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:37 smithi033 ceph-mon[36179]: pgmap v438: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 157 KiB/s wr, 225 op/s 2024-05-20T20:19:38.355 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:19:38.355 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:38.355 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 0, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 3, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 0, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 0, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 5, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 0, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.005565682, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.000000000, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.001791848, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-05-20T20:19:38.356 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 0, 2024-05-20T20:19:38.357 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 0 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 6, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 231, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 3312, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 200128, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 6, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.045222806, 2024-05-20T20:19:38.358 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.033964859, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.005391986, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 228, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.088997414, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000390339 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:38.359 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 3312, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 200128 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 313, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 300, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 2037744, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 184736, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 3, 2024-05-20T20:19:38.360 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 1, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.152804326, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.046431000, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.102868130, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.019002010, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 299, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.129302601, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000432450 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.361 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:38.362 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:38.362 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:38.362 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:38.362 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 2037744, 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 184736 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: "cct": { 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: "total_workers": 0, 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: "unhealthy_workers": 0 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.363 INFO:teuthology.orchestra.run.smithi028.stdout: "mempool": { 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:38.364 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:38.365 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_items": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_bytes": 148698, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_items": 29, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:38.366 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_bytes": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_items": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_bytes": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_items": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_items": 0, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_bytes": 928, 2024-05-20T20:19:38.367 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_items": 2, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_items": 0, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_items": 0 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 104857600, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 315, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 1993572, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 315, 2024-05-20T20:19:38.368 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 315, 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 1993572, 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:38.369 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:19:39.336 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.g 2024-05-20T20:19:39.336 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.g 2024-05-20T20:19:39.336 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.g 2024-05-20T20:19:39.336 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.g 2024-05-20T20:19:39.336 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.g 2024-05-20T20:19:39.336 DEBUG:tasks.check_counter:Getting stats from j 2024-05-20T20:19:39.336 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.j perf dump 2024-05-20T20:19:39.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:39 smithi028 ceph-mon[31178]: pgmap v439: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 157 KiB/s wr, 226 op/s 2024-05-20T20:19:39.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:39 smithi154 ceph-mon[36314]: pgmap v439: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 157 KiB/s wr, 226 op/s 2024-05-20T20:19:39.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:39 smithi033 ceph-mon[36179]: pgmap v439: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.3 KiB/s rd, 157 KiB/s wr, 226 op/s 2024-05-20T20:19:39.823 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:19:41.530 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:41 smithi028 ceph-mon[31178]: pgmap v440: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 146 KiB/s wr, 228 op/s 2024-05-20T20:19:41.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:41 smithi154 ceph-mon[36314]: pgmap v440: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 146 KiB/s wr, 228 op/s 2024-05-20T20:19:41.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:41 smithi033 ceph-mon[36179]: pgmap v440: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.6 KiB/s rd, 146 KiB/s wr, 228 op/s 2024-05-20T20:19:42.153 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-05-20T20:19:42.153 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 1066, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 1066, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 284132, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 338228, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 7, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 2, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.228920656, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.054869099, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.047520485, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.094405907, 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:42.154 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 1065, 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.187156544, 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000175733 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 2992, 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 448 2024-05-20T20:19:42.155 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 1171, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 1157, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 2132712, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 456367, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 4, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.347013554, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.124989388, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.141146168, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.098643273, 2024-05-20T20:19:42.156 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 1156, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.259906143, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000224832 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 1929248, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 193376 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.157 INFO:teuthology.orchestra.run.smithi028.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_messages": 1055, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages": 1287, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_bytes": 354128, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_bytes": 896198, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_created_connections": 6, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_active_connections": 4, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_total_time": 0.316942007, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_send_time": 0.142642438, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_recv_time": 0.061732590, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_running_fast_dispatch_time": 0.082185595, 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:42.158 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 1284, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.414133788, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000322534 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_recv_encrypted_bytes": 320, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "msgr_send_encrypted_bytes": 561344 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "cct": { 2024-05-20T20:19:42.159 INFO:teuthology.orchestra.run.smithi028.stdout: "total_workers": 1, 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "unhealthy_workers": 0 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "finisher-MDSRank": { 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "queue_len": 0, 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "complete_latency": { 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 4951, 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.270140955, 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000054562 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "finisher-PurgeQueue": { 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "queue_len": 0, 2024-05-20T20:19:42.160 INFO:teuthology.orchestra.run.smithi028.stdout: "complete_latency": { 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "mds": { 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "request": 0, 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "reply": 0, 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "reply_latency": { 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.161 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "slow_reply": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "forward": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_fetch_complete": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_fetch_keys": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_commit": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_split": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_merge": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_top": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_bottom": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_pin_tail": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_pinned": 0, 2024-05-20T20:19:42.162 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_expired": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "inodes_with_caps": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "caps": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "subtrees": 1, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_hit": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_forward": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_discover": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_remote_ino": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "traverse_lock": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "load_cent": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "q": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "exported": 0, 2024-05-20T20:19:42.163 INFO:teuthology.orchestra.run.smithi028.stdout: "exported_inodes": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "imported": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "imported_inodes": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "openino_dir_fetch": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "openino_peer_discover": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "root_rfiles": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "root_rbytes": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "root_rsnaps": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_set_tag": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:19:42.164 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "scrub_file_inodes": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_revoke": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_caps": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:19:42.165 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_cap_release": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "process_request_cap_release": 0 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_cache": { 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_update": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_update_receipt": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_try_discover": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_send_discover": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "dir_handle_discover": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "num_strays": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "num_strays_delayed": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_created": 0, 2024-05-20T20:19:42.166 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_enqueued": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_reintegrated": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "strays_migrated": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "num_recovering_processing": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "recovery_started": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "recovery_completed": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_quiesce_path": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_quiesce_inode": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_exportdir": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_flush": 0, 2024-05-20T20:19:42.167 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_fragstats": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "ireq_inodestats": 0 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_log": { 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "evlrg": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "evadd": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "evex": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "evtrm": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "ev": 8, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "evexg": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "evexd": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "segadd": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "segex": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "segtrm": 0, 2024-05-20T20:19:42.168 INFO:teuthology.orchestra.run.smithi028.stdout: "seg": 2, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "segmjr": 2, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "segexg": 0, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "segexd": 0, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "expos": 4194304, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "wrpos": 4196657, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "rdpos": 4196516, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "jlat": { 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "replayed": 8 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.169 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_mem": { 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "ino": 3, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "ino+": 3, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "ino-": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "dir": 1, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "dir+": 1, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "dir-": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "dn": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "dn+": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "dn-": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "cap": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "cap+": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "cap-": 0, 2024-05-20T20:19:42.170 INFO:teuthology.orchestra.run.smithi028.stdout: "rss": 48896, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "heap": 215416 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_server": { 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "dispatch_client_request": 0, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "dispatch_server_request": 0, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_request": 0, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_client_session": 0, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "handle_peer_request": 0, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "req_create_latency": { 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "req_getattr_latency": { 2024-05-20T20:19:42.171 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "req_getfilelock_latency": { 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "req_link_latency": { 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.172 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookup_latency": { 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookuphash_latency": { 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupino_latency": { 2024-05-20T20:19:42.173 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupname_latency": { 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupparent_latency": { 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.174 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lookupsnap_latency": { 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "req_lssnap_latency": { 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "req_mkdir_latency": { 2024-05-20T20:19:42.175 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: "req_mknod_latency": { 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.176 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "req_mksnap_latency": { 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "req_open_latency": { 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "req_readdir_latency": { 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.177 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rename_latency": { 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "req_renamesnap_latency": { 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "req_snapdiff_latency": { 2024-05-20T20:19:42.178 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rmdir_latency": { 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rmsnap_latency": { 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.179 INFO:teuthology.orchestra.run.smithi028.stdout: "req_rmxattr_latency": { 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setattr_latency": { 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setdirlayout_latency": { 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.180 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setfilelock_latency": { 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setlayout_latency": { 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "req_setxattr_latency": { 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.181 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "req_symlink_latency": { 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "req_unlink_latency": { 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.182 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "req_getvxattr_latency": { 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_sessions": { 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "session_count": 2, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "session_add": 4, 2024-05-20T20:19:42.183 INFO:teuthology.orchestra.run.smithi028.stdout: "session_remove": 2, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "sessions_open": 4, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "sessions_stale": 0, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "total_load": 0, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "average_load": 0, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "avg_session_uptime": 3751, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "mdthresh_evicted": 0 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "mempool": { 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:42.184 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:42.185 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:42.186 INFO:teuthology.orchestra.run.smithi028.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_items": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_bytes": 191992, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_anon_items": 56, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_bytes": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_items": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:42.187 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_bytes": 8128, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_items": 169, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "pgmap_items": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_bytes": 8512, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "mds_co_items": 13, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_1_items": 0, 2024-05-20T20:19:42.188 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "unittest_2_items": 0 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "objecter": { 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "op_active": 0, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "op_laggy": 0, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "op_send": 2942, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "op_send_bytes": 21, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "op_resend": 3, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "op_reply": 2942, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "op_latency": { 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 2942, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 22.768374460, 2024-05-20T20:19:42.189 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.007739080 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "op_inflight": 0, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "oplen_avg": { 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 2942, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 2943 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "op": 2942, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "op_r": 2942, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "op_w": 0, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "op_rmw": 0, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "op_pg": 0, 2024-05-20T20:19:42.190 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_stat": 1878, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_create": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_read": 1062, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_write": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_writefull": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_writesame": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_append": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_zero": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_truncate": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_delete": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_mapext": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_sparse_read": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_clonerange": 0, 2024-05-20T20:19:42.191 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_getxattr": 1, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_setxattr": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_rmxattr": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_call": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_watch": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_notify": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_pgls": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "osdop_other": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_active": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_send": 0, 2024-05-20T20:19:42.192 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_resend": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "linger_ping": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "poolop_active": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "poolop_send": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "poolop_resend": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "poolstat_active": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "poolstat_send": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "poolstat_resend": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "statfs_active": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "statfs_send": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "statfs_resend": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "command_active": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "command_send": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "command_resend": 0, 2024-05-20T20:19:42.193 INFO:teuthology.orchestra.run.smithi028.stdout: "map_epoch": 97, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "map_full": 0, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "map_inc": 19, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_sessions": 5, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_session_open": 6, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_session_close": 1, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "osd_laggy": 0, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_wr": 0, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_rd": 2, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_del": 0 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "oft": { 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_objs": 0, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:19:42.194 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_updates": 0, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "omap_total_removes": 0 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "osd-slow-ops": { 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "slow_ops_count": 0 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "purge_queue": { 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing_ops": 0, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing": 0, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executing_high_water": 0, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executed_ops": 0, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_executed": 0, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "pq_item_in_journal": 0 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.195 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 104857600, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 3288, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 2454431, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 3288, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 3288, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 2454431, 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:42.196 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-objecter_bytes": { 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 104857600, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 2942, 2024-05-20T20:19:42.197 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 2358, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 2942, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 2358, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-objecter_ops": { 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 1024, 2024-05-20T20:19:42.198 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 2942, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 2942, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 2942, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 2942, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.199 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 3758096384, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 0, 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:42.200 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: }, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "throttle-write_buf_throttle-0x55fa3d6a0780": { 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "val": 0, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "max": 3758096384, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "get_started": 0, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "get": 0, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "get_sum": 0, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:42.201 INFO:teuthology.orchestra.run.smithi028.stdout: "take": 0, 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: "take_sum": 0, 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: "put": 0, 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: "put_sum": 0, 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: "wait": { 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: "avgcount": 0, 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: "sum": 0.000000000, 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: "avgtime": 0.000000000 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout: } 2024-05-20T20:19:42.202 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-05-20T20:19:43.180 INFO:tasks.check_counter:Daemon mds.j mds.exported=0 2024-05-20T20:19:43.180 INFO:tasks.check_counter:Daemon mds.j mds.imported=0 2024-05-20T20:19:43.180 INFO:tasks.check_counter:Daemon mds.j mds.root_rsnaps=0 2024-05-20T20:19:43.181 INFO:tasks.check_counter:Daemon mds.j mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:19:43.181 INFO:tasks.check_counter:Daemon mds.j mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:19:43.181 DEBUG:tasks.check_counter:Getting stats from b 2024-05-20T20:19:43.181 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.b perf dump 2024-05-20T20:19:43.440 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T20:19:43.562 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:43 smithi033 ceph-mon[36179]: pgmap v441: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 597 B/s wr, 221 op/s 2024-05-20T20:19:43.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:43 smithi154 ceph-mon[36314]: pgmap v441: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 597 B/s wr, 221 op/s 2024-05-20T20:19:43.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:43 smithi028 ceph-mon[31178]: pgmap v441: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 597 B/s wr, 221 op/s 2024-05-20T20:19:45.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:45 smithi154 ceph-mon[36314]: pgmap v442: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.6 KiB/s rd, 597 B/s wr, 225 op/s 2024-05-20T20:19:45.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:45 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:19:45.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:45 smithi033 ceph-mon[36179]: pgmap v442: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.6 KiB/s rd, 597 B/s wr, 225 op/s 2024-05-20T20:19:45.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:45 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:19:45.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:45 smithi028 ceph-mon[31178]: pgmap v442: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.6 KiB/s rd, 597 B/s wr, 225 op/s 2024-05-20T20:19:45.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:45 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T20:19:45.944 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 3990, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 3579, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 1081748, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 15649155, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 226, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 6, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.433655196, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.127341960, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.178987295, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.093260470, 2024-05-20T20:19:45.945 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 3583, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.350885456, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000097930 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 3311, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000716739, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000216 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 42560, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 2416 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.946 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 6437, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 5242, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 2414189, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 7374978, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 24, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 8, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.596105948, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.170931060, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.296451390, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.160587833, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 5233, 2024-05-20T20:19:45.947 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.430843922, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000082332 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 5892, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.001222233, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000207 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 1767664, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 1577360 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 3591, 2024-05-20T20:19:45.948 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 3853, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 3087109, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 6444822, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 25, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 7, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.367837019, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.126787491, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.195397740, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.082245007, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 3853, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.265687778, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000068956 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.949 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 3996, 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000964278, 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000241 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 1969584, 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 608096 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "cct": { 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "total_workers": 1, 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "unhealthy_workers": 0 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-MDSRank": { 2024-05-20T20:19:45.950 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 2917, 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 1.343943194, 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000460727 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-PurgeQueue": { 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 2497, 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.076887540, 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000030791 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.951 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "request": 391, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "reply": 391, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "reply_latency": { 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 391, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 26.171355686, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.066934413 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_reply": 0, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "forward": 0, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_complete": 0, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_keys": 0, 2024-05-20T20:19:45.952 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_commit": 435, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_split": 1, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_merge": 0, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes": 180, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_top": 98, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_bottom": 82, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pin_tail": 0, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pinned": 39, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_expired": 0, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_with_caps": 3, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "caps": 3, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "subtrees": 2, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse": 592, 2024-05-20T20:19:45.953 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_hit": 581, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_forward": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_discover": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_remote_ino": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_lock": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "load_cent": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "q": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "exported": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "exported_inodes": 0, 2024-05-20T20:19:45.954 INFO:teuthology.orchestra.run.smithi033.stdout: "imported": 0, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "imported_inodes": 0, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_dir_fetch": 0, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_backtrace_fetch": 344, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_peer_discover": 0, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rfiles": 4, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rbytes": 234, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rsnaps": 6, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_fetch": 344, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_set_tag": 0, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:19:45.955 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_inodes": 130, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_base_inodes": 13, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dirfrag_rstats": 117, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_file_inodes": 214, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_inode_file_caps": 5, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_revoke": 269, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_grant": 452, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps": 1889, 2024-05-20T20:19:45.956 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps_dirty": 1636, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_cap_release": 9, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "process_request_cap_release": 66 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_cache": { 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update": 0, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update_receipt": 0, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_try_discover": 0, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_send_discover": 0, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_handle_discover": 7, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays": 4, 2024-05-20T20:19:45.957 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_delayed": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_created": 22, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_enqueued": 18, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_reintegrated": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_migrated": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_processing": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_started": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_completed": 0, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_path": 9, 2024-05-20T20:19:45.958 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_inode": 255, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_enqueue_scrub": 13, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_exportdir": 0, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_flush": 0, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragmentdir": 1, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragstats": 0, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_inodestats": 0 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_log": { 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "evlrg": 0, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "evadd": 2510, 2024-05-20T20:19:45.959 INFO:teuthology.orchestra.run.smithi033.stdout: "evex": 2462, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "evtrm": 2462, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "ev": 48, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "evexg": 0, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "evexd": 41, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "segadd": 63, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "segex": 62, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "segtrm": 62, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "seg": 1, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "segmjr": 1, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "segexg": 0, 2024-05-20T20:19:45.960 INFO:teuthology.orchestra.run.smithi033.stdout: "segexd": 1, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "expos": 24197530, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "wrpos": 24363212, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "rdpos": 0, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "jlat": { 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1078, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 20.485782749, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.019003509 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "replayed": 0 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_mem": { 2024-05-20T20:19:45.961 INFO:teuthology.orchestra.run.smithi033.stdout: "ino": 135, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "ino+": 166, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "ino-": 31, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "dir": 29, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "dir+": 30, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "dir-": 1, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "dn": 180, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "dn+": 181, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "dn-": 1, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "cap": 3, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "cap+": 45, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "cap-": 42, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "rss": 62264, 2024-05-20T20:19:45.962 INFO:teuthology.orchestra.run.smithi033.stdout: "heap": 231800 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_server": { 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_client_request": 516, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_server_request": 82, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_request": 391, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_session": 37, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_peer_request": 164, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "req_create_latency": { 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 22, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.020545517, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000933887 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getattr_latency": { 2024-05-20T20:19:45.963 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 170, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.472177994, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.002777517 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getfilelock_latency": { 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "req_link_latency": { 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.964 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookup_latency": { 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 98, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.437459002, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.004463867 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookuphash_latency": { 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupino_latency": { 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000158252, 2024-05-20T20:19:45.965 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000158252 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupname_latency": { 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupparent_latency": { 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.966 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupsnap_latency": { 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lssnap_latency": { 2024-05-20T20:19:45.967 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 14, 2024-05-20T20:19:45.968 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.002788188, 2024-05-20T20:19:45.968 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000199156 2024-05-20T20:19:45.968 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mkdir_latency": { 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 10, 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 24.955506426, 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 2.495550642 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mknod_latency": { 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.969 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mksnap_latency": { 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 14, 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.110997536, 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.007928395 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: "req_open_latency": { 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1, 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000263880, 2024-05-20T20:19:45.970 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000263880 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "req_readdir_latency": { 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 20, 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.030043890, 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.001502194 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rename_latency": { 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 2, 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.002263549, 2024-05-20T20:19:45.971 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.001131774 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "req_renamesnap_latency": { 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "req_snapdiff_latency": { 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmdir_latency": { 2024-05-20T20:19:45.972 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 4, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.032459097, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.008114774 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmsnap_latency": { 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 8, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.053623820, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.006702977 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmxattr_latency": { 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.973 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setattr_latency": { 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.001852930, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.001852930 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setdirlayout_latency": { 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setfilelock_latency": { 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.974 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setlayout_latency": { 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setxattr_latency": { 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 8, 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.024080283, 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.003010035 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "req_symlink_latency": { 2024-05-20T20:19:45.975 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "req_unlink_latency": { 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 18, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.027135322, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.001507517 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getvxattr_latency": { 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.976 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_sessions": { 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "session_count": 2, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "session_add": 5, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "session_remove": 3, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_open": 2, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_stale": 0, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "total_load": 95, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "average_load": 47, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "avg_session_uptime": 834, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "mdthresh_evicted": 0 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.977 INFO:teuthology.orchestra.run.smithi033.stdout: "mempool": { 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:45.978 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:45.979 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_items": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_bytes": 344709, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_items": 141, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:45.980 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_bytes": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_items": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_bytes": 8128, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_items": 169, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_items": 0, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_bytes": 513782, 2024-05-20T20:19:45.981 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_items": 4706, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_items": 0, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_items": 0 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "objecter": { 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "op_active": 0, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "op_laggy": 0, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send": 5405, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send_bytes": 21897164, 2024-05-20T20:19:45.982 INFO:teuthology.orchestra.run.smithi033.stdout: "op_resend": 37, 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "op_reply": 5405, 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "op_latency": { 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 5405, 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 92.522590395, 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.017117963 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "op_inflight": 0, 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "oplen_avg": { 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 5405, 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 6416 2024-05-20T20:19:45.983 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "op": 5405, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "op_r": 373, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "op_w": 5032, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "op_rmw": 0, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "op_pg": 0, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_stat": 405, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_create": 66, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_read": 16, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_write": 1081, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writefull": 206, 2024-05-20T20:19:45.984 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writesame": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_append": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_zero": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_truncate": 1, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_delete": 3098, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_mapext": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_sparse_read": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_clonerange": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_getxattr": 357, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_setxattr": 132, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_rmxattr": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_call": 0, 2024-05-20T20:19:45.985 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_watch": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_notify": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_other": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_active": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_send": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_resend": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_ping": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_active": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_send": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_resend": 0, 2024-05-20T20:19:45.986 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_active": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_send": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_resend": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_active": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_send": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_resend": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "command_active": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "command_send": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "command_resend": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "map_epoch": 97, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "map_full": 0, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "map_inc": 19, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_sessions": 12, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_open": 12, 2024-05-20T20:19:45.987 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_close": 0, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_laggy": 0, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_wr": 1021, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_rd": 0, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_del": 33 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "oft": { 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_objs": 1, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_updates": 0, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_removes": 0 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.988 INFO:teuthology.orchestra.run.smithi033.stdout: "osd-slow-ops": { 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_ops_count": 347 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "purge_queue": { 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops": 0, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops_high_water": 513, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing": 0, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_high_water": 6, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed_ops": 2578, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed": 18, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_item_in_journal": 0 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:45.989 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 14014, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 5108344, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 14014, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 14014, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 5108344, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.990 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_bytes": { 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 5405, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 21899806, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 5405, 2024-05-20T20:19:45.991 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 21899806, 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_ops": { 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:45.992 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 1024, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 5405, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 5405, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 5405, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 5405, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.993 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 18, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 2642, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 18, 2024-05-20T20:19:45.994 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 3, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 2642, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle-0x560e22fe0500": { 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:45.995 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 2510, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 20168908, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 2510, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 1078, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 20168908, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:45.996 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:45.997 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:45.997 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.997 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:45.997 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-05-20T20:19:47.587 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:47 smithi033 ceph-mon[36179]: pgmap v443: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:47.587 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:47 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:19:47.588 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:47 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:19:47.588 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:47 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:19:47.615 INFO:tasks.check_counter:Daemon mds.b mds.exported=0 2024-05-20T20:19:47.615 INFO:tasks.check_counter:Daemon mds.b mds.imported=0 2024-05-20T20:19:47.616 INFO:tasks.check_counter:Daemon mds.b mds.root_rsnaps=6 2024-05-20T20:19:47.616 INFO:tasks.check_counter:Daemon mds.b mds_server.req_mksnap_latency.avgcount=14 2024-05-20T20:19:47.616 INFO:tasks.check_counter:Daemon mds.b mds_server.req_rmsnap_latency.avgcount=8 2024-05-20T20:19:47.616 DEBUG:tasks.check_counter:Getting stats from e 2024-05-20T20:19:47.616 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.e perf dump 2024-05-20T20:19:47.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:47 smithi154 ceph-mon[36314]: pgmap v443: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:47.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:47 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:19:47.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:47 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:19:47.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:47 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:19:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:47 smithi028 ceph-mon[31178]: pgmap v443: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:47 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T20:19:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:47 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T20:19:47.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:47 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' 2024-05-20T20:19:47.863 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T20:19:49.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:49 smithi033 ceph-mon[36179]: pgmap v444: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:19:49.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:49 smithi028 ceph-mon[31178]: pgmap v444: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:19:49.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:49 smithi154 ceph-mon[36314]: pgmap v444: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:19:49.989 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-05-20T20:19:49.989 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 28, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 201, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 6857, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 455479, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 13, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 2, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.015826847, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.005959932, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.003855826, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.000520575, 2024-05-20T20:19:49.990 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 199, 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.050608505, 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000254314 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 4, 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000383, 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000095 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 160, 2024-05-20T20:19:49.991 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 394848 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 55, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 160, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 12967, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 481031, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 36, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 8, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.027300268, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.006931764, 2024-05-20T20:19:49.992 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.017513352, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.000353825, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 157, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.019090251, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000121593 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 83, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000019499, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000234 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.993 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 6208, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 184480 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 945, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 1160, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 2098919, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 525928, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 10, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 4, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.143201058, 2024-05-20T20:19:49.994 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.054233438, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.086997913, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.013673996, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1160, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.060091456, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000051802 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1314, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000343255, 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000261 2024-05-20T20:19:49.995 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 1951376, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 199264 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "cct": { 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "total_workers": 1, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "unhealthy_workers": 0 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-MDSRank": { 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 56, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.001270112, 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000022680 2024-05-20T20:19:49.996 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-PurgeQueue": { 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 2, 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000010992, 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000005496 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "request": 1, 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "reply": 1, 2024-05-20T20:19:49.997 INFO:teuthology.orchestra.run.smithi033.stdout: "reply_latency": { 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000138016, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000138016 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_reply": 0, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "forward": 0, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_complete": 0, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_keys": 0, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_commit": 11, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_split": 0, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_merge": 0, 2024-05-20T20:19:49.998 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes": 10, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_top": 0, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_bottom": 10, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pin_tail": 0, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pinned": 10, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_expired": 0, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_with_caps": 1, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "caps": 1, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "subtrees": 1, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse": 18, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_hit": 18, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_forward": 0, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_discover": 0, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:19:49.999 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_remote_ino": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_lock": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "load_cent": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "q": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "exported": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "exported_inodes": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "imported": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "imported_inodes": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_dir_fetch": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_peer_discover": 0, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rfiles": 22, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rbytes": 10738143985, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rsnaps": 6, 2024-05-20T20:19:50.000 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_set_tag": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_file_inodes": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_revoke": 1, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:19:50.001 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps": 1, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_cap_release": 0, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "process_request_cap_release": 0 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_cache": { 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update": 0, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update_receipt": 0, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_try_discover": 0, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_send_discover": 1, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_handle_discover": 0, 2024-05-20T20:19:50.002 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_delayed": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_created": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_enqueued": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_reintegrated": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_migrated": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_processing": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_started": 0, 2024-05-20T20:19:50.003 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_completed": 0, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_path": 9, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_inode": 9, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_exportdir": 0, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_flush": 0, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragstats": 0, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_inodestats": 0 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_log": { 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "evlrg": 0, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "evadd": 5, 2024-05-20T20:19:50.004 INFO:teuthology.orchestra.run.smithi033.stdout: "evex": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "evtrm": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "ev": 5, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "evexg": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "evexd": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "segadd": 2, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "segex": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "segtrm": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "seg": 2, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "segmjr": 2, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "segexg": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "segexd": 0, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "expos": 4194304, 2024-05-20T20:19:50.005 INFO:teuthology.orchestra.run.smithi033.stdout: "wrpos": 4195734, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "rdpos": 0, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "jlat": { 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 4, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.014352248, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.003588062 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "replayed": 0 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_mem": { 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "ino": 13, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "ino+": 13, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "ino-": 0, 2024-05-20T20:19:50.006 INFO:teuthology.orchestra.run.smithi033.stdout: "dir": 11, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "dir+": 12, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "dir-": 1, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "dn": 10, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "dn+": 10, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "dn-": 0, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "cap": 1, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "cap+": 1, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "cap-": 0, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "rss": 53320, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "heap": 223608 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_server": { 2024-05-20T20:19:50.007 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_client_request": 1, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_server_request": 0, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_request": 1, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_session": 16, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_peer_request": 9, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "req_create_latency": { 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getattr_latency": { 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.008 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getfilelock_latency": { 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "req_link_latency": { 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.009 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookup_latency": { 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookuphash_latency": { 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupino_latency": { 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1, 2024-05-20T20:19:50.010 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000138016, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000138016 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupname_latency": { 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupparent_latency": { 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.011 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupsnap_latency": { 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lssnap_latency": { 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.012 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mkdir_latency": { 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mknod_latency": { 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.013 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mksnap_latency": { 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "req_open_latency": { 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.014 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "req_readdir_latency": { 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rename_latency": { 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "req_renamesnap_latency": { 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.015 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "req_snapdiff_latency": { 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmdir_latency": { 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.016 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmsnap_latency": { 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmxattr_latency": { 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setattr_latency": { 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.017 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setdirlayout_latency": { 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setfilelock_latency": { 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.018 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setlayout_latency": { 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setxattr_latency": { 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "req_symlink_latency": { 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.019 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "req_unlink_latency": { 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getvxattr_latency": { 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.020 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_sessions": { 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "session_count": 1, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "session_add": 2, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "session_remove": 1, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_open": 1, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_stale": 0, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "total_load": 1, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "average_load": 0, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "avg_session_uptime": 800, 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: "mdthresh_evicted": 0 2024-05-20T20:19:50.021 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "mempool": { 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:50.022 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:50.023 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_items": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_bytes": 248055, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_items": 85, 2024-05-20T20:19:50.024 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_bytes": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_items": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_bytes": 8128, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_items": 169, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:50.025 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_items": 0, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_bytes": 51248, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_items": 201, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_items": 0, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_items": 0 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "objecter": { 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "op_active": 0, 2024-05-20T20:19:50.026 INFO:teuthology.orchestra.run.smithi033.stdout: "op_laggy": 0, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send": 41, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send_bytes": 11822, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "op_resend": 33, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "op_reply": 41, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "op_latency": { 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 41, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.429993217, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.010487639 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.027 INFO:teuthology.orchestra.run.smithi033.stdout: "op_inflight": 0, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "oplen_avg": { 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 41, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 63 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "op": 41, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "op_r": 0, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "op_w": 41, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "op_rmw": 0, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "op_pg": 0, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_stat": 0, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_create": 10, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_read": 0, 2024-05-20T20:19:50.028 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_write": 4, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writefull": 8, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writesame": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_append": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_zero": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_truncate": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_delete": 6, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_mapext": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_sparse_read": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_clonerange": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_getxattr": 0, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_setxattr": 20, 2024-05-20T20:19:50.029 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_rmxattr": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_call": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_watch": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_notify": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_other": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_active": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_send": 0, 2024-05-20T20:19:50.030 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_resend": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_ping": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_active": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_send": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_resend": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_active": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_send": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_resend": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_active": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_send": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_resend": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "command_active": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "command_send": 0, 2024-05-20T20:19:50.031 INFO:teuthology.orchestra.run.smithi033.stdout: "command_resend": 0, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "map_epoch": 97, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "map_full": 0, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "map_inc": 19, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_sessions": 8, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_open": 12, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_close": 4, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_laggy": 0, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_wr": 14, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_rd": 0, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_del": 1 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "oft": { 2024-05-20T20:19:50.032 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_objs": 1, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_updates": 0, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_removes": 0 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "osd-slow-ops": { 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_ops_count": 18 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "purge_queue": { 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops": 0, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing": 0, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_high_water": 0, 2024-05-20T20:19:50.033 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed_ops": 0, 2024-05-20T20:19:50.034 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed": 0, 2024-05-20T20:19:50.034 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_item_in_journal": 0 2024-05-20T20:19:50.034 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.034 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:50.034 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:50.034 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:50.034 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:50.035 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 1024, 2024-05-20T20:19:50.035 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 2003536, 2024-05-20T20:19:50.035 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:50.035 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 1024, 2024-05-20T20:19:50.035 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:50.035 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 1024, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 2003536, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_bytes": { 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:50.036 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 41, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 11822, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 41, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 11822, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:50.037 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_ops": { 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 1024, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 41, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 41, 2024-05-20T20:19:50.038 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 41, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 41, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:50.039 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 0, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 0, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:50.040 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle-0x5600ca0a4500": { 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 5, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 1430, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 5, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 4, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 1430, 2024-05-20T20:19:50.041 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:50.042 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:50.042 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:50.042 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:50.042 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:50.042 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:50.042 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-05-20T20:19:50.846 INFO:tasks.check_counter:Daemon mds.e mds.exported=0 2024-05-20T20:19:50.846 INFO:tasks.check_counter:Daemon mds.e mds.imported=0 2024-05-20T20:19:50.846 INFO:tasks.check_counter:Daemon mds.e mds.root_rsnaps=6 2024-05-20T20:19:50.846 INFO:tasks.check_counter:Daemon mds.e mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:19:50.846 INFO:tasks.check_counter:Daemon mds.e mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:19:50.846 DEBUG:tasks.check_counter:Getting stats from h 2024-05-20T20:19:50.847 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.h perf dump 2024-05-20T20:19:51.097 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T20:19:51.484 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:51 smithi033 ceph-mon[36179]: pgmap v445: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:19:51.677 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:51 smithi154 ceph-mon[36314]: pgmap v445: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:19:51.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:51 smithi028 ceph-mon[31178]: pgmap v445: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2024-05-20T20:19:53.178 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-05-20T20:19:53.178 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 581, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 819, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 151039, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 163633, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 9, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.087490936, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.037562645, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.039103963, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.006111486, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 817, 2024-05-20T20:19:53.179 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.049128973, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000060133 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1317, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000347960, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000264 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 0, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 0 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 410, 2024-05-20T20:19:53.180 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 557, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 1965323, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 720439, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 18, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 6, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.074194943, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.023850495, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.056906231, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.007920201, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 557, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.056545601, 2024-05-20T20:19:53.181 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000101518 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 17, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000002269, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000133 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 1961088, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 593568 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 35, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 97, 2024-05-20T20:19:53.182 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 6491, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 527372, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 32, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 4, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.017190382, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.003484348, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.011630572, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.000228750, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 94, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.014062887, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000149605 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.183 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 31, 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000005300, 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000170 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 1088, 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 251344 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "cct": { 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "total_workers": 1, 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "unhealthy_workers": 0 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-MDSRank": { 2024-05-20T20:19:53.184 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 52, 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.001638148, 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000031502 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-PurgeQueue": { 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 2, 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000009012, 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000004506 2024-05-20T20:19:53.185 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "request": 0, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "reply": 0, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "reply_latency": { 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_reply": 0, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "forward": 0, 2024-05-20T20:19:53.186 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_complete": 0, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_keys": 0, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_commit": 11, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_split": 0, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_merge": 0, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes": 10, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_top": 0, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_bottom": 10, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pin_tail": 0, 2024-05-20T20:19:53.187 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pinned": 10, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_expired": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_with_caps": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "caps": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "subtrees": 1, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse": 18, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_hit": 18, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_forward": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_discover": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_remote_ino": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_lock": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "load_cent": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "q": 0, 2024-05-20T20:19:53.188 INFO:teuthology.orchestra.run.smithi033.stdout: "exported": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "exported_inodes": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "imported": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "imported_inodes": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_dir_fetch": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_peer_discover": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rfiles": 22, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rbytes": 10738143985, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rsnaps": 6, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_set_tag": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:19:53.189 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_file_inodes": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_revoke": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:19:53.190 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_cap_release": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "process_request_cap_release": 0 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_cache": { 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update_receipt": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_try_discover": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_send_discover": 1, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_handle_discover": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_delayed": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_created": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_enqueued": 0, 2024-05-20T20:19:53.191 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_reintegrated": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_migrated": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_processing": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_started": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_completed": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_path": 9, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_inode": 9, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_exportdir": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_flush": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragstats": 0, 2024-05-20T20:19:53.192 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_inodestats": 0 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_log": { 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "evlrg": 0, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "evadd": 6, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "evex": 0, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "evtrm": 0, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "ev": 6, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "evexg": 0, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "evexd": 0, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "segadd": 2, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "segex": 0, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "segtrm": 0, 2024-05-20T20:19:53.193 INFO:teuthology.orchestra.run.smithi033.stdout: "seg": 2, 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "segmjr": 2, 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "segexg": 0, 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "segexd": 0, 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "expos": 4194304, 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "wrpos": 4195956, 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "rdpos": 0, 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "jlat": { 2024-05-20T20:19:53.194 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 5, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.011222006, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.002244401 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "replayed": 0 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_mem": { 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "ino": 13, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "ino+": 13, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "ino-": 0, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "dir": 11, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "dir+": 12, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "dir-": 1, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "dn": 10, 2024-05-20T20:19:53.195 INFO:teuthology.orchestra.run.smithi033.stdout: "dn+": 10, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "dn-": 0, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "cap": 0, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "cap+": 0, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "cap-": 0, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "rss": 52596, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "heap": 223608 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_server": { 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_client_request": 0, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_server_request": 0, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_request": 0, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_session": 8, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_peer_request": 9, 2024-05-20T20:19:53.196 INFO:teuthology.orchestra.run.smithi033.stdout: "req_create_latency": { 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getattr_latency": { 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getfilelock_latency": { 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.197 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "req_link_latency": { 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookup_latency": { 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookuphash_latency": { 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.198 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupino_latency": { 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupname_latency": { 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupparent_latency": { 2024-05-20T20:19:53.199 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupsnap_latency": { 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lssnap_latency": { 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.200 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.201 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.201 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.201 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mkdir_latency": { 2024-05-20T20:19:53.201 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mknod_latency": { 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.202 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mksnap_latency": { 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "req_open_latency": { 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "req_readdir_latency": { 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.203 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rename_latency": { 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "req_renamesnap_latency": { 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "req_snapdiff_latency": { 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.204 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmdir_latency": { 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmsnap_latency": { 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmxattr_latency": { 2024-05-20T20:19:53.205 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setattr_latency": { 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setdirlayout_latency": { 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.206 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setfilelock_latency": { 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setlayout_latency": { 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setxattr_latency": { 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.207 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "req_symlink_latency": { 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "req_unlink_latency": { 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:19:53.208 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getvxattr_latency": { 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_sessions": { 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "session_count": 0, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "session_add": 2, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "session_remove": 2, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_open": 0, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_stale": 0, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "total_load": 0, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "average_load": 0, 2024-05-20T20:19:53.209 INFO:teuthology.orchestra.run.smithi033.stdout: "avg_session_uptime": 89, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "mdthresh_evicted": 0 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "mempool": { 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:53.210 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:53.211 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_items": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_bytes": 247751, 2024-05-20T20:19:53.212 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_items": 82, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_bytes": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_items": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_bytes": 8128, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_items": 169, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:53.213 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_items": 0, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_bytes": 50880, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_items": 199, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_items": 0, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_items": 0 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "objecter": { 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "op_active": 0, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "op_laggy": 0, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send": 42, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send_bytes": 12044, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "op_resend": 33, 2024-05-20T20:19:53.214 INFO:teuthology.orchestra.run.smithi033.stdout: "op_reply": 42, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "op_latency": { 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 42, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.363994264, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.008666530 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "op_inflight": 0, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "oplen_avg": { 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 42, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 64 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "op": 42, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "op_r": 0, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "op_w": 42, 2024-05-20T20:19:53.215 INFO:teuthology.orchestra.run.smithi033.stdout: "op_rmw": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "op_pg": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_stat": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_create": 10, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_read": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_write": 5, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writefull": 8, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writesame": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_append": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_zero": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_truncate": 0, 2024-05-20T20:19:53.216 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_delete": 6, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_mapext": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_sparse_read": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_clonerange": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_getxattr": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_setxattr": 20, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_rmxattr": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_call": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_watch": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_notify": 0, 2024-05-20T20:19:53.217 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_other": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_active": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_send": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_resend": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_ping": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_active": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_send": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_resend": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_active": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_send": 0, 2024-05-20T20:19:53.218 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_resend": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_active": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_send": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_resend": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "command_active": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "command_send": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "command_resend": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "map_epoch": 97, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "map_full": 0, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "map_inc": 19, 2024-05-20T20:19:53.219 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_sessions": 9, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_open": 11, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_close": 2, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_laggy": 0, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_wr": 14, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_rd": 0, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_del": 1 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "oft": { 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_objs": 1, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_updates": 0, 2024-05-20T20:19:53.220 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_removes": 0 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "osd-slow-ops": { 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_ops_count": 18 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "purge_queue": { 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops": 0, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing": 0, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_high_water": 0, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed_ops": 0, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed": 0, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_item_in_journal": 0 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.221 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 1022, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 2008216, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 1022, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 1022, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 2008216, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.222 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_bytes": { 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 42, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 12044, 2024-05-20T20:19:53.223 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 42, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 12044, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_ops": { 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 1024, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:53.224 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 42, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 42, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 42, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 42, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:19:53.225 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.226 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle-0x557878c52500": { 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 6, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 1652, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 6, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 5, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 1652, 2024-05-20T20:19:53.227 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:53.228 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:53.228 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:53.228 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:53.228 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.228 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:53.228 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-05-20T20:19:53.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:53 smithi033 ceph-mon[36179]: pgmap v446: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:53.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:53 smithi028 ceph-mon[31178]: pgmap v446: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:53.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:53 smithi154 ceph-mon[36314]: pgmap v446: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:54.031 INFO:tasks.check_counter:Daemon mds.h mds.exported=0 2024-05-20T20:19:54.031 INFO:tasks.check_counter:Daemon mds.h mds.imported=0 2024-05-20T20:19:54.031 INFO:tasks.check_counter:Daemon mds.h mds.root_rsnaps=6 2024-05-20T20:19:54.031 INFO:tasks.check_counter:Daemon mds.h mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:19:54.031 INFO:tasks.check_counter:Daemon mds.h mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:19:54.031 DEBUG:tasks.check_counter:Getting stats from k 2024-05-20T20:19:54.031 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.k perf dump 2024-05-20T20:19:54.278 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.b/config 2024-05-20T20:19:55.656 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:55 smithi033 ceph-mon[36179]: pgmap v447: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2024-05-20T20:19:55.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:55 smithi028 ceph-mon[31178]: pgmap v447: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2024-05-20T20:19:55.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:55 smithi154 ceph-mon[36314]: pgmap v447: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2024-05-20T20:19:56.090 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-05-20T20:19:56.090 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:56.090 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 1437, 2024-05-20T20:19:56.090 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 1423, 2024-05-20T20:19:56.090 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 2296176, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 542560, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 5, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 1, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.176343519, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.062662434, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.073169765, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.040922483, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 1422, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.063224324, 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000044461 2024-05-20T20:19:56.091 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 1930608, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 196576 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 8, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 242, 2024-05-20T20:19:56.092 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 3868, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 568042, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 6, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 4, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.013691020, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.008867156, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.002304805, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.000077750, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 240, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.019563511, 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000081514 2024-05-20T20:19:56.093 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 3312, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 567344 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_messages": 2077, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages": 2079, 2024-05-20T20:19:56.094 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_bytes": 530647, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_bytes": 660493, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_created_connections": 5, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_total_time": 0.120742870, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_send_time": 0.029400693, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_recv_time": 0.032667574, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_running_fast_dispatch_time": 0.050618105, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 2077, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.049431157, 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000023799 2024-05-20T20:19:56.095 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_recv_encrypted_bytes": 0, 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "msgr_send_encrypted_bytes": 0 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "cct": { 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "total_workers": 1, 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: "unhealthy_workers": 0 2024-05-20T20:19:56.096 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-MDSRank": { 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 5332, 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.129378002, 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000024264 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "finisher-PurgeQueue": { 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "queue_len": 0, 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "complete_latency": { 2024-05-20T20:19:56.097 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "request": 0, 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "reply": 0, 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "reply_latency": { 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.098 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_reply": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "forward": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_complete": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_fetch_keys": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_commit": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_split": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_merge": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_top": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_bottom": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pin_tail": 0, 2024-05-20T20:19:56.099 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_pinned": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_expired": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "inodes_with_caps": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "caps": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "subtrees": 1, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_hit": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_forward": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_discover": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_remote_ino": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "traverse_lock": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "load_cent": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "q": 0, 2024-05-20T20:19:56.100 INFO:teuthology.orchestra.run.smithi033.stdout: "exported": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "exported_inodes": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "imported": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "imported_inodes": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_dir_fetch": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "openino_peer_discover": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rfiles": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rbytes": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "root_rsnaps": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_set_tag": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:19:56.101 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "scrub_file_inodes": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_revoke": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps": 0, 2024-05-20T20:19:56.102 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_cap_release": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "process_request_cap_release": 0 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_cache": { 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_update_receipt": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_try_discover": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_send_discover": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "dir_handle_discover": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_delayed": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_created": 0, 2024-05-20T20:19:56.103 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_enqueued": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_reintegrated": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "strays_migrated": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_processing": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_started": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "recovery_completed": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_path": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_quiesce_inode": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_exportdir": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_flush": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:19:56.104 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_fragstats": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "ireq_inodestats": 0 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_log": { 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "evlrg": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "evadd": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "evex": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "evtrm": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "ev": 6, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "evexg": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "evexd": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "segadd": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "segex": 0, 2024-05-20T20:19:56.105 INFO:teuthology.orchestra.run.smithi033.stdout: "segtrm": 0, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "seg": 2, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "segmjr": 2, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "segexg": 0, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "segexd": 0, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "expos": 4194304, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "wrpos": 4195956, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "rdpos": 4195815, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "jlat": { 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.106 INFO:teuthology.orchestra.run.smithi033.stdout: "replayed": 6 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_mem": { 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "ino": 3, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "ino+": 3, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "ino-": 0, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "dir": 1, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "dir+": 1, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "dir-": 0, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "dn": 0, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "dn+": 0, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "dn-": 0, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "cap": 0, 2024-05-20T20:19:56.107 INFO:teuthology.orchestra.run.smithi033.stdout: "cap+": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "cap-": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "rss": 50120, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "heap": 215416 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_server": { 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_client_request": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "dispatch_server_request": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_request": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_client_session": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "handle_peer_request": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "req_create_latency": { 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.108 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getattr_latency": { 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getfilelock_latency": { 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "req_link_latency": { 2024-05-20T20:19:56.109 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookup_latency": { 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookuphash_latency": { 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.110 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupino_latency": { 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupname_latency": { 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupparent_latency": { 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.111 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lookupsnap_latency": { 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "req_lssnap_latency": { 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mkdir_latency": { 2024-05-20T20:19:56.112 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.113 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.113 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.113 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.113 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mknod_latency": { 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "req_mksnap_latency": { 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "req_open_latency": { 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.114 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "req_readdir_latency": { 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rename_latency": { 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.115 INFO:teuthology.orchestra.run.smithi033.stdout: "req_renamesnap_latency": { 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "req_snapdiff_latency": { 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmdir_latency": { 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.116 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmsnap_latency": { 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "req_rmxattr_latency": { 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setattr_latency": { 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.117 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setdirlayout_latency": { 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setfilelock_latency": { 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.118 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setlayout_latency": { 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "req_setxattr_latency": { 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "req_symlink_latency": { 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.119 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "req_unlink_latency": { 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "req_getvxattr_latency": { 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.120 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_sessions": { 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "session_count": 0, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "session_add": 2, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "session_remove": 2, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_open": 1, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "sessions_stale": 0, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "total_load": 0, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "average_load": 0, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "avg_session_uptime": 1800, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "mdthresh_evicted": 0 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.121 INFO:teuthology.orchestra.run.smithi033.stdout: "mempool": { 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:56.122 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:56.123 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_items": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_bytes": 182860, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_anon_items": 51, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:56.124 INFO:teuthology.orchestra.run.smithi033.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_bytes": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_items": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_bytes": 8128, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_items": 169, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:56.125 INFO:teuthology.orchestra.run.smithi033.stdout: "pgmap_items": 0, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_bytes": 8512, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "mds_co_items": 13, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_1_items": 0, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "unittest_2_items": 0 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "objecter": { 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "op_active": 0, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "op_laggy": 0, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send": 3167, 2024-05-20T20:19:56.126 INFO:teuthology.orchestra.run.smithi033.stdout: "op_send_bytes": 21, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "op_resend": 3, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "op_reply": 3167, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "op_latency": { 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 3167, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 2.012968582, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000635607 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "op_inflight": 0, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "oplen_avg": { 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 3167, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 3168 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.127 INFO:teuthology.orchestra.run.smithi033.stdout: "op": 3167, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "op_r": 3167, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "op_w": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "op_rmw": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "op_pg": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_stat": 2069, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_create": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_read": 1096, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_write": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writefull": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_writesame": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_append": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_zero": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_truncate": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_delete": 0, 2024-05-20T20:19:56.128 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_mapext": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_sparse_read": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_clonerange": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_getxattr": 1, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_setxattr": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_rmxattr": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_call": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_watch": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_notify": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls": 0, 2024-05-20T20:19:56.129 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "osdop_other": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_active": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_send": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_resend": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "linger_ping": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_active": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_send": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "poolop_resend": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_active": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_send": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "poolstat_resend": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_active": 0, 2024-05-20T20:19:56.130 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_send": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "statfs_resend": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "command_active": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "command_send": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "command_resend": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "map_epoch": 97, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "map_full": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "map_inc": 19, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_sessions": 4, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_open": 5, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_session_close": 1, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "osd_laggy": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_wr": 0, 2024-05-20T20:19:56.131 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_rd": 2, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_del": 0 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "oft": { 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_objs": 0, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_updates": 0, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "omap_total_removes": 0 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "osd-slow-ops": { 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "slow_ops_count": 0 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "purge_queue": { 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops": 0, 2024-05-20T20:19:56.132 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing": 0, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executing_high_water": 0, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed_ops": 0, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_executed": 0, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "pq_item_in_journal": 0 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 3518, 2024-05-20T20:19:56.133 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 2493190, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 3518, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 3518, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 2493190, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.134 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_bytes": { 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 104857600, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 3167, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 1657, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 3167, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 1657, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.135 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-objecter_ops": { 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 1024, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 3167, 2024-05-20T20:19:56.136 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 3167, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 3167, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 3167, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:56.137 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.138 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "throttle-write_buf_throttle-0x555a5366e6e0": { 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "val": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "max": 3758096384, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "get_started": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "get": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "get_sum": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "take": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "take_sum": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "put": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "put_sum": 0, 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "wait": { 2024-05-20T20:19:56.139 INFO:teuthology.orchestra.run.smithi033.stdout: "avgcount": 0, 2024-05-20T20:19:56.140 INFO:teuthology.orchestra.run.smithi033.stdout: "sum": 0.000000000, 2024-05-20T20:19:56.140 INFO:teuthology.orchestra.run.smithi033.stdout: "avgtime": 0.000000000 2024-05-20T20:19:56.140 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.140 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-05-20T20:19:56.140 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-05-20T20:19:56.871 INFO:tasks.check_counter:Daemon mds.k mds.exported=0 2024-05-20T20:19:56.872 INFO:tasks.check_counter:Daemon mds.k mds.imported=0 2024-05-20T20:19:56.872 INFO:tasks.check_counter:Daemon mds.k mds.root_rsnaps=0 2024-05-20T20:19:56.872 INFO:tasks.check_counter:Daemon mds.k mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:19:56.872 INFO:tasks.check_counter:Daemon mds.k mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:19:56.872 DEBUG:tasks.check_counter:Getting stats from c 2024-05-20T20:19:56.872 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.c perf dump 2024-05-20T20:19:57.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:56 smithi028 ceph-mon[31178]: pgmap v448: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:57.123 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T20:19:57.154 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:56 smithi033 ceph-mon[36179]: pgmap v448: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:57.154 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:56 smithi154 ceph-mon[36314]: pgmap v448: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:19:58.846 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-05-20T20:19:58.846 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:19:58.846 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 826, 2024-05-20T20:19:58.846 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 918, 2024-05-20T20:19:58.846 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 215392, 2024-05-20T20:19:58.846 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 357430, 2024-05-20T20:19:58.846 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 13, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 3, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.086376387, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.030964153, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.046439803, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.006754739, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 917, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.047245257, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000051521 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:58.850 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 1659, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000378334, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000228 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 2992, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 448 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 392, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 549, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 1945481, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 656808, 2024-05-20T20:19:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 12, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 4, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.068131814, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.024368551, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.044422907, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.007799085, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 548, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.054048089, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000098627 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 4, 2024-05-20T20:19:58.852 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000399, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000099 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 1942400, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 597888 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 63, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 149, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 15013, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 560390, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 36, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 8, 2024-05-20T20:19:58.853 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.024812531, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.006221643, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.016034074, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.000372744, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 146, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.019235063, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000131747 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 84, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000016685, 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000198 2024-05-20T20:19:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 6880, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 261280 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "cct": { 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "total_workers": 1, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "unhealthy_workers": 0 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-MDSRank": { 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 57, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.001684921, 2024-05-20T20:19:58.855 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000029560 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-PurgeQueue": { 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 2, 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000008289, 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000004144 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "request": 4, 2024-05-20T20:19:58.856 INFO:teuthology.orchestra.run.smithi154.stdout: "reply": 3, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "reply_latency": { 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 3, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000993816, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000331272 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_reply": 0, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "forward": 1, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_complete": 0, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_keys": 0, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_commit": 11, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_split": 0, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_merge": 0, 2024-05-20T20:19:58.857 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes": 11, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_top": 0, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_bottom": 11, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pin_tail": 0, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pinned": 10, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_expired": 1, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_with_caps": 0, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "caps": 0, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "subtrees": 2, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse": 26, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_hit": 22, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_forward": 0, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_discover": 2, 2024-05-20T20:19:58.858 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_remote_ino": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_lock": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "load_cent": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "q": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "exported": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "exported_inodes": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "imported": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "imported_inodes": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_dir_fetch": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_peer_discover": 1, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rfiles": 22, 2024-05-20T20:19:58.859 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rbytes": 10738143985, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rsnaps": 6, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_set_tag": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_file_inodes": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_revoke": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:19:58.860 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_cap_release": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "process_request_cap_release": 0 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_cache": { 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update_receipt": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_try_discover": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_send_discover": 3, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_handle_discover": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays": 0, 2024-05-20T20:19:58.861 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_delayed": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_created": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_enqueued": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_reintegrated": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_migrated": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_processing": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_started": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_completed": 0, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_path": 9, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_inode": 18, 2024-05-20T20:19:58.862 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_exportdir": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_flush": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragstats": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_inodestats": 0 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_log": { 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "evlrg": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "evadd": 5, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "evex": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "evtrm": 0, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "ev": 5, 2024-05-20T20:19:58.863 INFO:teuthology.orchestra.run.smithi154.stdout: "evexg": 0, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "evexd": 0, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "segadd": 2, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "segex": 0, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "segtrm": 0, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "seg": 2, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "segmjr": 2, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "segexg": 0, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "segexd": 0, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "expos": 4194304, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "wrpos": 4195734, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "rdpos": 0, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "jlat": { 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 4, 2024-05-20T20:19:58.864 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.012208248, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.003052062 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "replayed": 0 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_mem": { 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "ino": 14, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "ino+": 14, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "ino-": 0, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "dir": 13, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "dir+": 14, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "dir-": 1, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "dn": 11, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "dn+": 12, 2024-05-20T20:19:58.865 INFO:teuthology.orchestra.run.smithi154.stdout: "dn-": 1, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "cap": 0, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "cap+": 2, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "cap-": 2, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "rss": 55732, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "heap": 223608 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_server": { 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_client_request": 6, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_server_request": 0, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_request": 4, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_session": 16, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_peer_request": 18, 2024-05-20T20:19:58.866 INFO:teuthology.orchestra.run.smithi154.stdout: "req_create_latency": { 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getattr_latency": { 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 1, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000190131, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000190131 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getfilelock_latency": { 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.867 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "req_link_latency": { 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookup_latency": { 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 2, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000803685, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000401842 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookuphash_latency": { 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.868 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupino_latency": { 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupname_latency": { 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupparent_latency": { 2024-05-20T20:19:58.869 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupsnap_latency": { 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lssnap_latency": { 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.870 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mkdir_latency": { 2024-05-20T20:19:58.871 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.871 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mknod_latency": { 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mksnap_latency": { 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "req_open_latency": { 2024-05-20T20:19:58.872 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "req_readdir_latency": { 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rename_latency": { 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.873 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "req_renamesnap_latency": { 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "req_snapdiff_latency": { 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmdir_latency": { 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.874 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmsnap_latency": { 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmxattr_latency": { 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setattr_latency": { 2024-05-20T20:19:58.875 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setdirlayout_latency": { 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setfilelock_latency": { 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.876 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setlayout_latency": { 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setxattr_latency": { 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "req_symlink_latency": { 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.877 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "req_unlink_latency": { 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getvxattr_latency": { 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_sessions": { 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "session_count": 1, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "session_add": 2, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "session_remove": 1, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_open": 1, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_stale": 0, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "total_load": 2, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "average_load": 2, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "avg_session_uptime": 806, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "mdthresh_evicted": 0 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.879 INFO:teuthology.orchestra.run.smithi154.stdout: "mempool": { 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_items": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:19:58.880 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_items": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_items": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_items": 0, 2024-05-20T20:19:58.881 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_items": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_bytes": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_items": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_bytes": 256135, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_items": 89, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:19:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_items": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_bytes": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_items": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_items": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_items": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_bytes": 8128, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_items": 169, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_bytes": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_items": 0, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_bytes": 56544, 2024-05-20T20:19:58.883 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_items": 224, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_bytes": 0, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_items": 0, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_bytes": 0, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_items": 0 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "objecter": { 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "op_active": 0, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "op_laggy": 0, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send": 43, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send_bytes": 11967, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "op_resend": 33, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "op_reply": 43, 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "op_latency": { 2024-05-20T20:19:58.884 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 43, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.375990031, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.008743954 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "op_inflight": 0, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "oplen_avg": { 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 43, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 67 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "op": 43, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "op_r": 0, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "op_w": 43, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "op_rmw": 0, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "op_pg": 0, 2024-05-20T20:19:58.885 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_stat": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_create": 10, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_read": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_write": 4, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writefull": 8, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writesame": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_append": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_zero": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_truncate": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_delete": 6, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_mapext": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_sparse_read": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_clonerange": 0, 2024-05-20T20:19:58.886 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_getxattr": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_setxattr": 20, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_rmxattr": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_call": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_watch": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_notify": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_other": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_active": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_send": 0, 2024-05-20T20:19:58.887 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_resend": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_ping": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_active": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_send": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_resend": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_active": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_send": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_resend": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_active": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_send": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_resend": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "command_active": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "command_send": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "command_resend": 0, 2024-05-20T20:19:58.888 INFO:teuthology.orchestra.run.smithi154.stdout: "map_epoch": 97, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "map_full": 0, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "map_inc": 19, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_sessions": 9, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_open": 13, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_close": 4, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_laggy": 0, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_wr": 17, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_rd": 0, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_del": 2 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "oft": { 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_objs": 1, 2024-05-20T20:19:58.889 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_updates": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_removes": 0 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "osd-slow-ops": { 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_ops_count": 27 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "purge_queue": { 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_high_water": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed_ops": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed": 0, 2024-05-20T20:19:58.890 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_item_in_journal": 0 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 1277, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 2037817, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 1277, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 1277, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 2037817, 2024-05-20T20:19:58.891 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_bytes": { 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:58.892 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 43, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 11967, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 43, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 11967, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_ops": { 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 1024, 2024-05-20T20:19:58.893 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 43, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 43, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 43, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 43, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 0, 2024-05-20T20:19:58.895 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle-0x55fd6f946500": { 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 5, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 1430, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:19:58.896 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 5, 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 4, 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 1430, 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:19:58.897 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-05-20T20:19:59.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:19:59 smithi028 ceph-mon[31178]: pgmap v449: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:19:59.596 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:19:59 smithi154 ceph-mon[36314]: pgmap v449: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:19:59.624 INFO:tasks.check_counter:Daemon mds.c mds.exported=0 2024-05-20T20:19:59.624 INFO:tasks.check_counter:Daemon mds.c mds.imported=0 2024-05-20T20:19:59.624 INFO:tasks.check_counter:Daemon mds.c mds.root_rsnaps=6 2024-05-20T20:19:59.624 INFO:tasks.check_counter:Daemon mds.c mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:19:59.624 INFO:tasks.check_counter:Daemon mds.c mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:19:59.624 DEBUG:tasks.check_counter:Getting stats from f 2024-05-20T20:19:59.625 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.f perf dump 2024-05-20T20:19:59.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:19:59 smithi033 ceph-mon[36179]: pgmap v449: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:19:59.870 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T20:20:00.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:00 smithi154 ceph-mon[36314]: overall HEALTH_OK 2024-05-20T20:20:00.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:00 smithi033 ceph-mon[36179]: overall HEALTH_OK 2024-05-20T20:20:00.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:00 smithi028 ceph-mon[31178]: overall HEALTH_OK 2024-05-20T20:20:01.575 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:01 smithi154 ceph-mon[36314]: pgmap v450: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 12 op/s 2024-05-20T20:20:01.611 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 2020, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 2022, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 8200927, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 642370, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 6, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 2, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.190182298, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.056114360, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.054343778, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.062528707, 2024-05-20T20:20:01.612 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 2020, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.073103035, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000036189 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 0, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 0 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.613 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 969, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 955, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 10516565, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 392678, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 4, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 3, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.111156769, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.030859652, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.063472288, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.024722044, 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:01.614 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 954, 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.033997480, 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000035636 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 1929952, 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 194816 2024-05-20T20:20:01.615 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 887, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 1119, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 4431687, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 850383, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 7, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 5, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.092595952, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.035506549, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.021329099, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.026726097, 2024-05-20T20:20:01.616 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 1117, 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.056593066, 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000050665 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.617 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 3312, 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 570224 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "cct": { 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "total_workers": 1, 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "unhealthy_workers": 0 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-MDSRank": { 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 5892, 2024-05-20T20:20:01.618 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.163719486, 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000027786 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-PurgeQueue": { 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-05-20T20:20:01.619 INFO:teuthology.orchestra.run.smithi154.stdout: "request": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "reply": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "reply_latency": { 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_reply": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "forward": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_complete": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_keys": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_commit": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_split": 0, 2024-05-20T20:20:01.620 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_merge": 0, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes": 170, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_top": 110, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_bottom": 60, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pin_tail": 0, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pinned": 31, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_expired": 0, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_with_caps": 0, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "caps": 0, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "subtrees": 2, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse": 0, 2024-05-20T20:20:01.621 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_hit": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_forward": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_discover": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_remote_ino": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_lock": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "load_cent": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "q": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "exported": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "exported_inodes": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "imported": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "imported_inodes": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_dir_fetch": 0, 2024-05-20T20:20:01.622 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_peer_discover": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rfiles": 4, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rbytes": 234, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rsnaps": 6, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_set_tag": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_file_inodes": 0, 2024-05-20T20:20:01.623 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_revoke": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_cap_release": 0, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "process_request_cap_release": 0 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_cache": { 2024-05-20T20:20:01.624 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update_receipt": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_try_discover": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_send_discover": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_handle_discover": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_delayed": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_created": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_enqueued": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_reintegrated": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_migrated": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_processing": 0, 2024-05-20T20:20:01.625 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_started": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_completed": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_path": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_inode": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_exportdir": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_flush": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragstats": 0, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_inodestats": 0 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.626 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_log": { 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "evlrg": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "evadd": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "evex": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "evtrm": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "ev": 2516, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "evexg": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "evexd": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "segadd": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "segex": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "segtrm": 0, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "seg": 2, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "segmjr": 59, 2024-05-20T20:20:01.627 INFO:teuthology.orchestra.run.smithi154.stdout: "segexg": 0, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "segexd": 0, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "expos": 24197530, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "wrpos": 24372010, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "rdpos": 24367847, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "jlat": { 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "replayed": 2516 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_mem": { 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "ino": 127, 2024-05-20T20:20:01.628 INFO:teuthology.orchestra.run.smithi154.stdout: "ino+": 145, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "ino-": 18, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "dir": 21, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "dir+": 22, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "dir-": 1, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "dn": 170, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "dn+": 170, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "dn-": 0, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "cap": 0, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "cap+": 0, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "cap-": 0, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "rss": 51948, 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: "heap": 215416 2024-05-20T20:20:01.629 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_server": { 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_client_request": 0, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_server_request": 0, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_request": 0, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_session": 0, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_peer_request": 0, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "req_create_latency": { 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.630 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getattr_latency": { 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getfilelock_latency": { 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "req_link_latency": { 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.631 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookup_latency": { 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookuphash_latency": { 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupino_latency": { 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.632 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupname_latency": { 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupparent_latency": { 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupsnap_latency": { 2024-05-20T20:20:01.633 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lssnap_latency": { 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.634 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mkdir_latency": { 2024-05-20T20:20:01.635 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:01 smithi033 ceph-mon[36179]: pgmap v450: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 12 op/s 2024-05-20T20:20:01.635 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.635 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.635 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.635 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.635 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mknod_latency": { 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mksnap_latency": { 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "req_open_latency": { 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.636 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "req_readdir_latency": { 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rename_latency": { 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.637 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "req_renamesnap_latency": { 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "req_snapdiff_latency": { 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.638 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmdir_latency": { 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmsnap_latency": { 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmxattr_latency": { 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.639 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setattr_latency": { 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setdirlayout_latency": { 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setfilelock_latency": { 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.640 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setlayout_latency": { 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setxattr_latency": { 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.641 INFO:teuthology.orchestra.run.smithi154.stdout: "req_symlink_latency": { 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "req_unlink_latency": { 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getvxattr_latency": { 2024-05-20T20:20:01.642 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_sessions": { 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "session_count": 2, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "session_add": 5, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "session_remove": 3, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_open": 4, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_stale": 0, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "total_load": 0, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "average_load": 0, 2024-05-20T20:20:01.643 INFO:teuthology.orchestra.run.smithi154.stdout: "avg_session_uptime": 4207, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "mdthresh_evicted": 0 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "mempool": { 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_items": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:20:01.644 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_items": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_items": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:20:01.645 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_items": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_items": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_bytes": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_items": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:20:01.646 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_bytes": 203850, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_items": 189, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_items": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_bytes": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_items": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_items": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_items": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_bytes": 8128, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_items": 169, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:20:01.647 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_bytes": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_items": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_bytes": 485744, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_items": 5108, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_bytes": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_items": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_bytes": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_items": 0 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "objecter": { 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "op_active": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "op_laggy": 0, 2024-05-20T20:20:01.648 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send": 3523, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send_bytes": 26, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "op_resend": 4, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "op_reply": 3523, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "op_latency": { 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 3523, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 7.423809691, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.002107240 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "op_inflight": 0, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "oplen_avg": { 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 3523, 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 3524 2024-05-20T20:20:01.649 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "op": 3523, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "op_r": 3523, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "op_w": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "op_rmw": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "op_pg": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_stat": 2168, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_create": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_read": 1352, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_write": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writefull": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writesame": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_append": 0, 2024-05-20T20:20:01.650 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_zero": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_truncate": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_delete": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_mapext": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_sparse_read": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_clonerange": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_getxattr": 2, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_setxattr": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_rmxattr": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_call": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_watch": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_notify": 0, 2024-05-20T20:20:01.651 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_other": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_active": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_send": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_resend": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_ping": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_active": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_send": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_resend": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_active": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_send": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_resend": 0, 2024-05-20T20:20:01.652 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_active": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_send": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_resend": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "command_active": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "command_send": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "command_resend": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "map_epoch": 97, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "map_full": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "map_inc": 19, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_sessions": 6, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_open": 6, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_close": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_laggy": 0, 2024-05-20T20:20:01.653 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_wr": 0, 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_rd": 2, 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_del": 0 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "oft": { 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_objs": 0, 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_updates": 0, 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_removes": 0 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "osd-slow-ops": { 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_ops_count": 0 2024-05-20T20:20:01.654 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "purge_queue": { 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops": 0, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing": 0, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_high_water": 0, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed_ops": 0, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed": 0, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_item_in_journal": 0 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:20:01.655 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 3872, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 22779928, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 3872, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 3872, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 22779928, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.656 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_bytes": { 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 3523, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 20232239, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 3523, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 20232239, 2024-05-20T20:20:01.657 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_ops": { 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 1024, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:01.658 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 3523, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 3523, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 3523, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 3523, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:01.659 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:01.660 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle-0x562bc726e6e0": { 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 0, 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:01.661 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:01.662 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:01.662 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:01.662 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.662 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:01.662 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-05-20T20:20:01.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:01 smithi028 ceph-mon[31178]: pgmap v450: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 12 op/s 2024-05-20T20:20:02.387 INFO:tasks.check_counter:Daemon mds.f mds.exported=0 2024-05-20T20:20:02.387 INFO:tasks.check_counter:Daemon mds.f mds.imported=0 2024-05-20T20:20:02.387 INFO:tasks.check_counter:Daemon mds.f mds.root_rsnaps=6 2024-05-20T20:20:02.387 INFO:tasks.check_counter:Daemon mds.f mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:20:02.387 INFO:tasks.check_counter:Daemon mds.f mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:20:02.387 DEBUG:tasks.check_counter:Getting stats from i 2024-05-20T20:20:02.388 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.i perf dump 2024-05-20T20:20:02.632 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T20:20:03.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:03 smithi154 ceph-mon[36314]: pgmap v451: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:20:03.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:03 smithi033 ceph-mon[36179]: pgmap v451: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:20:03.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:03 smithi028 ceph-mon[31178]: pgmap v451: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2024-05-20T20:20:04.273 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-05-20T20:20:04.273 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:20:04.273 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 1972, 2024-05-20T20:20:04.273 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 1629, 2024-05-20T20:20:04.273 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 2358814, 2024-05-20T20:20:04.273 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 750934, 2024-05-20T20:20:04.273 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 8, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 2, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.169316449, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.060039086, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.111455810, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.014535468, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 1630, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.072741286, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000044626 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.274 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 2752, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000773995, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000281 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 1955104, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 207088 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 56, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 291, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 14178, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 857559, 2024-05-20T20:20:04.275 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 34, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 5, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.026225348, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.010429268, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.011733980, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.000578104, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 289, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.034410761, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000119068 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 37, 2024-05-20T20:20:04.276 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000005469, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000147 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 5168, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 575008 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 60, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 128, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 18285, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 158008, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 20, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 8, 2024-05-20T20:20:04.277 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.019804448, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.004842364, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.013740997, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.000313576, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 124, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.028141175, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000226944 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 109, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000029060, 2024-05-20T20:20:04.278 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000266 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 13312, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 29840 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "cct": { 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "total_workers": 1, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "unhealthy_workers": 0 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-MDSRank": { 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 67, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.002460857, 2024-05-20T20:20:04.279 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000036729 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-PurgeQueue": { 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 2, 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000008891, 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000004445 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "request": 9, 2024-05-20T20:20:04.280 INFO:teuthology.orchestra.run.smithi154.stdout: "reply": 6, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "reply_latency": { 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 6, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.004039396, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000673232 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_reply": 0, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "forward": 3, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_complete": 0, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_keys": 0, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_commit": 11, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_split": 0, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_merge": 0, 2024-05-20T20:20:04.281 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes": 14, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_top": 0, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_bottom": 14, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pin_tail": 0, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pinned": 14, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_expired": 0, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_with_caps": 1, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "caps": 1, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "subtrees": 2, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse": 32, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_hit": 29, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_forward": 1, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_discover": 1, 2024-05-20T20:20:04.282 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_remote_ino": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_lock": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "load_cent": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "q": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "exported": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "exported_inodes": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "imported": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "imported_inodes": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_dir_fetch": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_peer_discover": 0, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rfiles": 22, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rbytes": 10738143985, 2024-05-20T20:20:04.283 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rsnaps": 6, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_set_tag": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_file_inodes": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_revoke": 13, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_grant": 14, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:20:04.284 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps": 15, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_cap_release": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "process_request_cap_release": 1 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_cache": { 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update_receipt": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_try_discover": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_send_discover": 2, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_handle_discover": 0, 2024-05-20T20:20:04.285 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_delayed": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_created": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_enqueued": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_reintegrated": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_migrated": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_processing": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_started": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_completed": 0, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_path": 9, 2024-05-20T20:20:04.286 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_inode": 45, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_exportdir": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_flush": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragstats": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_inodestats": 0 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_log": { 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "evlrg": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "evadd": 8, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "evex": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "evtrm": 0, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "ev": 8, 2024-05-20T20:20:04.287 INFO:teuthology.orchestra.run.smithi154.stdout: "evexg": 0, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "evexd": 0, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "segadd": 2, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "segex": 0, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "segtrm": 0, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "seg": 2, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "segmjr": 2, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "segexg": 0, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "segexd": 0, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "expos": 4194304, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "wrpos": 4196657, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "rdpos": 0, 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "jlat": { 2024-05-20T20:20:04.288 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 7, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.016163431, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.002309061 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "replayed": 0 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_mem": { 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "ino": 17, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "ino+": 17, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "ino-": 0, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "dir": 15, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "dir+": 16, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "dir-": 1, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "dn": 14, 2024-05-20T20:20:04.289 INFO:teuthology.orchestra.run.smithi154.stdout: "dn+": 14, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "dn-": 0, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "cap": 1, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "cap+": 6, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "cap-": 5, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "rss": 53544, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "heap": 223608 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_server": { 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_client_request": 13, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_server_request": 0, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_request": 9, 2024-05-20T20:20:04.290 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_session": 31, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_peer_request": 46, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "req_create_latency": { 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getattr_latency": { 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 3, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.003371199, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.001123733 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getfilelock_latency": { 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.291 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "req_link_latency": { 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookup_latency": { 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 3, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000668197, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000222732 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookuphash_latency": { 2024-05-20T20:20:04.292 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupino_latency": { 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupname_latency": { 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.293 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupparent_latency": { 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupsnap_latency": { 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.294 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lssnap_latency": { 2024-05-20T20:20:04.295 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.295 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.295 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.295 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.295 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mkdir_latency": { 2024-05-20T20:20:04.295 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mknod_latency": { 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mksnap_latency": { 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.296 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "req_open_latency": { 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "req_readdir_latency": { 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rename_latency": { 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.297 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "req_renamesnap_latency": { 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "req_snapdiff_latency": { 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.298 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmdir_latency": { 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmsnap_latency": { 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmxattr_latency": { 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.299 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setattr_latency": { 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setdirlayout_latency": { 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setfilelock_latency": { 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.300 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setlayout_latency": { 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setxattr_latency": { 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.301 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "req_symlink_latency": { 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "req_unlink_latency": { 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:20:04.302 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getvxattr_latency": { 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_sessions": { 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "session_count": 2, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "session_add": 4, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "session_remove": 2, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_open": 2, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_stale": 0, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "total_load": 5, 2024-05-20T20:20:04.303 INFO:teuthology.orchestra.run.smithi154.stdout: "average_load": 1, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "avg_session_uptime": 852, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "mdthresh_evicted": 0 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "mempool": { 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_items": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:20:04.304 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_items": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_items": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:20:04.305 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_items": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_items": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_bytes": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_items": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:20:04.306 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_bytes": 256103, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_items": 86, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_items": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_bytes": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_items": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_items": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_items": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_bytes": 8128, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_items": 169, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:20:04.307 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_bytes": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_items": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_bytes": 68133, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_items": 300, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_bytes": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_items": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_bytes": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_items": 0 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "objecter": { 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "op_active": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "op_laggy": 0, 2024-05-20T20:20:04.308 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send": 51, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send_bytes": 13703, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "op_resend": 28, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "op_reply": 51, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "op_latency": { 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 51, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.433988463, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.008509577 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "op_inflight": 0, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "oplen_avg": { 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 51, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 80 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.309 INFO:teuthology.orchestra.run.smithi154.stdout: "op": 51, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "op_r": 0, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "op_w": 51, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "op_rmw": 0, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "op_pg": 0, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_stat": 0, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_create": 10, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_read": 0, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_write": 7, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writefull": 8, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writesame": 0, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_append": 0, 2024-05-20T20:20:04.310 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_zero": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_truncate": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_delete": 6, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_mapext": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_sparse_read": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_clonerange": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_getxattr": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_setxattr": 20, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_rmxattr": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_call": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_watch": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_notify": 0, 2024-05-20T20:20:04.311 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_other": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_active": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_send": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_resend": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_ping": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_active": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_send": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_resend": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_active": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_send": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_resend": 0, 2024-05-20T20:20:04.312 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_active": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_send": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_resend": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "command_active": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "command_send": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "command_resend": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "map_epoch": 97, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "map_full": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "map_inc": 19, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_sessions": 8, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_open": 13, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_close": 5, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_laggy": 0, 2024-05-20T20:20:04.313 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_wr": 27, 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_rd": 0, 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_del": 2 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "oft": { 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_objs": 1, 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_updates": 0, 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_removes": 0 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "osd-slow-ops": { 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_ops_count": 54 2024-05-20T20:20:04.314 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "purge_queue": { 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops": 0, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing": 0, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_high_water": 0, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed_ops": 0, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed": 0, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_item_in_journal": 0 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:20:04.315 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 2084, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 2179149, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 2084, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 2084, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 2179149, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.316 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_bytes": { 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 51, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 13703, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 51, 2024-05-20T20:20:04.317 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 13703, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_ops": { 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 1024, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:04.318 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 51, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 51, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 51, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 51, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:20:04.319 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.320 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle-0x5600771e8500": { 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 8, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 2353, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 8, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:04.321 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 7, 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 2353, 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:04.322 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-05-20T20:20:05.063 INFO:tasks.check_counter:Daemon mds.i mds.exported=0 2024-05-20T20:20:05.064 INFO:tasks.check_counter:Daemon mds.i mds.imported=0 2024-05-20T20:20:05.064 INFO:tasks.check_counter:Daemon mds.i mds.root_rsnaps=6 2024-05-20T20:20:05.064 INFO:tasks.check_counter:Daemon mds.i mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:20:05.064 INFO:tasks.check_counter:Daemon mds.i mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:20:05.064 DEBUG:tasks.check_counter:Getting stats from l 2024-05-20T20:20:05.064 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph daemon mds.l perf dump 2024-05-20T20:20:05.304 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.c/config 2024-05-20T20:20:05.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:05 smithi154 ceph-mon[36314]: pgmap v452: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 45 KiB/s wr, 29 op/s 2024-05-20T20:20:05.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:05 smithi033 ceph-mon[36179]: pgmap v452: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 45 KiB/s wr, 29 op/s 2024-05-20T20:20:05.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:05 smithi028 ceph-mon[31178]: pgmap v452: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 45 KiB/s wr, 29 op/s 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-0": { 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 1452, 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 1438, 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 2300960, 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 548618, 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 5, 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 1, 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.180187705, 2024-05-20T20:20:06.982 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.055368963, 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.077108668, 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.047840626, 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 1437, 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.064357923, 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000044786 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:06.983 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 1931696, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 199136 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-1": { 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 877, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 1114, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 218230, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 849766, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 8, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 5, 2024-05-20T20:20:06.984 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.089599460, 2024-05-20T20:20:06.985 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.034427587, 2024-05-20T20:20:06.985 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.018598849, 2024-05-20T20:20:06.985 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.028482517, 2024-05-20T20:20:06.985 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:06.985 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 1112, 2024-05-20T20:20:06.986 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.055136075, 2024-05-20T20:20:06.986 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000049582 2024-05-20T20:20:06.986 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.986 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:06.987 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:06.987 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:06.987 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:06.987 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.987 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 3312, 2024-05-20T20:20:06.987 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 572720 2024-05-20T20:20:06.987 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "AsyncMessenger::Worker-2": { 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_messages": 1105, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages": 1107, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_bytes": 290901, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_bytes": 351391, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_created_connections": 5, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_active_connections": 3, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_total_time": 0.091222081, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_send_time": 0.036138249, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_recv_time": 0.021889945, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_running_fast_dispatch_time": 0.024655124, 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_messages_queue_lat": { 2024-05-20T20:20:06.988 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 1105, 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.033661525, 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000030462 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_handle_ack_lat": { 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_recv_encrypted_bytes": 0, 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: "msgr_send_encrypted_bytes": 0 2024-05-20T20:20:06.989 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "cct": { 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "total_workers": 1, 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "unhealthy_workers": 0 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-MDSRank": { 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 5184, 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.143368632, 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000027655 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:06.990 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "finisher-PurgeQueue": { 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "queue_len": 0, 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "complete_latency": { 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "request": 0, 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "reply": 0, 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "reply_latency": { 2024-05-20T20:20:06.991 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_reply": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "forward": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_complete": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_fetch_keys": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_commit": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_split": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_merge": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_top": 0, 2024-05-20T20:20:06.992 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_bottom": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pin_tail": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_pinned": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_expired": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "inodes_with_caps": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "caps": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "subtrees": 1, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_hit": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_forward": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_discover": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_dir_fetch": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_remote_ino": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "traverse_lock": 0, 2024-05-20T20:20:06.993 INFO:teuthology.orchestra.run.smithi154.stdout: "load_cent": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "q": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "exported": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "exported_inodes": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "imported": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "imported_inodes": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_dir_fetch": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_backtrace_fetch": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "openino_peer_discover": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rfiles": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rbytes": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "root_rsnaps": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_fetch": 0, 2024-05-20T20:20:06.994 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_set_tag": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_backtrace_repaired": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_inotable_repaired": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_inodes": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dir_base_inodes": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_dirfrag_rstats": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "scrub_file_inodes": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_inode_file_caps": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_revoke": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_grant": 0, 2024-05-20T20:20:06.995 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_trunc": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_cap_op_flush_ack": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_caps_dirty": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_cap_release": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "process_request_cap_release": 0 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_cache": { 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_update_receipt": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_try_discover": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_send_discover": 0, 2024-05-20T20:20:06.996 INFO:teuthology.orchestra.run.smithi154.stdout: "dir_handle_discover": 0, 2024-05-20T20:20:06.997 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays": 0, 2024-05-20T20:20:06.997 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_delayed": 0, 2024-05-20T20:20:06.997 INFO:teuthology.orchestra.run.smithi154.stdout: "num_strays_enqueuing": 0, 2024-05-20T20:20:06.997 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_created": 0, 2024-05-20T20:20:06.997 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_enqueued": 0, 2024-05-20T20:20:06.997 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_reintegrated": 0, 2024-05-20T20:20:06.998 INFO:teuthology.orchestra.run.smithi154.stdout: "strays_migrated": 0, 2024-05-20T20:20:06.998 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_processing": 0, 2024-05-20T20:20:06.998 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_enqueued": 0, 2024-05-20T20:20:06.998 INFO:teuthology.orchestra.run.smithi154.stdout: "num_recovering_prioritized": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_started": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "recovery_completed": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_path": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_quiesce_inode": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_enqueue_scrub": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_exportdir": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_flush": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragmentdir": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_fragstats": 0, 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: "ireq_inodestats": 0 2024-05-20T20:20:06.999 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_log": { 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "evlrg": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "evadd": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "evex": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "evtrm": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "ev": 5, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "evexg": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "evexd": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "segadd": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "segex": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "segtrm": 0, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "seg": 2, 2024-05-20T20:20:07.000 INFO:teuthology.orchestra.run.smithi154.stdout: "segmjr": 2, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "segexg": 0, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "segexd": 0, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "expos": 4194304, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "wrpos": 4195734, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "rdpos": 4195593, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "jlat": { 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "replayed": 5 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_mem": { 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "ino": 3, 2024-05-20T20:20:07.001 INFO:teuthology.orchestra.run.smithi154.stdout: "ino+": 3, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "ino-": 0, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "dir": 1, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "dir+": 1, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "dir-": 0, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "dn": 0, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "dn+": 0, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "dn-": 0, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "cap": 0, 2024-05-20T20:20:07.002 INFO:teuthology.orchestra.run.smithi154.stdout: "cap+": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "cap-": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "rss": 53960, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "heap": 215416 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_server": { 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_client_request": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "dispatch_server_request": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_request": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_client_session": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "handle_peer_request": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "req_create_latency": { 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.003 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getattr_latency": { 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getfilelock_latency": { 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "req_link_latency": { 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.004 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookup_latency": { 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookuphash_latency": { 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.005 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupino_latency": { 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupname_latency": { 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.006 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupparent_latency": { 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lookupsnap_latency": { 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.007 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: "req_lssnap_latency": { 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mkdir_latency": { 2024-05-20T20:20:07.008 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mknod_latency": { 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.009 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "req_mksnap_latency": { 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "req_open_latency": { 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "req_readdir_latency": { 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.010 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rename_latency": { 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "req_renamesnap_latency": { 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.011 INFO:teuthology.orchestra.run.smithi154.stdout: "req_snapdiff_latency": { 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmdir_latency": { 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmsnap_latency": { 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.012 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "req_rmxattr_latency": { 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setattr_latency": { 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setdirlayout_latency": { 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.013 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setfilelock_latency": { 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setlayout_latency": { 2024-05-20T20:20:07.014 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "req_setxattr_latency": { 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "req_symlink_latency": { 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.015 INFO:teuthology.orchestra.run.smithi154.stdout: "req_unlink_latency": { 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_revoke_eviction": 0, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "cap_acquisition_throttle": 0, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "req_getvxattr_latency": { 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.016 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_sessions": { 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "session_count": 1, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "session_add": 2, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "session_remove": 1, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_open": 2, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "sessions_stale": 0, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "total_load": 0, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "average_load": 0, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "avg_session_uptime": 4212, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "mdthresh_evicted": 0 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "mempool": { 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_bytes": 0, 2024-05-20T20:20:07.017 INFO:teuthology.orchestra.run.smithi154.stdout: "bloom_filter_items": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_bytes": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_alloc_items": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_bytes": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_data_items": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_bytes": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_onode_items": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_bytes": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_meta_items": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_bytes": 0, 2024-05-20T20:20:07.018 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_other_items": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_bytes": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_cache_buffer_items": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_bytes": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_extent_items": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_bytes": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_blob_items": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_bytes": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_shared_blob_items": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_bytes": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_inline_bl_items": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_bytes": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_fsck_items": 0, 2024-05-20T20:20:07.019 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_bytes": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_txc_items": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_bytes": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_deferred_items": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_bytes": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluestore_writing_items": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_bytes": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_items": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_bytes": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_reader_items": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_bytes": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "bluefs_file_writer_items": 0, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_bytes": 190923, 2024-05-20T20:20:07.020 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_anon_items": 54, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_bytes": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "buffer_meta_items": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_bytes": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_items": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_bytes": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_mapbl_items": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_bytes": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_pglog_items": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_bytes": 8128, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_items": 169, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_bytes": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "osdmap_mapping_items": 0, 2024-05-20T20:20:07.021 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_bytes": 0, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "pgmap_items": 0, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_bytes": 8512, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "mds_co_items": 13, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_bytes": 0, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_1_items": 0, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_bytes": 0, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "unittest_2_items": 0 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.022 INFO:teuthology.orchestra.run.smithi154.stdout: "objecter": { 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_active": 0, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_laggy": 0, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send": 3075, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_send_bytes": 21, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_resend": 3, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_reply": 3075, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_latency": { 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 3075, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 1.690955698, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000549904 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.023 INFO:teuthology.orchestra.run.smithi154.stdout: "op_inflight": 0, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "oplen_avg": { 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 3075, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 3076 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "op": 3075, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "op_r": 3075, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "op_w": 0, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "op_rmw": 0, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "op_pg": 0, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_stat": 1967, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_create": 0, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_read": 1106, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_write": 0, 2024-05-20T20:20:07.024 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writefull": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_writesame": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_append": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_zero": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_truncate": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_delete": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_mapext": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_sparse_read": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_clonerange": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_getxattr": 1, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_setxattr": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_cmpxattr": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_rmxattr": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_resetxattrs": 0, 2024-05-20T20:20:07.025 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_call": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_watch": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_notify": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_src_cmpxattr": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_pgls_filter": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "osdop_other": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_active": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_send": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_resend": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "linger_ping": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_active": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_send": 0, 2024-05-20T20:20:07.026 INFO:teuthology.orchestra.run.smithi154.stdout: "poolop_resend": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_active": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_send": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "poolstat_resend": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_active": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_send": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "statfs_resend": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "command_active": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "command_send": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "command_resend": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "map_epoch": 97, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "map_full": 0, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "map_inc": 19, 2024-05-20T20:20:07.027 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_sessions": 5, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_open": 7, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_session_close": 2, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "osd_laggy": 0, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_wr": 0, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_rd": 2, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_del": 0 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "oft": { 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_objs": 0, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_kv_pairs": 0, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_updates": 0, 2024-05-20T20:20:07.028 INFO:teuthology.orchestra.run.smithi154.stdout: "omap_total_removes": 0 2024-05-20T20:20:07.029 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.029 INFO:teuthology.orchestra.run.smithi154.stdout: "osd-slow-ops": { 2024-05-20T20:20:07.029 INFO:teuthology.orchestra.run.smithi154.stdout: "slow_ops_count": 0 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "purge_queue": { 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops": 0, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_ops_high_water": 0, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing": 0, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executing_high_water": 0, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed_ops": 0, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_executed": 0, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "pq_item_in_journal": 0 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:07.030 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 3430, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 2480302, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 3430, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 3430, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 2480302, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.031 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_bytes": { 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 104857600, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 3075, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 1435, 2024-05-20T20:20:07.032 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 3075, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 1435, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-objecter_ops": { 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 1024, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:07.033 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 3075, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 3075, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 3075, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 3075, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle": { 2024-05-20T20:20:07.034 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.035 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "throttle-write_buf_throttle-0x55994bf986e0": { 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "val": 0, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "max": 3758096384, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "get_started": 0, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "get": 0, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "get_sum": 0, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_fail": 0, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "get_or_fail_success": 0, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "take": 0, 2024-05-20T20:20:07.036 INFO:teuthology.orchestra.run.smithi154.stdout: "take_sum": 0, 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: "put": 0, 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: "put_sum": 0, 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: "wait": { 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: "avgcount": 0, 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: "sum": 0.000000000, 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: "avgtime": 0.000000000 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-05-20T20:20:07.037 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-05-20T20:20:07.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:07 smithi154 ceph-mon[36314]: pgmap v453: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 25 op/s 2024-05-20T20:20:07.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:07 smithi033 ceph-mon[36179]: pgmap v453: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 25 op/s 2024-05-20T20:20:07.670 INFO:tasks.check_counter:Daemon mds.l mds.exported=0 2024-05-20T20:20:07.671 INFO:tasks.check_counter:Daemon mds.l mds.imported=0 2024-05-20T20:20:07.671 INFO:tasks.check_counter:Daemon mds.l mds.root_rsnaps=0 2024-05-20T20:20:07.671 INFO:tasks.check_counter:Daemon mds.l mds_server.req_mksnap_latency.avgcount=0 2024-05-20T20:20:07.671 INFO:tasks.check_counter:Daemon mds.l mds_server.req_rmsnap_latency.avgcount=0 2024-05-20T20:20:07.671 DEBUG:teuthology.run_tasks:Unwinding manager kclient 2024-05-20T20:20:07.688 INFO:tasks.kclient:Unmounting kernel clients... 2024-05-20T20:20:07.688 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T20:20:07.688 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/proc/self/mounts of=/dev/stdout 2024-05-20T20:20:07.762 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-05-20T20:20:07.762 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/proc/self/mounts of=/dev/stdout 2024-05-20T20:20:07.804 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:07 smithi028 ceph-mon[31178]: pgmap v453: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 25 op/s 2024-05-20T20:20:07.826 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd blocklist ls 2024-05-20T20:20:08.375 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:09.521 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:09 smithi028 ceph-mon[31178]: pgmap v454: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 26 op/s 2024-05-20T20:20:09.521 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:09 smithi028 ceph-mon[31178]: osdmap e98: 12 total, 12 up, 12 in 2024-05-20T20:20:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:09 smithi154 ceph-mon[36314]: pgmap v454: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 26 op/s 2024-05-20T20:20:09.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:09 smithi154 ceph-mon[36314]: osdmap e98: 12 total, 12 up, 12 in 2024-05-20T20:20:09.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:09 smithi033 ceph-mon[36179]: pgmap v454: 97 pgs: 97 active+clean; 47 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 26 op/s 2024-05-20T20:20:09.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:09 smithi033 ceph-mon[36179]: osdmap e98: 12 total, 12 up, 12 in 2024-05-20T20:20:10.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.840+0000 7fa9c7200700 1 -- 172.21.15.28:0/4206318092 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8106010 msgr2=0x7fa9c8112520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:10.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.840+0000 7fa9c7200700 1 --2- 172.21.15.28:0/4206318092 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8106010 0x7fa9c8112520 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7fa9c000b0a0 tx=0x7fa9c000b3b0 comp rx=0 tx=0).stop 2024-05-20T20:20:10.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.840+0000 7fa9c7200700 1 -- 172.21.15.28:0/4206318092 shutdown_connections 2024-05-20T20:20:10.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.840+0000 7fa9c7200700 1 --2- 172.21.15.28:0/4206318092 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8106010 0x7fa9c8112520 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:10.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.840+0000 7fa9c7200700 1 --2- 172.21.15.28:0/4206318092 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9c810b650 0x7fa9c810ba30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:10.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.840+0000 7fa9c7200700 1 --2- 172.21.15.28:0/4206318092 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9c8105650 0x7fa9c8105ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:10.842 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.840+0000 7fa9c7200700 1 -- 172.21.15.28:0/4206318092 >> 172.21.15.28:0/4206318092 conn(0x7fa9c8101120 msgr2=0x7fa9c8103540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:10.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.841+0000 7fa9c7200700 1 -- 172.21.15.28:0/4206318092 shutdown_connections 2024-05-20T20:20:10.843 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.841+0000 7fa9c7200700 1 -- 172.21.15.28:0/4206318092 wait complete. 2024-05-20T20:20:10.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 Processor -- start 2024-05-20T20:20:10.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 -- start start 2024-05-20T20:20:10.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8105650 0x7fa9c81a1930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:10.844 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9c8106010 0x7fa9c81a1e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9c810b650 0x7fa9c819bb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fa9c81158a0 con 0x7fa9c8106010 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fa9c8115720 con 0x7fa9c8105650 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.842+0000 7fa9c7200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fa9c8115a20 con 0x7fa9c810b650 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9c8106010 0x7fa9c81a1e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9c8106010 0x7fa9c81a1e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47452/0 (socket says 172.21.15.28:47452) 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5400700 1 -- 172.21.15.28:0/248280989 learned_addr learned my addr 172.21.15.28:0/248280989 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5e00700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8105650 0x7fa9c81a1930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c6800700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9c810b650 0x7fa9c819bb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5e00700 1 -- 172.21.15.28:0/248280989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9c810b650 msgr2=0x7fa9c819bb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5e00700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9c810b650 0x7fa9c819bb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5e00700 1 -- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9c8106010 msgr2=0x7fa9c81a1e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:10.845 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5e00700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9c8106010 0x7fa9c81a1e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5e00700 1 -- 172.21.15.28:0/248280989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa9c819c2a0 con 0x7fa9c8105650 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.843+0000 7fa9c5e00700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8105650 0x7fa9c81a1930 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7fa9b800e9f0 tx=0x7fa9b800ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.844+0000 7fa9bea00700 1 -- 172.21.15.28:0/248280989 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9b800cbd0 con 0x7fa9c8105650 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.844+0000 7fa9bea00700 1 -- 172.21.15.28:0/248280989 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fa9b800cd70 con 0x7fa9c8105650 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.844+0000 7fa9bea00700 1 -- 172.21.15.28:0/248280989 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa9b8018680 con 0x7fa9c8105650 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.844+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa9c819c590 con 0x7fa9c8105650 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.844+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa9c81a8800 con 0x7fa9c8105650 2024-05-20T20:20:10.846 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.845+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa9c806a230 con 0x7fa9c8105650 2024-05-20T20:20:10.848 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.847+0000 7fa9bea00700 1 -- 172.21.15.28:0/248280989 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fa9b804ab30 con 0x7fa9c8105650 2024-05-20T20:20:10.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.849+0000 7fa9bea00700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9ac0b2a70 0x7fa9ac0b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:10.849 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.849+0000 7fa9bea00700 1 -- 172.21.15.28:0/248280989 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(98..98 src has 1..98) ==== 8586+0+0 (secure 0 0 0) 0x7fa9b80e2790 con 0x7fa9c8105650 2024-05-20T20:20:10.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.849+0000 7fa9c5400700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9ac0b2a70 0x7fa9ac0b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:10.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.850+0000 7fa9c5400700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9ac0b2a70 0x7fa9ac0b4f30 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fa9b4009680 tx=0x7fa9b4009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:10.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:10.851+0000 7fa9bea00700 1 -- 172.21.15.28:0/248280989 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fa9b80aa0d0 con 0x7fa9c8105650 2024-05-20T20:20:11.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.093+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) -- 0x7fa9c802e560 con 0x7fa9c8105650 2024-05-20T20:20:11.095 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.094+0000 7fa9bea00700 1 -- 172.21.15.28:0/248280989 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 16 entries v98) ==== 81+0+941 (secure 0 0 0) 0x7fa9b804ae20 con 0x7fa9c8105650 2024-05-20T20:20:11.095 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:6801/2019900521 2024-05-21T20:05:37.589137+0000 2024-05-20T20:20:11.095 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:6800/2019900521 2024-05-21T20:05:37.589137+0000 2024-05-20T20:20:11.095 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/2759010565 2024-05-21T20:05:37.589137+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:6801/811935446 2024-05-21T19:53:03.221125+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:6800/811935446 2024-05-21T19:53:03.221125+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/1496067961 2024-05-21T20:05:37.589137+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/3213207110 2024-05-21T19:53:03.221125+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/443467971 2024-05-21T19:52:27.405869+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/1975849601 2024-05-21T20:05:37.589137+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/3330495421 2024-05-21T19:52:27.405869+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:6800/1506400984 2024-05-21T19:52:27.405869+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/19609679 2024-05-21T20:05:37.589137+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/4072125252 2024-05-21T19:53:03.221125+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/3854117144 2024-05-21T19:52:27.405869+0000 2024-05-20T20:20:11.096 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:6801/1506400984 2024-05-21T19:52:27.405869+0000 2024-05-20T20:20:11.097 INFO:teuthology.orchestra.run.smithi028.stdout:172.21.15.28:0/1163220027 2024-05-21T19:53:03.221125+0000 2024-05-20T20:20:11.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9ac0b2a70 msgr2=0x7fa9ac0b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:11.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9ac0b2a70 0x7fa9ac0b4f30 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fa9b4009680 tx=0x7fa9b4009260 comp rx=0 tx=0).stop 2024-05-20T20:20:11.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8105650 msgr2=0x7fa9c81a1930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:11.098 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8105650 0x7fa9c81a1930 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7fa9b800e9f0 tx=0x7fa9b800ed00 comp rx=0 tx=0).stop 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 shutdown_connections 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa9c810b650 0x7fa9c819bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fa9ac0b2a70 0x7fa9ac0b4f30 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fa9c8105650 0x7fa9c81a1930 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 --2- 172.21.15.28:0/248280989 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fa9c8106010 0x7fa9c81a1e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.097+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 >> 172.21.15.28:0/248280989 conn(0x7fa9c8101120 msgr2=0x7fa9c8101700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.098+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 shutdown_connections 2024-05-20T20:20:11.099 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:11.098+0000 7fa9c7200700 1 -- 172.21.15.28:0/248280989 wait complete. 2024-05-20T20:20:11.100 INFO:teuthology.orchestra.run.smithi028.stderr:listed 16 entries 2024-05-20T20:20:11.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:11 smithi028 ceph-mon[31178]: pgmap v456: 97 pgs: 39 active+clean+snaptrim_wait, 22 active+clean+snaptrim, 36 active+clean; 44 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 54 KiB/s wr, 30 op/s 2024-05-20T20:20:11.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:11 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/248280989' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T20:20:11.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:11 smithi154 ceph-mon[36314]: pgmap v456: 97 pgs: 39 active+clean+snaptrim_wait, 22 active+clean+snaptrim, 36 active+clean; 44 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 54 KiB/s wr, 30 op/s 2024-05-20T20:20:11.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:11 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/248280989' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T20:20:11.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:11 smithi033 ceph-mon[36179]: pgmap v456: 97 pgs: 39 active+clean+snaptrim_wait, 22 active+clean+snaptrim, 36 active+clean; 44 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 54 KiB/s wr, 30 op/s 2024-05-20T20:20:11.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:11 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/248280989' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-05-20T20:20:12.267 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-05-20T20:20:12.267 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:20:12.267 DEBUG:teuthology.orchestra.run.smithi028:> sudo umount /home/ubuntu/cephtest/mnt.0 2024-05-20T20:20:12.441 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:20:12.441 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:20:12.441 DEBUG:teuthology.orchestra.run.smithi028:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.0) 2024-05-20T20:20:12.749 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:20:12.749 DEBUG:teuthology.orchestra.run.smithi028:> ip netns list 2024-05-20T20:20:12.847 INFO:teuthology.orchestra.run.smithi028.stdout:ceph-ns--home-ubuntu-cephtest-mnt.0 (id: 0) 2024-05-20T20:20:12.848 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:20:12.848 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.0 2024-05-20T20:20:12.949 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-05-20T20:20:12.949 DEBUG:teuthology.orchestra.run.smithi028:> sudo ip link delete ceph-brx 2024-05-20T20:20:13.089 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-20T20:20:13.122 INFO:tasks.cephadm:Teardown begin 2024-05-20T20:20:13.122 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-05-20T20:20:13.122 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:20:13.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:13 smithi028 ceph-mon[31178]: pgmap v457: 97 pgs: 39 active+clean+snaptrim_wait, 22 active+clean+snaptrim, 36 active+clean; 44 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 54 KiB/s wr, 30 op/s 2024-05-20T20:20:13.579 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:13.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:13 smithi154 ceph-mon[36314]: pgmap v457: 97 pgs: 39 active+clean+snaptrim_wait, 22 active+clean+snaptrim, 36 active+clean; 44 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 54 KiB/s wr, 30 op/s 2024-05-20T20:20:13.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:13 smithi033 ceph-mon[36179]: pgmap v457: 97 pgs: 39 active+clean+snaptrim_wait, 22 active+clean+snaptrim, 36 active+clean; 44 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 54 KiB/s wr, 30 op/s 2024-05-20T20:20:15.571 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:15 smithi028 ceph-mon[31178]: pgmap v458: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 1023 B/s wr, 11 op/s 2024-05-20T20:20:15.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:15 smithi154 ceph-mon[36314]: pgmap v458: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 1023 B/s wr, 11 op/s 2024-05-20T20:20:15.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:15 smithi033 ceph-mon[36179]: pgmap v458: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 1023 B/s wr, 11 op/s 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 -- 172.21.15.28:0/2223967689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c105760 msgr2=0x7f961c105be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 --2- 172.21.15.28:0/2223967689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c105760 0x7f961c105be0 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f9608009990 tx=0x7f9608009ca0 comp rx=0 tx=0).stop 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 -- 172.21.15.28:0/2223967689 shutdown_connections 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 --2- 172.21.15.28:0/2223967689 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f961c106120 0x7f961c112630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 --2- 172.21.15.28:0/2223967689 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c10b760 0x7f961c10bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 --2- 172.21.15.28:0/2223967689 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c105760 0x7f961c105be0 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 -- 172.21.15.28:0/2223967689 >> 172.21.15.28:0/2223967689 conn(0x7f961c101250 msgr2=0x7f961c103670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 -- 172.21.15.28:0/2223967689 shutdown_connections 2024-05-20T20:20:15.850 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.849+0000 7f9624000700 1 -- 172.21.15.28:0/2223967689 wait complete. 2024-05-20T20:20:15.851 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.850+0000 7f9624000700 1 Processor -- start 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.850+0000 7f9624000700 1 -- start start 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.850+0000 7f9624000700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c105760 0x7f961c19be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9624000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c106120 0x7f961c19c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9624000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f961c10b760 0x7f961c1a06e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9624000700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f961c115940 con 0x7f961c105760 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9624000700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f961c1157c0 con 0x7f961c10b760 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9624000700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f961c115ac0 con 0x7f961c106120 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9621600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c105760 0x7f961c19be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9621600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c105760 0x7f961c19be10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:47474/0 (socket says 172.21.15.28:47474) 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9622000700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f961c10b760 0x7f961c1a06e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:15.852 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9621600700 1 -- 172.21.15.28:0/4003142782 learned_addr learned my addr 172.21.15.28:0/4003142782 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:15.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.851+0000 7f9620c00700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c106120 0x7f961c19c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:15.853 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9621600700 1 -- 172.21.15.28:0/4003142782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c106120 msgr2=0x7f961c19c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9621600700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c106120 0x7f961c19c350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9621600700 1 -- 172.21.15.28:0/4003142782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f961c10b760 msgr2=0x7f961c1a06e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9621600700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f961c10b760 0x7f961c1a06e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9621600700 1 -- 172.21.15.28:0/4003142782 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f961c1a0e20 con 0x7f961c105760 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9622000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f961c10b760 0x7f961c1a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9620c00700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c106120 0x7f961c19c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9621600700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c105760 0x7f961c19be10 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f961000e9a0 tx=0x7f961000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f961a000700 1 -- 172.21.15.28:0/4003142782 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f961000ca10 con 0x7f961c105760 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f961a000700 1 -- 172.21.15.28:0/4003142782 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f961000cbb0 con 0x7f961c105760 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f961c1a1110 con 0x7f961c105760 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.852+0000 7f961a000700 1 -- 172.21.15.28:0/4003142782 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9610018680 con 0x7f961c105760 2024-05-20T20:20:15.854 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.853+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f961c1a8990 con 0x7f961c105760 2024-05-20T20:20:15.855 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.854+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f961c06a230 con 0x7f961c105760 2024-05-20T20:20:15.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.856+0000 7f961a000700 1 -- 172.21.15.28:0/4003142782 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f9610010430 con 0x7f961c105760 2024-05-20T20:20:15.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.858+0000 7f961a000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f96040b29f0 0x7f96040b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:15.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.858+0000 7f961a000700 1 -- 172.21.15.28:0/4003142782 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(99..99 src has 1..99) ==== 8542+0+0 (secure 0 0 0) 0x7f96100e27a0 con 0x7f961c105760 2024-05-20T20:20:15.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.858+0000 7f9620c00700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f96040b29f0 0x7f96040b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:15.859 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.859+0000 7f9620c00700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f96040b29f0 0x7f96040b4eb0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f96080098e0 tx=0x7f9608005960 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:15.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:15.862+0000 7f961a000700 1 -- 172.21.15.28:0/4003142782 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f96100aa110 con 0x7f961c105760 2024-05-20T20:20:16.163 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.163+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f961c02e560 con 0x7f961c105760 2024-05-20T20:20:16.164 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.164+0000 7f961a000700 1 -- 172.21.15.28:0/4003142782 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f96100b2190 con 0x7f961c105760 2024-05-20T20:20:16.165 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:20:16.165 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:20:16.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f96040b29f0 msgr2=0x7f96040b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:16.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f96040b29f0 0x7f96040b4eb0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f96080098e0 tx=0x7f9608005960 comp rx=0 tx=0).stop 2024-05-20T20:20:16.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c105760 msgr2=0x7f961c19be10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:16.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c105760 0x7f961c19be10 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7f961000e9a0 tx=0x7f961000ecb0 comp rx=0 tx=0).stop 2024-05-20T20:20:16.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 shutdown_connections 2024-05-20T20:20:16.168 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f96040b29f0 0x7f96040b4eb0 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:16.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f961c10b760 0x7f961c1a06e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:16.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f961c105760 0x7f961c19be10 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:16.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 --2- 172.21.15.28:0/4003142782 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f961c106120 0x7f961c19c350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:16.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.167+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 >> 172.21.15.28:0/4003142782 conn(0x7f961c101250 msgr2=0x7f961c1019a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:16.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.168+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 shutdown_connections 2024-05-20T20:20:16.169 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:16.168+0000 7f9624000700 1 -- 172.21.15.28:0/4003142782 wait complete. 2024-05-20T20:20:16.170 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:20:17.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:16 smithi028 ceph-mon[31178]: osdmap e99: 12 total, 12 up, 12 in 2024-05-20T20:20:17.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:16 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/4003142782' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:20:17.055 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:16 smithi028 ceph-mon[31178]: pgmap v460: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:20:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:16 smithi154 ceph-mon[36314]: osdmap e99: 12 total, 12 up, 12 in 2024-05-20T20:20:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:16 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/4003142782' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:20:17.125 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:16 smithi154 ceph-mon[36314]: pgmap v460: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:20:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:16 smithi033 ceph-mon[36179]: osdmap e99: 12 total, 12 up, 12 in 2024-05-20T20:20:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:16 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/4003142782' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:20:17.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:16 smithi033 ceph-mon[36179]: pgmap v460: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-05-20T20:20:18.733 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs dump --format=json 2024-05-20T20:20:19.203 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:19.561 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:19 smithi028 ceph-mon[31178]: pgmap v461: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 1.0 KiB/s wr, 12 op/s 2024-05-20T20:20:19.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:19 smithi154 ceph-mon[36314]: pgmap v461: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 1.0 KiB/s wr, 12 op/s 2024-05-20T20:20:19.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:19 smithi033 ceph-mon[36179]: pgmap v461: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 1.0 KiB/s wr, 12 op/s 2024-05-20T20:20:21.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:21 smithi028 ceph-mon[31178]: pgmap v462: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T20:20:21.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.558+0000 7f679b600700 1 -- 172.21.15.28:0/4045709729 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 msgr2=0x7f679406c630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:21.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.558+0000 7f679b600700 1 --2- 172.21.15.28:0/4045709729 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f679406c630 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7f6784009990 tx=0x7f6784009ca0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.559+0000 7f679b600700 1 -- 172.21.15.28:0/4045709729 shutdown_connections 2024-05-20T20:20:21.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.559+0000 7f679b600700 1 --2- 172.21.15.28:0/4045709729 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6794105f70 0x7f6794106420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.559+0000 7f679b600700 1 --2- 172.21.15.28:0/4045709729 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6794108460 0x7f679406bc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.559 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.559+0000 7f679b600700 1 --2- 172.21.15.28:0/4045709729 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f679406c630 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.559+0000 7f679b600700 1 -- 172.21.15.28:0/4045709729 >> 172.21.15.28:0/4045709729 conn(0x7f67940793f0 msgr2=0x7f6794079800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:21.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.559+0000 7f679b600700 1 -- 172.21.15.28:0/4045709729 shutdown_connections 2024-05-20T20:20:21.560 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.559+0000 7f679b600700 1 -- 172.21.15.28:0/4045709729 wait complete. 2024-05-20T20:20:21.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 Processor -- start 2024-05-20T20:20:21.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 -- start start 2024-05-20T20:20:21.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f67941a0230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:21.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6794105f70 0x7f67941a0770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:21.563 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6794108460 0x7f67941a4b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f6794119d20 con 0x7f679406c1b0 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f6794119ba0 con 0x7f6794105f70 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.560+0000 7f679b600700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f6794119ea0 con 0x7f6794108460 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6798c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f67941a0230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6799600700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6794108460 0x7f67941a4b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6793e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6794105f70 0x7f67941a0770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6798c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f67941a0230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:55200/0 (socket says 172.21.15.28:55200) 2024-05-20T20:20:21.564 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6798c00700 1 -- 172.21.15.28:0/4197355008 learned_addr learned my addr 172.21.15.28:0/4197355008 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6799600700 1 -- 172.21.15.28:0/4197355008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6794105f70 msgr2=0x7f67941a0770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6799600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6794105f70 0x7f67941a0770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6799600700 1 -- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 msgr2=0x7f67941a0230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6799600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f67941a0230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6799600700 1 -- 172.21.15.28:0/4197355008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f67941a5240 con 0x7f6794108460 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6798c00700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f67941a0230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.561+0000 7f6793e00700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6794105f70 0x7f67941a0770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.562+0000 7f6799600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6794108460 0x7f67941a4b00 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f678c00efc0 tx=0x7f678c00c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:21.565 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.562+0000 7f6791600700 1 -- 172.21.15.28:0/4197355008 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f678c0092e0 con 0x7f6794108460 2024-05-20T20:20:21.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.562+0000 7f6791600700 1 -- 172.21.15.28:0/4197355008 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f678c004bb0 con 0x7f6794108460 2024-05-20T20:20:21.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.562+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f67941a54d0 con 0x7f6794108460 2024-05-20T20:20:21.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.562+0000 7f6791600700 1 -- 172.21.15.28:0/4197355008 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f678c004020 con 0x7f6794108460 2024-05-20T20:20:21.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.562+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f67941acdb0 con 0x7f6794108460 2024-05-20T20:20:21.566 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.563+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f679406a230 con 0x7f6794108460 2024-05-20T20:20:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.566+0000 7f6791600700 1 -- 172.21.15.28:0/4197355008 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f678c01b030 con 0x7f6794108460 2024-05-20T20:20:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.567+0000 7f6791600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f677c0b29f0 0x7f677c0b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:21.567 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.567+0000 7f6791600700 1 -- 172.21.15.28:0/4197355008 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(99..99 src has 1..99) ==== 8542+0+0 (secure 0 0 0) 0x7f678c0e2b50 con 0x7f6794108460 2024-05-20T20:20:21.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.568+0000 7f6798c00700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f677c0b29f0 0x7f677c0b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:21.568 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.568+0000 7f6798c00700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f677c0b29f0 0x7f677c0b4eb0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f6788011550 tx=0x7f6788009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:21.569 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.569+0000 7f6791600700 1 -- 172.21.15.28:0/4197355008 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f678c022020 con 0x7f6794108460 2024-05-20T20:20:21.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:21 smithi154 ceph-mon[36314]: pgmap v462: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T20:20:21.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:21 smithi033 ceph-mon[36179]: pgmap v462: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T20:20:21.857 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.857+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) -- 0x7f67941acf80 con 0x7f6794108460 2024-05-20T20:20:21.858 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.859+0000 7f6791600700 1 -- 172.21.15.28:0/4197355008 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 29 v29) ==== 94+0+11925 (secure 0 0 0) 0x7f678c010040 con 0x7f6794108460 2024-05-20T20:20:21.859 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:20:21.859 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":29,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14364,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6837/1950403416","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6836","nonce":1950403416},{"type":"v1","addr":"172.21.15.28:6837","nonce":1950403416}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":5},{"gid":14388,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.28:6839/892373280","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6838","nonce":892373280},{"type":"v1","addr":"172.21.15.28:6839","nonce":892373280}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":11}],"filesystems":[{"mdsmap":{"epoch":29,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-05-20T20:01:26.231967+0000","modified":"2024-05-20T20:05:36.255795+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24542,"mds_1":24500,"mds_2":24493,"mds_3":24482,"mds_4":24475},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14400":{"gid":14400,"name":"a","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6841/1736910968","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6840","nonce":1736910968},{"type":"v1","addr":"172.21.15.28:6841","nonce":1736910968}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24427":{"gid":24427,"name":"l","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6833/2654302434","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":2654302434},{"type":"v1","addr":"172.21.15.154:6833","nonce":2654302434}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24442":{"gid":24442,"name":"j","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.28:6835/34026174","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6834","nonce":34026174},{"type":"v1","addr":"172.21.15.28:6835","nonce":34026174}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24455":{"gid":24455,"name":"k","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.33:6833/260147664","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":260147664},{"type":"v1","addr":"172.21.15.33:6833","nonce":260147664}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24463":{"gid":24463,"name":"f","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6835/3965936826","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6834","nonce":3965936826},{"type":"v1","addr":"172.21.15.154:6835","nonce":3965936826}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24475":{"gid":24475,"name":"c","rank":4,"incarnation":25,"state":"up:active","state_seq":17,"addr":"172.21.15.154:6837/2808233682","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6836","nonce":2808233682},{"type":"v1","addr":"172.21.15.154:6837","nonce":2808233682}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24482":{"gid":24482,"name":"e","rank":3,"incarnation":23,"state":"up:active","state_seq":18,"addr":"172.21.15.33:6835/4076526106","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":4076526106},{"type":"v1","addr":"172.21.15.33:6835","nonce":4076526106}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24493":{"gid":24493,"name":"i","rank":2,"incarnation":21,"state":"up:active","state_seq":15,"addr":"172.21.15.154:6839/1291909351","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6838","nonce":1291909351},{"type":"v1","addr":"172.21.15.154:6839","nonce":1291909351}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24500":{"gid":24500,"name":"h","rank":1,"incarnation":19,"state":"up:active","state_seq":15,"addr":"172.21.15.33:6837/4183908883","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6836","nonce":4183908883},{"type":"v1","addr":"172.21.15.33:6837","nonce":4183908883}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}},"gid_24542":{"gid":24542,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":7,"addr":"172.21.15.33:6839/2646117588","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6838","nonce":2646117588},{"type":"v1","addr":"172.21.15.33:6839","nonce":2646117588}]},"join_fscid":-1,"export_targets":[],"features":4540701547738038271,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":24542,"qdb_cluster":[24542,24500,24493,24475,24482]},"id":1}]} 2024-05-20T20:20:21.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f677c0b29f0 msgr2=0x7f677c0b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:21.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f677c0b29f0 0x7f677c0b4eb0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f6788011550 tx=0x7f6788009260 comp rx=0 tx=0).stop 2024-05-20T20:20:21.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6794108460 msgr2=0x7f67941a4b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:21.863 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6794108460 0x7f67941a4b00 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f678c00efc0 tx=0x7f678c00c3a0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 shutdown_connections 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f6794108460 0x7f67941a4b00 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f679406c1b0 0x7f67941a0230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f677c0b29f0 0x7f677c0b4eb0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 --2- 172.21.15.28:0/4197355008 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f6794105f70 0x7f67941a0770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.863+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 >> 172.21.15.28:0/4197355008 conn(0x7f67940793f0 msgr2=0x7f6794104f50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.864+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 shutdown_connections 2024-05-20T20:20:21.864 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:21.864+0000 7f679b600700 1 -- 172.21.15.28:0/4197355008 wait complete. 2024-05-20T20:20:21.865 INFO:teuthology.orchestra.run.smithi028.stderr:dumped fsmap epoch 29 2024-05-20T20:20:22.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:22 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/4197355008' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:20:22.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:22 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/4197355008' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:20:22.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:22 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/4197355008' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-05-20T20:20:22.705 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T20:20:23.194 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:23.604 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:23 smithi028 ceph-mon[31178]: pgmap v463: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T20:20:23.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:23 smithi154 ceph-mon[36314]: pgmap v463: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T20:20:23.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:23 smithi033 ceph-mon[36179]: pgmap v463: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.5 KiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T20:20:25.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 -- 172.21.15.28:0/226658589 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c076290 msgr2=0x7efc2c10f760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 --2- 172.21.15.28:0/226658589 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c076290 0x7efc2c10f760 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7efc18009990 tx=0x7efc18009ca0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 -- 172.21.15.28:0/226658589 shutdown_connections 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 --2- 172.21.15.28:0/226658589 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efc2c10fca0 0x7efc2c112110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 --2- 172.21.15.28:0/226658589 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efc2c075970 0x7efc2c075d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 --2- 172.21.15.28:0/226658589 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c076290 0x7efc2c10f760 unknown :-1 s=CLOSED pgs=640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 -- 172.21.15.28:0/226658589 >> 172.21.15.28:0/226658589 conn(0x7efc2c0fef50 msgr2=0x7efc2c101370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 -- 172.21.15.28:0/226658589 shutdown_connections 2024-05-20T20:20:25.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.429+0000 7efc32a00700 1 -- 172.21.15.28:0/226658589 wait complete. 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.430+0000 7efc32a00700 1 Processor -- start 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.430+0000 7efc32a00700 1 -- start start 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc32a00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c075970 0x7efc2c19be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc32a00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efc2c076290 0x7efc2c19c360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc32a00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efc2c10fca0 0x7efc2c1a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc32a00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7efc2c115890 con 0x7efc2c10fca0 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc32a00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7efc2c115710 con 0x7efc2c075970 2024-05-20T20:20:25.436 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc32a00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7efc2c115a10 con 0x7efc2c076290 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc31600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c075970 0x7efc2c19be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc30c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efc2c076290 0x7efc2c19c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc31600700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c075970 0x7efc2c19be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:39046/0 (socket says 172.21.15.28:39046) 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc31600700 1 -- 172.21.15.28:0/2940267439 learned_addr learned my addr 172.21.15.28:0/2940267439 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.431+0000 7efc32000700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efc2c10fca0 0x7efc2c1a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc31600700 1 -- 172.21.15.28:0/2940267439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efc2c076290 msgr2=0x7efc2c19c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc31600700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efc2c076290 0x7efc2c19c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc31600700 1 -- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efc2c10fca0 msgr2=0x7efc2c1a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc31600700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efc2c10fca0 0x7efc2c1a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc31600700 1 -- 172.21.15.28:0/2940267439 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7efc2c1a0de0 con 0x7efc2c075970 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc32000700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efc2c10fca0 0x7efc2c1a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc30c00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efc2c076290 0x7efc2c19c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:20:25.437 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc31600700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c075970 0x7efc2c19be20 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7efc2000ef10 tx=0x7efc2000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:25.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc2a000700 1 -- 172.21.15.28:0/2940267439 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc2000ca70 con 0x7efc2c075970 2024-05-20T20:20:25.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc2a000700 1 -- 172.21.15.28:0/2940267439 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7efc20019070 con 0x7efc2c075970 2024-05-20T20:20:25.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7efc2c1a1070 con 0x7efc2c075970 2024-05-20T20:20:25.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc2a000700 1 -- 172.21.15.28:0/2940267439 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7efc20014480 con 0x7efc2c075970 2024-05-20T20:20:25.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.432+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7efc2c1a8950 con 0x7efc2c075970 2024-05-20T20:20:25.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.434+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7efc2c06a230 con 0x7efc2c075970 2024-05-20T20:20:25.438 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.436+0000 7efc2a000700 1 -- 172.21.15.28:0/2940267439 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7efc20012050 con 0x7efc2c075970 2024-05-20T20:20:25.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.437+0000 7efc2a000700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efc140b29f0 0x7efc140b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:25.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.438+0000 7efc2a000700 1 -- 172.21.15.28:0/2940267439 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(99..99 src has 1..99) ==== 8542+0+0 (secure 0 0 0) 0x7efc200e2b00 con 0x7efc2c075970 2024-05-20T20:20:25.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.438+0000 7efc30c00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efc140b29f0 0x7efc140b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:25.439 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.438+0000 7efc30c00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efc140b29f0 0x7efc140b4eb0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7efc2c1110d0 tx=0x7efc1801a040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:25.440 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.440+0000 7efc2a000700 1 -- 172.21.15.28:0/2940267439 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7efc20010040 con 0x7efc2c075970 2024-05-20T20:20:25.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:25 smithi028 ceph-mon[31178]: pgmap v464: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2024-05-20T20:20:25.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:25 smithi154 ceph-mon[36314]: pgmap v464: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2024-05-20T20:20:25.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:25 smithi033 ceph-mon[36179]: pgmap v464: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2024-05-20T20:20:25.685 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.685+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7efc2c02e560 con 0x7efc2c075970 2024-05-20T20:20:25.687 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.687+0000 7efc2a000700 1 -- 172.21.15.28:0/2940267439 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v99) ==== 74+0+23038 (secure 0 0 0) 0x7efc200aa470 con 0x7efc2c075970 2024-05-20T20:20:25.688 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-05-20T20:20:25.689 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":99,"fsid":"3fbec9b2-16e2-11ef-bc9a-c7b262605968","created":"2024-05-20T19:51:52.746711+0000","modified":"2024-05-20T20:20:15.771410+0000","last_up_change":"2024-05-20T20:00:06.000959+0000","last_in_change":"2024-05-20T19:59:46.905281+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T19:55:47.376785+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-05-20T20:01:17.448722+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"78","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-05-20T20:01:22.420379+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"98","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":25,"snap_epoch":98,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ee508d61-6223-4ba1-a7c6-6be05c87bad9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6803","nonce":3612620015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6805","nonce":3612620015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6809","nonce":3612620015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":3612620015},{"type":"v1","addr":"172.21.15.28:6807","nonce":3612620015}]},"public_addr":"172.21.15.28:6803/3612620015","cluster_addr":"172.21.15.28:6805/3612620015","heartbeat_back_addr":"172.21.15.28:6809/3612620015","heartbeat_front_addr":"172.21.15.28:6807/3612620015","state":["exists","up"]},{"osd":1,"uuid":"c93dcd4d-a863-4d99-922d-2ba54fe25033","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6801","nonce":3302429642}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6803","nonce":3302429642}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6807","nonce":3302429642}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3302429642},{"type":"v1","addr":"172.21.15.33:6805","nonce":3302429642}]},"public_addr":"172.21.15.33:6801/3302429642","cluster_addr":"172.21.15.33:6803/3302429642","heartbeat_back_addr":"172.21.15.33:6807/3302429642","heartbeat_front_addr":"172.21.15.33:6805/3302429642","state":["exists","up"]},{"osd":2,"uuid":"a98e172e-f3be-463f-a25d-bc9fc778ce48","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6801","nonce":152922770}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6803","nonce":152922770}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6807","nonce":152922770}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":152922770},{"type":"v1","addr":"172.21.15.154:6805","nonce":152922770}]},"public_addr":"172.21.15.154:6801/152922770","cluster_addr":"172.21.15.154:6803/152922770","heartbeat_back_addr":"172.21.15.154:6807/152922770","heartbeat_front_addr":"172.21.15.154:6805/152922770","state":["exists","up"]},{"osd":3,"uuid":"43c813b2-4e9a-4c76-9e2e-6c19825adf7a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6811","nonce":1329515601}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6813","nonce":1329515601}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6817","nonce":1329515601}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":1329515601},{"type":"v1","addr":"172.21.15.28:6815","nonce":1329515601}]},"public_addr":"172.21.15.28:6811/1329515601","cluster_addr":"172.21.15.28:6813/1329515601","heartbeat_back_addr":"172.21.15.28:6817/1329515601","heartbeat_front_addr":"172.21.15.28:6815/1329515601","state":["exists","up"]},{"osd":4,"uuid":"57abdf0f-f317-465a-99d5-1d761d481da1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6809","nonce":3112020300}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6811","nonce":3112020300}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6815","nonce":3112020300}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":3112020300},{"type":"v1","addr":"172.21.15.33:6813","nonce":3112020300}]},"public_addr":"172.21.15.33:6809/3112020300","cluster_addr":"172.21.15.33:6811/3112020300","heartbeat_back_addr":"172.21.15.33:6815/3112020300","heartbeat_front_addr":"172.21.15.33:6813/3112020300","state":["exists","up"]},{"osd":5,"uuid":"967b3cfb-1bfd-4e94-b261-deb4fc1a4eb1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6809","nonce":666306624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6811","nonce":666306624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6815","nonce":666306624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":666306624},{"type":"v1","addr":"172.21.15.154:6813","nonce":666306624}]},"public_addr":"172.21.15.154:6809/666306624","cluster_addr":"172.21.15.154:6811/666306624","heartbeat_back_addr":"172.21.15.154:6815/666306624","heartbeat_front_addr":"172.21.15.154:6813/666306624","state":["exists","up"]},{"osd":6,"uuid":"1fb2acac-f855-46cf-8255-833f111bdddd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6819","nonce":163025282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6821","nonce":163025282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6825","nonce":163025282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":163025282},{"type":"v1","addr":"172.21.15.28:6823","nonce":163025282}]},"public_addr":"172.21.15.28:6819/163025282","cluster_addr":"172.21.15.28:6821/163025282","heartbeat_back_addr":"172.21.15.28:6825/163025282","heartbeat_front_addr":"172.21.15.28:6823/163025282","state":["exists","up"]},{"osd":7,"uuid":"a66dec84-09f9-4179-9130-3eb1e6cc5044","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6817","nonce":867501728}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6819","nonce":867501728}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6823","nonce":867501728}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":867501728},{"type":"v1","addr":"172.21.15.33:6821","nonce":867501728}]},"public_addr":"172.21.15.33:6817/867501728","cluster_addr":"172.21.15.33:6819/867501728","heartbeat_back_addr":"172.21.15.33:6823/867501728","heartbeat_front_addr":"172.21.15.33:6821/867501728","state":["exists","up"]},{"osd":8,"uuid":"8eb55633-1f82-4288-bf5c-e1e988c1d9fc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6817","nonce":1056926817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6819","nonce":1056926817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6823","nonce":1056926817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1056926817},{"type":"v1","addr":"172.21.15.154:6821","nonce":1056926817}]},"public_addr":"172.21.15.154:6817/1056926817","cluster_addr":"172.21.15.154:6819/1056926817","heartbeat_back_addr":"172.21.15.154:6823/1056926817","heartbeat_front_addr":"172.21.15.154:6821/1056926817","state":["exists","up"]},{"osd":9,"uuid":"4de26914-f99d-42d0-a3f9-9050e7bf73b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6827","nonce":3479914961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6828","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6829","nonce":3479914961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6832","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6833","nonce":3479914961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6830","nonce":3479914961},{"type":"v1","addr":"172.21.15.28:6831","nonce":3479914961}]},"public_addr":"172.21.15.28:6827/3479914961","cluster_addr":"172.21.15.28:6829/3479914961","heartbeat_back_addr":"172.21.15.28:6833/3479914961","heartbeat_front_addr":"172.21.15.28:6831/3479914961","state":["exists","up"]},{"osd":10,"uuid":"28571f48-e611-45cd-9835-72989012a981","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":64,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6825","nonce":924561268}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6827","nonce":924561268}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6831","nonce":924561268}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":924561268},{"type":"v1","addr":"172.21.15.33:6829","nonce":924561268}]},"public_addr":"172.21.15.33:6825/924561268","cluster_addr":"172.21.15.33:6827/924561268","heartbeat_back_addr":"172.21.15.33:6831/924561268","heartbeat_front_addr":"172.21.15.33:6829/924561268","state":["exists","up"]},{"osd":11,"uuid":"cb59bcc3-6dcc-42dd-8492-46bf37eb3be6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":70,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6825","nonce":2796531640}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6827","nonce":2796531640}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6831","nonce":2796531640}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":2796531640},{"type":"v1","addr":"172.21.15.154:6829","nonce":2796531640}]},"public_addr":"172.21.15.154:6825/2796531640","cluster_addr":"172.21.15.154:6827/2796531640","heartbeat_back_addr":"172.21.15.154:6831/2796531640","heartbeat_front_addr":"172.21.15.154:6829/2796531640","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:54:55.341838+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:20.731262+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:55:44.662246+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:18.263275+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:56:45.055374+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:10.045094+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:57:44.617507+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:12.075041+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:58:37.192670+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:11.769173+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T19:59:39.511015+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T20:00:04.878903+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.10","mappings":[{"from":2,"to":4}]},{"pgid":"2.13","mappings":[{"from":2,"to":11}]},{"pgid":"3.6","mappings":[{"from":4,"to":6}]},{"pgid":"3.1a","mappings":[{"from":5,"to":7}]},{"pgid":"3.1d","mappings":[{"from":5,"to":9}]},{"pgid":"3.22","mappings":[{"from":3,"to":9}]},{"pgid":"3.36","mappings":[{"from":4,"to":9}]},{"pgid":"3.3a","mappings":[{"from":4,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.28:6801/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6800/2019900521":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/2759010565":"2024-05-21T20:05:37.589137+0000","172.21.15.28:6801/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:6800/811935446":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/1496067961":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3213207110":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/443467971":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1975849601":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/3330495421":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6800/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/19609679":"2024-05-21T20:05:37.589137+0000","172.21.15.28:0/4072125252":"2024-05-21T19:53:03.221125+0000","172.21.15.28:0/3854117144":"2024-05-21T19:52:27.405869+0000","172.21.15.28:6801/1506400984":"2024-05-21T19:52:27.405869+0000","172.21.15.28:0/1163220027":"2024-05-21T19:53:03.221125+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":12,"length":1},{"begin":25,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.690+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efc140b29f0 msgr2=0x7efc140b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efc140b29f0 0x7efc140b4eb0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7efc2c1110d0 tx=0x7efc1801a040 comp rx=0 tx=0).stop 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c075970 msgr2=0x7efc2c19be20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c075970 0x7efc2c19be20 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7efc2000ef10 tx=0x7efc2000c3a0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 shutdown_connections 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7efc2c10fca0 0x7efc2c1a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7efc140b29f0 0x7efc140b4eb0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7efc2c075970 0x7efc2c19be20 unknown :-1 s=CLOSED pgs=641 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 --2- 172.21.15.28:0/2940267439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7efc2c076290 0x7efc2c19c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:25.691 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 >> 172.21.15.28:0/2940267439 conn(0x7efc2c0fef50 msgr2=0x7efc2c0ffd00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:25.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 shutdown_connections 2024-05-20T20:20:25.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:25.691+0000 7efc32a00700 1 -- 172.21.15.28:0/2940267439 wait complete. 2024-05-20T20:20:26.598 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:26 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/2940267439' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:20:26.603 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs fail cephfs 2024-05-20T20:20:26.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:26 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/2940267439' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:20:26.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:26 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/2940267439' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T20:20:27.085 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:27.494 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:27 smithi028 ceph-mon[31178]: pgmap v465: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 9 op/s 2024-05-20T20:20:27.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:27 smithi154 ceph-mon[36314]: pgmap v465: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 9 op/s 2024-05-20T20:20:27.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:27 smithi033 ceph-mon[36179]: pgmap v465: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 9 op/s 2024-05-20T20:20:29.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 -- 172.21.15.28:0/3906465566 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 msgr2=0x7fac3810f770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:29.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 --2- 172.21.15.28:0/3906465566 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac3810f770 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7fac28009990 tx=0x7fac28009ca0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 -- 172.21.15.28:0/3906465566 shutdown_connections 2024-05-20T20:20:29.489 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 --2- 172.21.15.28:0/3906465566 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac3810fcb0 0x7fac38112120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 --2- 172.21.15.28:0/3906465566 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 0x7fac38075d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 --2- 172.21.15.28:0/3906465566 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac3810f770 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 -- 172.21.15.28:0/3906465566 >> 172.21.15.28:0/3906465566 conn(0x7fac380fefc0 msgr2=0x7fac381013e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:29.490 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 -- 172.21.15.28:0/3906465566 shutdown_connections 2024-05-20T20:20:29.490 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:29 smithi028 ceph-mon[31178]: pgmap v466: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:20:29.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.489+0000 7fac3e200700 1 -- 172.21.15.28:0/3906465566 wait complete. 2024-05-20T20:20:29.491 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 Processor -- start 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 -- start start 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 0x7fac381a1a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac381a1f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac3810fcb0 0x7fac3819bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fac38115850 con 0x7fac38076290 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fac381156d0 con 0x7fac38075970 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.491+0000 7fac3e200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fac381159d0 con 0x7fac3810fcb0 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 0x7fac381a1a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac37e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac381a1f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 0x7fac381a1a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:50276/0 (socket says 172.21.15.28:50276) 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac37e00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac381a1f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36764/0 (socket says 172.21.15.28:36764) 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 -- 172.21.15.28:0/1328887693 learned_addr learned my addr 172.21.15.28:0/1328887693 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:29.492 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3d800700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac3810fcb0 0x7fac3819bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:29.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 -- 172.21.15.28:0/1328887693 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac3810fcb0 msgr2=0x7fac3819bb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:29.493 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac3810fcb0 0x7fac3819bb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 -- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 msgr2=0x7fac381a1f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac381a1f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 -- 172.21.15.28:0/1328887693 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fac3819c3f0 con 0x7fac38075970 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3d800700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac3810fcb0 0x7fac3819bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac37e00700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac381a1f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.492+0000 7fac3ce00700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 0x7fac381a1a40 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fac2c00d950 tx=0x7fac2c00dc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.493+0000 7fac35600700 1 -- 172.21.15.28:0/1328887693 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac2c016070 con 0x7fac38075970 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.493+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fac3819c6e0 con 0x7fac38075970 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.493+0000 7fac35600700 1 -- 172.21.15.28:0/1328887693 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fac2c009df0 con 0x7fac38075970 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.493+0000 7fac35600700 1 -- 172.21.15.28:0/1328887693 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fac2c00f3c0 con 0x7fac38075970 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.493+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fac381a89c0 con 0x7fac38075970 2024-05-20T20:20:29.494 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.494+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fac38077020 con 0x7fac38075970 2024-05-20T20:20:29.496 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.496+0000 7fac35600700 1 -- 172.21.15.28:0/1328887693 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fac2c019070 con 0x7fac38075970 2024-05-20T20:20:29.497 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.498+0000 7fac35600700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac200b29f0 0x7fac200b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:29.498 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.498+0000 7fac35600700 1 -- 172.21.15.28:0/1328887693 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(99..99 src has 1..99) ==== 8542+0+0 (secure 0 0 0) 0x7fac2c0e22c0 con 0x7fac38075970 2024-05-20T20:20:29.498 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.498+0000 7fac37e00700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac200b29f0 0x7fac200b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:29.498 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.499+0000 7fac37e00700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac200b29f0 0x7fac200b4eb0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fac2800b2e0 tx=0x7fac2801a040 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:29.502 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.502+0000 7fac35600700 1 -- 172.21.15.28:0/1328887693 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fac2c0a9c30 con 0x7fac38075970 2024-05-20T20:20:29.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:29 smithi154 ceph-mon[36314]: pgmap v466: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:20:29.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:29 smithi033 ceph-mon[36179]: pgmap v466: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 8 op/s 2024-05-20T20:20:29.794 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.794+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) -- 0x7fac3819cfd0 con 0x7fac38075970 2024-05-20T20:20:29.806 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.806+0000 7fac35600700 1 -- 172.21.15.28:0/1328887693 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v30) ==== 161+0+0 (secure 0 0 0) 0x7fac2c0a9530 con 0x7fac38075970 2024-05-20T20:20:29.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac200b29f0 msgr2=0x7fac200b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:29.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac200b29f0 0x7fac200b4eb0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fac2800b2e0 tx=0x7fac2801a040 comp rx=0 tx=0).stop 2024-05-20T20:20:29.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 msgr2=0x7fac381a1a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:29.810 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 0x7fac381a1a40 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fac2c00d950 tx=0x7fac2c00dc60 comp rx=0 tx=0).stop 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 shutdown_connections 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fac3810fcb0 0x7fac3819bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fac200b29f0 0x7fac200b4eb0 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fac38075970 0x7fac381a1a40 unknown :-1 s=CLOSED pgs=642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 --2- 172.21.15.28:0/1328887693 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fac38076290 0x7fac381a1f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.810+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 >> 172.21.15.28:0/1328887693 conn(0x7fac380fefc0 msgr2=0x7fac380ffd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.811+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 shutdown_connections 2024-05-20T20:20:29.811 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:29.811+0000 7fac3e200700 1 -- 172.21.15.28:0/1328887693 wait complete. 2024-05-20T20:20:29.812 INFO:teuthology.orchestra.run.smithi028.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-05-20T20:20:30.054 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a[31174]: 2024-05-20T20:20:29.796+0000 7f0091000700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T20:20:30.055 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:20:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a[73673]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.055 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:20:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a[73673]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.055 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:20:29 smithi028 ceph-mds[73678]: -- 172.21.15.28:0/2393980060 <== mon.0 v2:172.21.15.28:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558dac835a40 con 0x558dac808800 2024-05-20T20:20:30.055 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:20:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a[73673]: starting mds.a at 2024-05-20T20:20:30.055 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:20:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j[72079]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.055 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:20:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j[72079]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.056 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:20:29 smithi028 ceph-mds[72083]: -- 172.21.15.28:0/515065421 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56513ab37a40 con 0x56513ab0a800 2024-05-20T20:20:30.056 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:20:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j[72079]: starting mds.j at 2024-05-20T20:20:30.125 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i[67476]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.125 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i[67476]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.125 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:20:29 smithi154 ceph-mds[67480]: -- 172.21.15.154:0/4230878798 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c335fb5a40 con 0x55c335f89000 2024-05-20T20:20:30.125 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i[67476]: starting mds.i at 2024-05-20T20:20:30.126 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l[66553]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.126 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l[66553]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.126 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:20:29 smithi154 ceph-mds[66557]: -- 172.21.15.154:0/558770852 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b5da457a40 con 0x55b5da42b000 2024-05-20T20:20:30.126 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l[66553]: starting mds.l at 2024-05-20T20:20:30.126 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c[67158]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.126 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c[67158]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.127 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:20:29 smithi154 ceph-mds[67162]: -- 172.21.15.154:0/362308619 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5653371a3a40 con 0x565337176800 2024-05-20T20:20:30.127 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c[67158]: starting mds.c at 2024-05-20T20:20:30.127 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f[66852]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.127 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f[66852]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.127 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:20:29 smithi154 ceph-mds[66856]: -- 172.21.15.154:0/3091855808 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564ce3199a40 con 0x564ce316c800 2024-05-20T20:20:30.127 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:20:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f[66852]: starting mds.f at 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:20:29 smithi033 ceph-mds[66627]: -- 172.21.15.33:0/4039315939 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5633f450fa40 con 0x5633f44e2c00 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: starting mds.b at 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h[66295]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h[66295]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:20:29 smithi033 ceph-mds[66299]: -- 172.21.15.33:0/3069767940 <== mon.1 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c8326afa40 con 0x55c832683000 2024-05-20T20:20:30.128 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h[66295]: starting mds.h at 2024-05-20T20:20:30.129 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e[65989]: ignoring --setuser ceph since I am not root 2024-05-20T20:20:30.129 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e[65989]: ignoring --setgroup ceph since I am not root 2024-05-20T20:20:30.129 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:20:29 smithi033 ceph-mds[65993]: -- 172.21.15.33:0/579305904 <== mon.2 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5606b6d1fa40 con 0x5606b6cf2c00 2024-05-20T20:20:30.129 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:20:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e[65989]: starting mds.e at 2024-05-20T20:20:30.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:30 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/1328887693' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-05-20T20:20:30.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:30 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-05-20T20:20:30.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:30 smithi028 ceph-mon[31178]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-05-20T20:20:30.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:30 smithi028 ceph-mon[31178]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T20:20:30.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:30 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-05-20T20:20:30.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:30 smithi028 ceph-mon[31178]: osdmap e100: 12 total, 12 up, 12 in 2024-05-20T20:20:30.555 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:30 smithi028 ceph-mon[31178]: fsmap cephfs:0/5 2 up:standby, 5 failed 2024-05-20T20:20:30.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:30 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/1328887693' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-05-20T20:20:30.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:30 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-05-20T20:20:30.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:30 smithi154 ceph-mon[36314]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-05-20T20:20:30.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:30 smithi154 ceph-mon[36314]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T20:20:30.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:30 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-05-20T20:20:30.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:30 smithi154 ceph-mon[36314]: osdmap e100: 12 total, 12 up, 12 in 2024-05-20T20:20:30.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:30 smithi154 ceph-mon[36314]: fsmap cephfs:0/5 2 up:standby, 5 failed 2024-05-20T20:20:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:30 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/1328887693' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-05-20T20:20:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:30 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-05-20T20:20:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:30 smithi033 ceph-mon[36179]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-05-20T20:20:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:30 smithi033 ceph-mon[36179]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T20:20:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:30 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-05-20T20:20:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:30 smithi033 ceph-mon[36179]: osdmap e100: 12 total, 12 up, 12 in 2024-05-20T20:20:30.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:30 smithi033 ceph-mon[36179]: fsmap cephfs:0/5 2 up:standby, 5 failed 2024-05-20T20:20:30.729 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph fs rm cephfs --yes-i-really-mean-it 2024-05-20T20:20:31.190 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: pgmap v468: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2024-05-20T20:20:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6832/2988947759,v1:172.21.15.154:6834/2988947759] up:boot 2024-05-20T20:20:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6833/2237007200,v1:172.21.15.154:6835/2237007200] up:boot 2024-05-20T20:20:31.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6832/2364094881,v1:172.21.15.33:6834/2364094881] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6833/301439804,v1:172.21.15.33:6835/301439804] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6836/2753765302,v1:172.21.15.154:6838/2753765302] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.154:6837/1444754522,v1:172.21.15.154:6839/1444754522] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6836/3619265372,v1:172.21.15.33:6837/3619265372] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.33:6838/1005804717,v1:172.21.15.33:6839/1005804717] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.28:6834/3405431328,v1:172.21.15.28:6835/3405431328] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: mds.? [v2:172.21.15.28:6840/66900653,v1:172.21.15.28:6841/66900653] up:boot 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: fsmap cephfs:0/5 12 up:standby, 5 failed 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:20:31.626 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:31 smithi154 ceph-mon[36314]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: pgmap v468: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6832/2988947759,v1:172.21.15.154:6834/2988947759] up:boot 2024-05-20T20:20:31.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6833/2237007200,v1:172.21.15.154:6835/2237007200] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6832/2364094881,v1:172.21.15.33:6834/2364094881] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6833/301439804,v1:172.21.15.33:6835/301439804] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6836/2753765302,v1:172.21.15.154:6838/2753765302] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.154:6837/1444754522,v1:172.21.15.154:6839/1444754522] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6836/3619265372,v1:172.21.15.33:6837/3619265372] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.33:6838/1005804717,v1:172.21.15.33:6839/1005804717] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.28:6834/3405431328,v1:172.21.15.28:6835/3405431328] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: mds.? [v2:172.21.15.28:6840/66900653,v1:172.21.15.28:6841/66900653] up:boot 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: fsmap cephfs:0/5 12 up:standby, 5 failed 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:20:31.628 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:20:31.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:20:31.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:20:31.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:20:31.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:20:31.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:20:31.629 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:31 smithi033 ceph-mon[36179]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:20:31.685 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: pgmap v468: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2024-05-20T20:20:31.685 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6832/2988947759,v1:172.21.15.154:6834/2988947759] up:boot 2024-05-20T20:20:31.685 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6833/2237007200,v1:172.21.15.154:6835/2237007200] up:boot 2024-05-20T20:20:31.685 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6832/2364094881,v1:172.21.15.33:6834/2364094881] up:boot 2024-05-20T20:20:31.685 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6833/301439804,v1:172.21.15.33:6835/301439804] up:boot 2024-05-20T20:20:31.685 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6836/2753765302,v1:172.21.15.154:6838/2753765302] up:boot 2024-05-20T20:20:31.685 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.154:6837/1444754522,v1:172.21.15.154:6839/1444754522] up:boot 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6836/3619265372,v1:172.21.15.33:6837/3619265372] up:boot 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.33:6838/1005804717,v1:172.21.15.33:6839/1005804717] up:boot 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.28:6834/3405431328,v1:172.21.15.28:6835/3405431328] up:boot 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: mds.? [v2:172.21.15.28:6840/66900653,v1:172.21.15.28:6841/66900653] up:boot 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: fsmap cephfs:0/5 12 up:standby, 5 failed 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-05-20T20:20:31.686 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-05-20T20:20:31.687 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-05-20T20:20:31.687 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:31 smithi028 ceph-mon[31178]: from='mgr.25120 172.21.15.28:0/626918933' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-05-20T20:20:33.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:33 smithi028 ceph-mon[31178]: pgmap v469: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.582+0000 7fafe1c00700 1 -- 172.21.15.28:0/845254153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1041a0 msgr2=0x7fafdc1081c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.582+0000 7fafe1c00700 1 --2- 172.21.15.28:0/845254153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1041a0 0x7fafdc1081c0 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7fafc40099c0 tx=0x7fafc4009cd0 comp rx=0 tx=0).stop 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.582+0000 7fafe1c00700 1 -- 172.21.15.28:0/845254153 shutdown_connections 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.582+0000 7fafe1c00700 1 --2- 172.21.15.28:0/845254153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafdc1088f0 0x7fafdc110470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.582+0000 7fafe1c00700 1 --2- 172.21.15.28:0/845254153 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fafdc1037f0 0x7fafdc103bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.582+0000 7fafe1c00700 1 --2- 172.21.15.28:0/845254153 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1041a0 0x7fafdc1081c0 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.582+0000 7fafe1c00700 1 -- 172.21.15.28:0/845254153 >> 172.21.15.28:0/845254153 conn(0x7fafdc07a6a0 msgr2=0x7fafdc07aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.583+0000 7fafe1c00700 1 -- 172.21.15.28:0/845254153 shutdown_connections 2024-05-20T20:20:33.583 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.583+0000 7fafe1c00700 1 -- 172.21.15.28:0/845254153 wait complete. 2024-05-20T20:20:33.584 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 Processor -- start 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 -- start start 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1037f0 0x7fafdc19bd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafdc1041a0 0x7fafdc19c280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fafdc1088f0 0x7fafdc1a0610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fafdc1136b0 con 0x7fafdc1037f0 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fafdc113530 con 0x7fafdc1088f0 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.584+0000 7fafe1c00700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fafdc113830 con 0x7fafdc1041a0 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1037f0 0x7fafdc19bd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1037f0 0x7fafdc19bd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:36816/0 (socket says 172.21.15.28:36816) 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 -- 172.21.15.28:0/221285883 learned_addr learned my addr 172.21.15.28:0/221285883 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdb400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fafdc1088f0 0x7fafdc1a0610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:33.585 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafd2000700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafdc1041a0 0x7fafdc19c280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:33.586 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 -- 172.21.15.28:0/221285883 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafdc1041a0 msgr2=0x7fafdc19c280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafdc1041a0 0x7fafdc19c280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 -- 172.21.15.28:0/221285883 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fafdc1088f0 msgr2=0x7fafdc1a0610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fafdc1088f0 0x7fafdc1a0610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdaa00700 1 -- 172.21.15.28:0/221285883 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fafdc1a0d50 con 0x7fafdc1037f0 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.585+0000 7fafdb400700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fafdc1088f0 0x7fafdc1a0610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.586+0000 7fafdaa00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1037f0 0x7fafdc19bd40 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fafcc00e9a0 tx=0x7fafcc00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.586+0000 7fafd3e00700 1 -- 172.21.15.28:0/221285883 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fafcc00c9c0 con 0x7fafdc1037f0 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.586+0000 7fafd3e00700 1 -- 172.21.15.28:0/221285883 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fafcc00cb60 con 0x7fafdc1037f0 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.586+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fafdc1a1040 con 0x7fafdc1037f0 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.586+0000 7fafd3e00700 1 -- 172.21.15.28:0/221285883 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fafcc018680 con 0x7fafdc1037f0 2024-05-20T20:20:33.587 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.586+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fafdc1a88c0 con 0x7fafdc1037f0 2024-05-20T20:20:33.588 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.587+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fafdc06a230 con 0x7fafdc1037f0 2024-05-20T20:20:33.591 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.591+0000 7fafd3e00700 1 -- 172.21.15.28:0/221285883 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fafcc010430 con 0x7fafdc1037f0 2024-05-20T20:20:33.592 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.593+0000 7fafd3e00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fafc00b29f0 0x7fafc00b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:33.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.593+0000 7fafd3e00700 1 -- 172.21.15.28:0/221285883 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(100..100 src has 1..100) ==== 8928+0+0 (secure 0 0 0) 0x7fafcc0e2df0 con 0x7fafdc1037f0 2024-05-20T20:20:33.593 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.593+0000 7fafd2000700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fafc00b29f0 0x7fafc00b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:33.594 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.594+0000 7fafd2000700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fafc00b29f0 0x7fafc00b4eb0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fafdc1108b0 tx=0x7fafc4005fb0 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:33.595 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.596+0000 7fafd3e00700 1 -- 172.21.15.28:0/221285883 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fafcc0aa600 con 0x7fafdc1037f0 2024-05-20T20:20:33.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:33 smithi154 ceph-mon[36314]: pgmap v469: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2024-05-20T20:20:33.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:33 smithi033 ceph-mon[36179]: pgmap v469: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2024-05-20T20:20:33.883 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:33.883+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) -- 0x7fafdc1a8a90 con 0x7fafdc1037f0 2024-05-20T20:20:34.382 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.382+0000 7fafd3e00700 1 -- 172.21.15.28:0/221285883 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v32) ==== 104+0+0 (secure 0 0 0) 0x7fafcc0a9f00 con 0x7fafdc1037f0 2024-05-20T20:20:34.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.387+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fafc00b29f0 msgr2=0x7fafc00b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:34.387 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.387+0000 7fafe1c00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fafc00b29f0 0x7fafc00b4eb0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fafdc1108b0 tx=0x7fafc4005fb0 comp rx=0 tx=0).stop 2024-05-20T20:20:34.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1037f0 msgr2=0x7fafdc19bd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:34.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1037f0 0x7fafdc19bd40 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7fafcc00e9a0 tx=0x7fafcc00ecb0 comp rx=0 tx=0).stop 2024-05-20T20:20:34.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 shutdown_connections 2024-05-20T20:20:34.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fafc00b29f0 0x7fafc00b4eb0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:34.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fafdc1088f0 0x7fafdc1a0610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:34.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fafdc1037f0 0x7fafdc19bd40 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:34.388 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 --2- 172.21.15.28:0/221285883 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafdc1041a0 0x7fafdc19c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:34.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 >> 172.21.15.28:0/221285883 conn(0x7fafdc07a6a0 msgr2=0x7fafdc101630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:34.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 shutdown_connections 2024-05-20T20:20:34.389 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:34.388+0000 7fafe1c00700 1 -- 172.21.15.28:0/221285883 wait complete. 2024-05-20T20:20:34.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:34 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/221285883' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T20:20:34.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:34 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/221285883' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T20:20:34.779 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:34 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/221285883' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T20:20:35.213 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-05-20T20:20:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:35 smithi028 ceph-mon[31178]: pgmap v470: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-05-20T20:20:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:35 smithi028 ceph-mon[31178]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-05-20T20:20:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:35 smithi028 ceph-mon[31178]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T20:20:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:35 smithi028 ceph-mon[31178]: Cluster is now healthy 2024-05-20T20:20:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:35 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/221285883' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-05-20T20:20:35.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:35 smithi028 ceph-mon[31178]: fsmap 12 up:standby 2024-05-20T20:20:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:35 smithi154 ceph-mon[36314]: pgmap v470: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-05-20T20:20:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:35 smithi154 ceph-mon[36314]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-05-20T20:20:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:35 smithi154 ceph-mon[36314]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T20:20:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:35 smithi154 ceph-mon[36314]: Cluster is now healthy 2024-05-20T20:20:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:35 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/221285883' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-05-20T20:20:35.625 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:35 smithi154 ceph-mon[36314]: fsmap 12 up:standby 2024-05-20T20:20:35.626 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:35 smithi033 ceph-mon[36179]: pgmap v470: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-05-20T20:20:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:35 smithi033 ceph-mon[36179]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-05-20T20:20:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:35 smithi033 ceph-mon[36179]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T20:20:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:35 smithi033 ceph-mon[36179]: Cluster is now healthy 2024-05-20T20:20:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:35 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/221285883' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-05-20T20:20:35.627 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:35 smithi033 ceph-mon[36179]: fsmap 12 up:standby 2024-05-20T20:20:35.697 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:37.625 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:37 smithi028 ceph-mon[31178]: pgmap v471: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-05-20T20:20:37.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:37 smithi154 ceph-mon[36314]: pgmap v471: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-05-20T20:20:37.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:37 smithi033 ceph-mon[36179]: pgmap v471: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 5 op/s 2024-05-20T20:20:37.928 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 -- 172.21.15.28:0/2566743426 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80075770 msgr2=0x7f8f80075b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 --2- 172.21.15.28:0/2566743426 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80075770 0x7f8f80075b50 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f8f70009990 tx=0x7f8f70009ca0 comp rx=0 tx=0).stop 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 -- 172.21.15.28:0/2566743426 shutdown_connections 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 --2- 172.21.15.28:0/2566743426 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f8010ff40 0x7f8f80112330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 --2- 172.21.15.28:0/2566743426 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80075770 0x7f8f80075b50 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 --2- 172.21.15.28:0/2566743426 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f80076090 0x7f8f8010fa00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 -- 172.21.15.28:0/2566743426 >> 172.21.15.28:0/2566743426 conn(0x7f8f8007a6a0 msgr2=0x7f8f8007aab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 -- 172.21.15.28:0/2566743426 shutdown_connections 2024-05-20T20:20:37.929 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.927+0000 7f8f86400700 1 -- 172.21.15.28:0/2566743426 wait complete. 2024-05-20T20:20:37.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.929+0000 7f8f86400700 1 Processor -- start 2024-05-20T20:20:37.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.929+0000 7f8f86400700 1 -- start start 2024-05-20T20:20:37.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f86400700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f80075770 0x7f8f8019bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:37.930 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f86400700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80076090 0x7f8f8019c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:37.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f86400700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f8010ff40 0x7f8f801a06a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:37.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f86400700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7f8f80115940 con 0x7f8f80076090 2024-05-20T20:20:37.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f86400700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7f8f801157c0 con 0x7f8f80075770 2024-05-20T20:20:37.931 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f86400700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7f8f80115ac0 con 0x7f8f8010ff40 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f7ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80076090 0x7f8f8019c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f7ea00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80076090 0x7f8f8019c310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.28:3300/0 says I am v2:172.21.15.28:40498/0 (socket says 172.21.15.28:40498) 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f7ea00700 1 -- 172.21.15.28:0/130209662 learned_addr learned my addr 172.21.15.28:0/130209662 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f7f400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f80075770 0x7f8f8019bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.930+0000 7f8f7fe00700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f8010ff40 0x7f8f801a06a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f7ea00700 1 -- 172.21.15.28:0/130209662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f8010ff40 msgr2=0x7f8f801a06a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f7ea00700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f8010ff40 0x7f8f801a06a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f7ea00700 1 -- 172.21.15.28:0/130209662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f80075770 msgr2=0x7f8f8019bdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f7ea00700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f80075770 0x7f8f8019bdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f7ea00700 1 -- 172.21.15.28:0/130209662 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8f801a0de0 con 0x7f8f80076090 2024-05-20T20:20:37.932 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f7ea00700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80076090 0x7f8f8019c310 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f8f6c00d980 tx=0x7f8f6c00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:37.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f77e00700 1 -- 172.21.15.28:0/130209662 <== mon.0 v2:172.21.15.28:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f6c016070 con 0x7f8f80076090 2024-05-20T20:20:37.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8f801a10d0 con 0x7f8f80076090 2024-05-20T20:20:37.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f77e00700 1 -- 172.21.15.28:0/130209662 <== mon.0 v2:172.21.15.28:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7f8f6c00fd40 con 0x7f8f80076090 2024-05-20T20:20:37.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.931+0000 7f8f77e00700 1 -- 172.21.15.28:0/130209662 <== mon.0 v2:172.21.15.28:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f6c019360 con 0x7f8f80076090 2024-05-20T20:20:37.933 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.932+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8f801a8950 con 0x7f8f80076090 2024-05-20T20:20:37.934 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.933+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8f8006a230 con 0x7f8f80076090 2024-05-20T20:20:37.937 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.936+0000 7f8f77e00700 1 -- 172.21.15.28:0/130209662 <== mon.0 v2:172.21.15.28:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7f8f6c019070 con 0x7f8f80076090 2024-05-20T20:20:37.939 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.939+0000 7f8f77e00700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8f640b2a70 0x7f8f640b4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:37.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.939+0000 7f8f77e00700 1 -- 172.21.15.28:0/130209662 <== mon.0 v2:172.21.15.28:3300/0 5 ==== osd_map(100..100 src has 1..100) ==== 8928+0+0 (secure 0 0 0) 0x7f8f6c0e2740 con 0x7f8f80076090 2024-05-20T20:20:37.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.939+0000 7f8f7f400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8f640b2a70 0x7f8f640b4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:37.940 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.940+0000 7f8f7f400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8f640b2a70 0x7f8f640b4f30 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f8f70006010 tx=0x7f8f7000b540 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:37.941 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:37.941+0000 7f8f77e00700 1 -- 172.21.15.28:0/130209662 <== mon.0 v2:172.21.15.28:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7f8f6c0a9f50 con 0x7f8f80076090 2024-05-20T20:20:38.181 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.180+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) -- 0x7f8f8002e560 con 0x7f8f80076090 2024-05-20T20:20:38.391 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.390+0000 7f8f77e00700 1 -- 172.21.15.28:0/130209662 <== mon.0 v2:172.21.15.28:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v101) ==== 181+0+0 (secure 0 0 0) 0x7f8f6c0a9850 con 0x7f8f80076090 2024-05-20T20:20:38.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.397+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8f640b2a70 msgr2=0x7f8f640b4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:38.398 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.397+0000 7f8f86400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8f640b2a70 0x7f8f640b4f30 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f8f70006010 tx=0x7f8f7000b540 comp rx=0 tx=0).stop 2024-05-20T20:20:38.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.397+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80076090 msgr2=0x7f8f8019c310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:38.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.397+0000 7f8f86400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80076090 0x7f8f8019c310 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7f8f6c00d980 tx=0x7f8f6c00dc90 comp rx=0 tx=0).stop 2024-05-20T20:20:38.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 shutdown_connections 2024-05-20T20:20:38.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f8f8010ff40 0x7f8f801a06a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:38.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7f8f640b2a70 0x7f8f640b4f30 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:38.399 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7f8f80075770 0x7f8f8019bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:38.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 --2- 172.21.15.28:0/130209662 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7f8f80076090 0x7f8f8019c310 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:38.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 >> 172.21.15.28:0/130209662 conn(0x7f8f8007a6a0 msgr2=0x7f8f801006d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:38.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 shutdown_connections 2024-05-20T20:20:38.400 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:38.398+0000 7f8f86400700 1 -- 172.21.15.28:0/130209662 wait complete. 2024-05-20T20:20:38.401 INFO:teuthology.orchestra.run.smithi028.stderr:pool 'cephfs_metadata' removed 2024-05-20T20:20:38.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:38 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/130209662' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:38.877 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:38 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/130209662' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:38.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:38 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/130209662' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:39.332 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634 shell --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 -- ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-05-20T20:20:39.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:39 smithi028 ceph-mon[31178]: pgmap v472: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-05-20T20:20:39.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:39 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/130209662' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-05-20T20:20:39.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:39 smithi028 ceph-mon[31178]: osdmap e101: 12 total, 12 up, 12 in 2024-05-20T20:20:39.820 INFO:teuthology.orchestra.run.smithi028.stderr:Inferring config /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/mon.a/config 2024-05-20T20:20:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:39 smithi154 ceph-mon[36314]: pgmap v472: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-05-20T20:20:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:39 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/130209662' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-05-20T20:20:39.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:39 smithi154 ceph-mon[36314]: osdmap e101: 12 total, 12 up, 12 in 2024-05-20T20:20:39.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:39 smithi033 ceph-mon[36179]: pgmap v472: 97 pgs: 97 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-05-20T20:20:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:39 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/130209662' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-05-20T20:20:39.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:39 smithi033 ceph-mon[36179]: osdmap e101: 12 total, 12 up, 12 in 2024-05-20T20:20:41.694 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:41 smithi028 ceph-mon[31178]: pgmap v474: 65 pgs: 65 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:20:41.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:41 smithi154 ceph-mon[36314]: pgmap v474: 65 pgs: 65 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:20:41.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:41 smithi033 ceph-mon[36179]: pgmap v474: 65 pgs: 65 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:20:42.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 -- 172.21.15.28:0/461676561 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe0109a80 msgr2=0x7fcfe0109f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:42.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 --2- 172.21.15.28:0/461676561 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe0109a80 0x7fcfe0109f00 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7fcfc8009990 tx=0x7fcfc8009ca0 comp rx=0 tx=0).stop 2024-05-20T20:20:42.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 -- 172.21.15.28:0/461676561 shutdown_connections 2024-05-20T20:20:42.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 --2- 172.21.15.28:0/461676561 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcfe010a440 0x7fcfe01168e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:42.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 --2- 172.21.15.28:0/461676561 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcfe010e950 0x7fcfe010ed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:42.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 --2- 172.21.15.28:0/461676561 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe0109a80 0x7fcfe0109f00 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:42.296 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 -- 172.21.15.28:0/461676561 >> 172.21.15.28:0/461676561 conn(0x7fcfe00792d0 msgr2=0x7fcfe0108360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:42.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 -- 172.21.15.28:0/461676561 shutdown_connections 2024-05-20T20:20:42.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.294+0000 7fcfdf200700 1 -- 172.21.15.28:0/461676561 wait complete. 2024-05-20T20:20:42.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.295+0000 7fcfdf200700 1 Processor -- start 2024-05-20T20:20:42.297 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.295+0000 7fcfdf200700 1 -- start start 2024-05-20T20:20:42.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfdf200700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcfe0109a80 0x7fcfe01a00c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:42.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfdf200700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe010a440 0x7fcfe01a0600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:42.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfdf200700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcfe010e950 0x7fcfe01a4990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:42.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfdf200700 1 -- --> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] -- mon_getmap magic: 0 -- 0x7fcfe0119c10 con 0x7fcfe0109a80 2024-05-20T20:20:42.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfdf200700 1 -- --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_getmap magic: 0 -- 0x7fcfe0119a90 con 0x7fcfe010a440 2024-05-20T20:20:42.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfdf200700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 -- 0x7fcfe0119d90 con 0x7fcfe010e950 2024-05-20T20:20:42.298 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfdde00700 1 --2- >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcfe0109a80 0x7fcfe01a00c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfd7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe010a440 0x7fcfe01a0600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfde800700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcfe010e950 0x7fcfe01a4990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfd7e00700 1 --2- >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe010a440 0x7fcfe01a0600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.33:3300/0 says I am v2:172.21.15.28:35016/0 (socket says 172.21.15.28:35016) 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.296+0000 7fcfd7e00700 1 -- 172.21.15.28:0/194708011 learned_addr learned my addr 172.21.15.28:0/194708011 (peer_addr_for_me v2:172.21.15.28:0/0) 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd7e00700 1 -- 172.21.15.28:0/194708011 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcfe010e950 msgr2=0x7fcfe01a4990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd7e00700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcfe010e950 0x7fcfe01a4990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd7e00700 1 -- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcfe0109a80 msgr2=0x7fcfe01a00c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd7e00700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcfe0109a80 0x7fcfe01a00c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd7e00700 1 -- 172.21.15.28:0/194708011 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcfe01a50d0 con 0x7fcfe010a440 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd7e00700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe010a440 0x7fcfe01a0600 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7fcfc80057c0 tx=0x7fcfc80057f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd6a00700 1 -- 172.21.15.28:0/194708011 <== mon.2 v2:172.21.15.33:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcfc8027070 con 0x7fcfe010a440 2024-05-20T20:20:42.299 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd6a00700 1 -- 172.21.15.28:0/194708011 <== mon.2 v2:172.21.15.33:3300/0 2 ==== config(26 keys) ==== 1101+0+0 (secure 0 0 0) 0x7fcfc8022470 con 0x7fcfe010a440 2024-05-20T20:20:42.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfd6a00700 1 -- 172.21.15.28:0/194708011 <== mon.2 v2:172.21.15.33:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcfc8017650 con 0x7fcfe010a440 2024-05-20T20:20:42.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.297+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcfe01a5360 con 0x7fcfe010a440 2024-05-20T20:20:42.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.298+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcfe01acc40 con 0x7fcfe010a440 2024-05-20T20:20:42.300 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.299+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcfe010b170 con 0x7fcfe010a440 2024-05-20T20:20:42.301 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.301+0000 7fcfd6a00700 1 -- 172.21.15.28:0/194708011 <== mon.2 v2:172.21.15.33:3300/0 4 ==== mgrmap(e 27) ==== 149783+0+0 (secure 0 0 0) 0x7fcfc802a070 con 0x7fcfe010a440 2024-05-20T20:20:42.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.303+0000 7fcfd6a00700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcfc40b29f0 0x7fcfc40b4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-05-20T20:20:42.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.303+0000 7fcfd6a00700 1 -- 172.21.15.28:0/194708011 <== mon.2 v2:172.21.15.33:3300/0 5 ==== osd_map(101..101 src has 1..101) ==== 8403+0+0 (secure 0 0 0) 0x7fcfc80e3380 con 0x7fcfe010a440 2024-05-20T20:20:42.303 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.303+0000 7fcfdde00700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcfc40b29f0 0x7fcfc40b4eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-05-20T20:20:42.304 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.304+0000 7fcfdde00700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcfc40b29f0 0x7fcfc40b4eb0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fcfd0009be0 tx=0x7fcfd0009260 comp rx=0 tx=0).ready entity=mgr.25120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-05-20T20:20:42.305 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.305+0000 7fcfd6a00700 1 -- 172.21.15.28:0/194708011 <== mon.2 v2:172.21.15.33:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+191963 (secure 0 0 0) 0x7fcfc8013070 con 0x7fcfe010a440 2024-05-20T20:20:42.543 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:42.542+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 --> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) -- 0x7fcfe01a1420 con 0x7fcfe010a440 2024-05-20T20:20:43.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.418+0000 7fcfd6a00700 1 -- 172.21.15.28:0/194708011 <== mon.2 v2:172.21.15.33:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v102) ==== 169+0+0 (secure 0 0 0) 0x7fcfc80a4e20 con 0x7fcfe010a440 2024-05-20T20:20:43.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.428+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcfc40b29f0 msgr2=0x7fcfc40b4eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:43.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.428+0000 7fcfdf200700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcfc40b29f0 0x7fcfc40b4eb0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fcfd0009be0 tx=0x7fcfd0009260 comp rx=0 tx=0).stop 2024-05-20T20:20:43.429 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.428+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe010a440 msgr2=0x7fcfe01a0600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-05-20T20:20:43.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.428+0000 7fcfdf200700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe010a440 0x7fcfe01a0600 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7fcfc80057c0 tx=0x7fcfc80057f0 comp rx=0 tx=0).stop 2024-05-20T20:20:43.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 shutdown_connections 2024-05-20T20:20:43.430 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcfe010e950 0x7fcfe01a4990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:43.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] conn(0x7fcfe0109a80 0x7fcfe01a00c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:43.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.28:6800/2022328614,v1:172.21.15.28:6801/2022328614] conn(0x7fcfc40b29f0 0x7fcfc40b4eb0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:43.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 --2- 172.21.15.28:0/194708011 >> [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] conn(0x7fcfe010a440 0x7fcfe01a0600 unknown :-1 s=CLOSED pgs=651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-05-20T20:20:43.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 >> 172.21.15.28:0/194708011 conn(0x7fcfe00792d0 msgr2=0x7fcfe0079950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-05-20T20:20:43.431 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 shutdown_connections 2024-05-20T20:20:43.432 INFO:teuthology.orchestra.run.smithi028.stderr:2024-05-20T20:20:43.429+0000 7fcfdf200700 1 -- 172.21.15.28:0/194708011 wait complete. 2024-05-20T20:20:43.435 INFO:teuthology.orchestra.run.smithi028.stderr:pool 'cephfs_data' removed 2024-05-20T20:20:43.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:43 smithi028 ceph-mon[31178]: pgmap v475: 65 pgs: 65 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:20:43.805 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:43 smithi028 ceph-mon[31178]: from='client.? 172.21.15.28:0/194708011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:43.806 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:43 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:43.876 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:43 smithi154 ceph-mon[36314]: pgmap v475: 65 pgs: 65 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:20:43.877 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:43 smithi154 ceph-mon[36314]: from='client.? 172.21.15.28:0/194708011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:43.877 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:43 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:43 smithi033 ceph-mon[36179]: pgmap v475: 65 pgs: 65 active+clean; 37 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail 2024-05-20T20:20:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:43 smithi033 ceph-mon[36179]: from='client.? 172.21.15.28:0/194708011' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:43.878 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:43 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-05-20T20:20:44.239 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T20:20:44.350 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T20:20:44.383 DEBUG:teuthology.orchestra.run.smithi154:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T20:20:44.416 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-20T20:20:44.416 DEBUG:teuthology.orchestra.run.smithi028:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-20T20:20:44.466 DEBUG:teuthology.orchestra.run.smithi033:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-20T20:20:44.484 DEBUG:teuthology.orchestra.run.smithi154:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-20T20:20:44.501 INFO:tasks.cephadm:Stopping all daemons... 2024-05-20T20:20:44.502 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-05-20T20:20:44.502 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a 2024-05-20T20:20:44.553 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:44 smithi028 ceph-mon[31178]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-05-20T20:20:44.554 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:44 smithi028 ceph-mon[31178]: osdmap e102: 12 total, 12 up, 12 in 2024-05-20T20:20:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:44 smithi154 ceph-mon[36314]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-05-20T20:20:44.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:44 smithi154 ceph-mon[36314]: osdmap e102: 12 total, 12 up, 12 in 2024-05-20T20:20:44.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:44 smithi033 ceph-mon[36179]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-05-20T20:20:44.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:44 smithi033 ceph-mon[36179]: osdmap e102: 12 total, 12 up, 12 in 2024-05-20T20:20:45.053 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:44 smithi028 systemd[1]: Stopping Ceph mon.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:20:45.451 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a[31174]: 2024-05-20T20:20:45.135+0000 7f0097e00700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:20:45.451 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a[31174]: 2024-05-20T20:20:45.135+0000 7f0097e00700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-05-20T20:20:45.451 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:45 smithi028 podman[124721]: 2024-05-20 20:20:45.249844707 +0000 UTC m=+0.311679797 container died ea4da3bc48153474d3c2669c839a61eb5ff45779cc6c5684da9482da426a7dea (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-05-20T20:20:46.090 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:45 smithi028 podman[124721]: 2024-05-20 20:20:45.805208408 +0000 UTC m=+0.867043492 container cleanup ea4da3bc48153474d3c2669c839a61eb5ff45779cc6c5684da9482da426a7dea (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2) 2024-05-20T20:20:46.090 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:45 smithi028 bash[124721]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a 2024-05-20T20:20:46.438 INFO:journalctl@ceph.mon.a.smithi028.stdout:May 20 20:20:46 smithi028 podman[124733]: 2024-05-20 20:20:46.090207343 +0000 UTC m=+0.825321411 container remove ea4da3bc48153474d3c2669c839a61eb5ff45779cc6c5684da9482da426a7dea (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-a, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514) 2024-05-20T20:20:46.467 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.a.service' 2024-05-20T20:20:46.570 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:20:46.570 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-05-20T20:20:46.571 INFO:tasks.cephadm.mon.c:Stopping mon.b... 2024-05-20T20:20:46.571 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.b 2024-05-20T20:20:46.876 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:46 smithi033 systemd[1]: Stopping Ceph mon.b for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:20:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:47 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-b[36175]: 2024-05-20T20:20:47.664+0000 7f021f592700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:20:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:47 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-b[36175]: 2024-05-20T20:20:47.664+0000 7f021f592700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-05-20T20:20:48.127 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:47 smithi033 podman[73177]: 2024-05-20 20:20:47.899804442 +0000 UTC m=+1.051678944 container died 63bf016a4da01c17928f9b31bd96c6c313b8d887808c99a225f94316df9b24ac (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-20T20:20:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:51 smithi033 podman[73177]: 2024-05-20 20:20:51.425752242 +0000 UTC m=+4.577626726 container cleanup 63bf016a4da01c17928f9b31bd96c6c313b8d887808c99a225f94316df9b24ac (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-05-20T20:20:51.877 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:51 smithi033 bash[73177]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-b 2024-05-20T20:20:53.988 INFO:journalctl@ceph.mon.b.smithi033.stdout:May 20 20:20:53 smithi033 podman[73189]: 2024-05-20 20:20:53.719816634 +0000 UTC m=+5.813628423 container remove 63bf016a4da01c17928f9b31bd96c6c313b8d887808c99a225f94316df9b24ac (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-20T20:20:53.992 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.b.service' 2024-05-20T20:20:54.196 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:20:54.196 INFO:tasks.cephadm.mon.c:Stopped mon.b 2024-05-20T20:20:54.196 INFO:tasks.cephadm.mon.c:Stopping mon.c... 2024-05-20T20:20:54.197 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.c 2024-05-20T20:20:54.521 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:54 smithi154 systemd[1]: Stopping Ceph mon.c for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:20:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:54 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c[36310]: 2024-05-20T20:20:54.520+0000 7f8935ec5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:20:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:54 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c[36310]: 2024-05-20T20:20:54.520+0000 7f8935ec5700 -1 mon.c@1(probing) e3 *** Got Signal Terminated *** 2024-05-20T20:20:54.875 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:54 smithi154 podman[74325]: 2024-05-20 20:20:54.745687653 +0000 UTC m=+0.348962728 container died 7216247a8a467d2978d24ab6059ba653b89fb4da236f1cb9b791d9b98a07d077 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-20T20:20:55.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:55 smithi154 podman[74325]: 2024-05-20 20:20:55.090258772 +0000 UTC m=+0.693533842 container cleanup 7216247a8a467d2978d24ab6059ba653b89fb4da236f1cb9b791d9b98a07d077 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:20:55.375 INFO:journalctl@ceph.mon.c.smithi154.stdout:May 20 20:20:55 smithi154 bash[74325]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mon-c 2024-05-20T20:20:55.527 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mon.c.service' 2024-05-20T20:20:55.572 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:20:55.572 INFO:tasks.cephadm.mon.c:Stopped mon.c 2024-05-20T20:20:55.572 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-05-20T20:20:55.573 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.x 2024-05-20T20:20:56.054 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:20:55 smithi028 systemd[1]: Stopping Ceph mgr.x for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:20:56.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:20:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x[31410]: 2024-05-20T20:20:56.208+0000 7f7de6200700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:20:56.554 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:20:56 smithi028 podman[124843]: 2024-05-20 20:20:56.306223916 +0000 UTC m=+0.257679156 container died f9f76a24a6606ebf569f656c2589b1febd68650f1ce00b86bafdf6865bc9e98b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS) 2024-05-20T20:20:57.304 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:20:56 smithi028 podman[124843]: 2024-05-20 20:20:56.919323234 +0000 UTC m=+0.870778472 container cleanup f9f76a24a6606ebf569f656c2589b1febd68650f1ce00b86bafdf6865bc9e98b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-05-20T20:20:57.305 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:20:56 smithi028 bash[124843]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x 2024-05-20T20:20:57.305 INFO:journalctl@ceph.mgr.x.smithi028.stdout:May 20 20:20:57 smithi028 podman[124856]: 2024-05-20 20:20:57.137926116 +0000 UTC m=+0.820576311 container remove f9f76a24a6606ebf569f656c2589b1febd68650f1ce00b86bafdf6865bc9e98b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:4ebd1d5361d457738957dc0455bba90102296634, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T20:20:57.671 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.x.service' 2024-05-20T20:20:57.790 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:20:57.791 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-05-20T20:20:57.791 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-05-20T20:20:57.791 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.y 2024-05-20T20:20:58.126 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:20:57 smithi033 systemd[1]: Stopping Ceph mgr.y for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:20:58.465 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:20:58 smithi033 podman[73301]: 2024-05-20 20:20:58.135287224 +0000 UTC m=+0.142846604 container died 466c25eca87c4317992dcd6655c7efef088a563e9349162f41d06022bc04bcf1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T20:20:58.799 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:20:58 smithi033 podman[73301]: 2024-05-20 20:20:58.565152242 +0000 UTC m=+0.572711614 container cleanup 466c25eca87c4317992dcd6655c7efef088a563e9349162f41d06022bc04bcf1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-20T20:20:58.799 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:20:58 smithi033 bash[73301]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y 2024-05-20T20:20:59.049 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:20:58 smithi033 podman[73315]: 2024-05-20 20:20:58.799913234 +0000 UTC m=+0.659342490 container remove 466c25eca87c4317992dcd6655c7efef088a563e9349162f41d06022bc04bcf1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-y, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-20T20:20:59.050 INFO:journalctl@ceph.mgr.y.smithi033.stdout:May 20 20:20:58 smithi033 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-05-20T20:20:59.052 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.y.service' 2024-05-20T20:20:59.086 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:20:59.087 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-05-20T20:20:59.087 INFO:tasks.cephadm.mgr.z:Stopping mgr.z... 2024-05-20T20:20:59.087 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.z 2024-05-20T20:20:59.375 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:20:59 smithi154 systemd[1]: Stopping Ceph mgr.z for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:20:59.875 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:20:59 smithi154 podman[74449]: 2024-05-20 20:20:59.433090189 +0000 UTC m=+0.147246478 container died 445f1d9e01f49f0f710389465fd7c2fd7370e3c5d42f00566b6abd0986a82887 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-20T20:21:00.205 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:20:59 smithi154 podman[74449]: 2024-05-20 20:20:59.979197317 +0000 UTC m=+0.693353605 container cleanup 445f1d9e01f49f0f710389465fd7c2fd7370e3c5d42f00566b6abd0986a82887 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-05-20T20:21:00.205 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:20:59 smithi154 bash[74449]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z 2024-05-20T20:21:00.453 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.z.service' 2024-05-20T20:21:00.474 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:21:00 smithi154 podman[74464]: 2024-05-20 20:21:00.205896953 +0000 UTC m=+0.764811723 container remove 445f1d9e01f49f0f710389465fd7c2fd7370e3c5d42f00566b6abd0986a82887 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mgr-z, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-05-20T20:21:00.474 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:21:00 smithi154 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.z.service: Main process exited, code=exited, status=143/n/a 2024-05-20T20:21:00.474 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:21:00 smithi154 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.z.service: Failed with result 'exit-code'. 2024-05-20T20:21:00.474 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:21:00 smithi154 systemd[1]: Stopped Ceph mgr.z for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:21:00.474 INFO:journalctl@ceph.mgr.z.smithi154.stdout:May 20 20:21:00 smithi154 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mgr.z.service: Consumed 22.800s CPU time. 2024-05-20T20:21:00.487 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:21:00.487 INFO:tasks.cephadm.mgr.z:Stopped mgr.z 2024-05-20T20:21:00.487 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-20T20:21:00.487 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.0 2024-05-20T20:21:01.054 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:00 smithi028 systemd[1]: Stopping Ceph osd.0 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:21:01.554 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0[45678]: 2024-05-20T20:21:01.072+0000 7efe3a400700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:21:01.555 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0[45678]: 2024-05-20T20:21:01.072+0000 7efe3a400700 -1 osd.0 102 *** Got signal Terminated *** 2024-05-20T20:21:01.555 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0[45678]: 2024-05-20T20:21:01.072+0000 7efe3a400700 -1 osd.0 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:21:07.263 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:06 smithi028 podman[124965]: 2024-05-20 20:21:06.879794582 +0000 UTC m=+5.931143017 container stop f0c972ccb51636a7a62e1aec660949844845c21b465eb9801c41aa97e43472ca (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-05-20T20:21:07.264 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:06 smithi028 podman[124965]: 2024-05-20 20:21:06.905676518 +0000 UTC m=+5.957024954 container died f0c972ccb51636a7a62e1aec660949844845c21b465eb9801c41aa97e43472ca (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-20T20:21:07.513 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:07 smithi028 podman[124965]: 2024-05-20 20:21:07.491227796 +0000 UTC m=+6.542576272 container cleanup f0c972ccb51636a7a62e1aec660949844845c21b465eb9801c41aa97e43472ca (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-20T20:21:07.513 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:07 smithi028 bash[124965]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0 2024-05-20T20:21:08.054 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:07 smithi028 podman[124984]: 2024-05-20 20:21:07.709730819 +0000 UTC m=+0.808006495 container remove f0c972ccb51636a7a62e1aec660949844845c21b465eb9801c41aa97e43472ca (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, io.buildah.version=1.33.5) 2024-05-20T20:21:08.804 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:08 smithi028 podman[125069]: 2024-05-20 20:21:08.437947594 +0000 UTC m=+0.037333100 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:21:08.804 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:08 smithi028 podman[125069]: 2024-05-20 20:21:08.56070588 +0000 UTC m=+0.160091429 container create ba1f83a35301ac9886b7c4edfa8899c80eca973962f7dffe60ba71ad21185b3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T20:21:09.304 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:08 smithi028 podman[125069]: 2024-05-20 20:21:08.847912814 +0000 UTC m=+0.447298364 container init ba1f83a35301ac9886b7c4edfa8899c80eca973962f7dffe60ba71ad21185b3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:21:09.305 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:08 smithi028 podman[125069]: 2024-05-20 20:21:08.859805446 +0000 UTC m=+0.459190971 container start ba1f83a35301ac9886b7c4edfa8899c80eca973962f7dffe60ba71ad21185b3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-05-20T20:21:09.305 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:08 smithi028 podman[125069]: 2024-05-20 20:21:08.963325025 +0000 UTC m=+0.562710611 container attach ba1f83a35301ac9886b7c4edfa8899c80eca973962f7dffe60ba71ad21185b3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-20T20:21:10.054 INFO:journalctl@ceph.osd.0.smithi028.stdout:May 20 20:21:09 smithi028 podman[125069]: 2024-05-20 20:21:09.71581988 +0000 UTC m=+1.315205429 container died ba1f83a35301ac9886b7c4edfa8899c80eca973962f7dffe60ba71ad21185b3d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.build-date=20240514, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-05-20T20:21:10.607 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.0.service' 2024-05-20T20:21:10.723 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:21:10.724 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-20T20:21:10.724 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-20T20:21:10.724 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.1 2024-05-20T20:21:11.053 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:10 smithi033 systemd[1]: Stopping Ceph osd.1 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:21:11.377 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1[40730]: 2024-05-20T20:21:11.052+0000 7f7140cd6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:21:11.377 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1[40730]: 2024-05-20T20:21:11.052+0000 7f7140cd6700 -1 osd.1 102 *** Got signal Terminated *** 2024-05-20T20:21:11.377 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1[40730]: 2024-05-20T20:21:11.052+0000 7f7140cd6700 -1 osd.1 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:21:17.070 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:16 smithi033 podman[73435]: 2024-05-20 20:21:16.755510946 +0000 UTC m=+5.824541574 container died d0985d6330fbd1698b3d09f92d7e9aa0636b602f1a64114c55d07965d0ad9efb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:21:17.326 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:17 smithi033 podman[73435]: 2024-05-20 20:21:17.169713399 +0000 UTC m=+6.238744008 container cleanup d0985d6330fbd1698b3d09f92d7e9aa0636b602f1a64114c55d07965d0ad9efb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:21:17.326 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:17 smithi033 bash[73435]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1 2024-05-20T20:21:17.626 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:17 smithi033 podman[73450]: 2024-05-20 20:21:17.396165829 +0000 UTC m=+0.643417270 container remove d0985d6330fbd1698b3d09f92d7e9aa0636b602f1a64114c55d07965d0ad9efb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T20:21:18.091 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:17 smithi033 podman[73540]: 2024-05-20 20:21:17.710930844 +0000 UTC m=+0.017628670 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:21:18.092 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:17 smithi033 podman[73540]: 2024-05-20 20:21:17.838752273 +0000 UTC m=+0.145450091 container create 0f9239a877079784d6ad526aedae4d9dfb24a17eb0392ff95c2e0d9aecc39cff (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:21:18.377 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:18 smithi033 podman[73540]: 2024-05-20 20:21:18.092277588 +0000 UTC m=+0.398975406 container init 0f9239a877079784d6ad526aedae4d9dfb24a17eb0392ff95c2e0d9aecc39cff (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-05-20T20:21:18.377 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:18 smithi033 podman[73540]: 2024-05-20 20:21:18.097335894 +0000 UTC m=+0.404033715 container start 0f9239a877079784d6ad526aedae4d9dfb24a17eb0392ff95c2e0d9aecc39cff (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1-deactivate, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T20:21:18.377 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:18 smithi033 podman[73540]: 2024-05-20 20:21:18.191716604 +0000 UTC m=+0.498414425 container attach 0f9239a877079784d6ad526aedae4d9dfb24a17eb0392ff95c2e0d9aecc39cff (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-20T20:21:18.745 INFO:journalctl@ceph.osd.1.smithi033.stdout:May 20 20:21:18 smithi033 podman[73540]: 2024-05-20 20:21:18.41509221 +0000 UTC m=+0.721790031 container died 0f9239a877079784d6ad526aedae4d9dfb24a17eb0392ff95c2e0d9aecc39cff (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-1-deactivate, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:21:19.073 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.1.service' 2024-05-20T20:21:19.113 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:21:19.113 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-20T20:21:19.113 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-20T20:21:19.113 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.2 2024-05-20T20:21:19.440 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:19 smithi154 systemd[1]: Stopping Ceph osd.2 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:21:19.875 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2[41409]: 2024-05-20T20:21:19.440+0000 7efe54e73700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:21:19.875 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2[41409]: 2024-05-20T20:21:19.440+0000 7efe54e73700 -1 osd.2 102 *** Got signal Terminated *** 2024-05-20T20:21:19.875 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2[41409]: 2024-05-20T20:21:19.440+0000 7efe54e73700 -1 osd.2 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:21:25.295 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:25 smithi154 podman[74573]: 2024-05-20 20:21:25.041241329 +0000 UTC m=+5.728804084 container died 74ea13dcc91687549c15614bedb809dd302aa2b9dbfed492b1346e0cb90ef8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:21:25.562 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:25 smithi154 podman[74573]: 2024-05-20 20:21:25.507992673 +0000 UTC m=+6.195555419 container cleanup 74ea13dcc91687549c15614bedb809dd302aa2b9dbfed492b1346e0cb90ef8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-05-20T20:21:25.562 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:25 smithi154 bash[74573]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2 2024-05-20T20:21:25.875 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:25 smithi154 podman[74590]: 2024-05-20 20:21:25.767791075 +0000 UTC m=+0.723092429 container remove 74ea13dcc91687549c15614bedb809dd302aa2b9dbfed492b1346e0cb90ef8e7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240514) 2024-05-20T20:21:26.375 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:26 smithi154 podman[74684]: 2024-05-20 20:21:26.015873095 +0000 UTC m=+0.018933758 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:21:26.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:26 smithi154 podman[74684]: 2024-05-20 20:21:26.143761626 +0000 UTC m=+0.146822270 container create c042c5274a816ef219e6a6492c81071f3c9f1b1c523629bbc55d084085b2c606 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD) 2024-05-20T20:21:26.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:26 smithi154 podman[74684]: 2024-05-20 20:21:26.354641132 +0000 UTC m=+0.357701786 container init c042c5274a816ef219e6a6492c81071f3c9f1b1c523629bbc55d084085b2c606 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True) 2024-05-20T20:21:26.376 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:26 smithi154 podman[74684]: 2024-05-20 20:21:26.358278253 +0000 UTC m=+0.361338912 container start c042c5274a816ef219e6a6492c81071f3c9f1b1c523629bbc55d084085b2c606 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-20T20:21:26.661 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:26 smithi154 podman[74684]: 2024-05-20 20:21:26.505262683 +0000 UTC m=+0.508323334 container attach c042c5274a816ef219e6a6492c81071f3c9f1b1c523629bbc55d084085b2c606 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-05-20T20:21:27.025 INFO:journalctl@ceph.osd.2.smithi154.stdout:May 20 20:21:26 smithi154 podman[74684]: 2024-05-20 20:21:26.661745688 +0000 UTC m=+0.664806340 container died c042c5274a816ef219e6a6492c81071f3c9f1b1c523629bbc55d084085b2c606 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:21:27.364 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.2.service' 2024-05-20T20:21:27.403 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:21:27.403 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-20T20:21:27.403 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-20T20:21:27.404 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.3 2024-05-20T20:21:27.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:27.341+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:27.804 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:27 smithi028 systemd[1]: Stopping Ceph osd.3 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:21:28.304 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3[52606]: 2024-05-20T20:21:28.034+0000 7f1ba2c00700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:21:28.305 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3[52606]: 2024-05-20T20:21:28.034+0000 7f1ba2c00700 -1 osd.3 102 *** Got signal Terminated *** 2024-05-20T20:21:28.305 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:28 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3[52606]: 2024-05-20T20:21:28.034+0000 7f1ba2c00700 -1 osd.3 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:21:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:27.941+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:28.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:28.304+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:29.126 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:28.990+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:29.554 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3[52606]: 2024-05-20T20:21:29.249+0000 7f1b9e400700 -1 osd.3 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:03.128163+0000 front 2024-05-20T20:21:03.129319+0000 (oldest deadline 2024-05-20T20:21:28.428347+0000) 2024-05-20T20:21:29.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:29.303+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:30.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:29 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:29.710+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:30.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:29.964+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:30.554 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:30 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3[52606]: 2024-05-20T20:21:30.204+0000 7f1b9e400700 -1 osd.3 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:03.128163+0000 front 2024-05-20T20:21:03.129319+0000 (oldest deadline 2024-05-20T20:21:28.428347+0000) 2024-05-20T20:21:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:30.299+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:31.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:30 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:30.699+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:31.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:30.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:31.489 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:31 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3[52606]: 2024-05-20T20:21:31.234+0000 7f1b9e400700 -1 osd.3 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:03.128163+0000 front 2024-05-20T20:21:03.129319+0000 (oldest deadline 2024-05-20T20:21:28.428347+0000) 2024-05-20T20:21:31.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:31.283+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:31.748 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:31 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:31.488+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:32.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:31 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:31.746+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:32.238 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:31.926+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:32.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:31.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:32.512 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:32 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3[52606]: 2024-05-20T20:21:32.253+0000 7f1b9e400700 -1 osd.3 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:03.128163+0000 front 2024-05-20T20:21:03.129319+0000 (oldest deadline 2024-05-20T20:21:28.428347+0000) 2024-05-20T20:21:32.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:32.238+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:32.789 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:32 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:32.512+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:33.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:32 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:32.788+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:33.375 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:32.966+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:33.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:33.194+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:33.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:32.982+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:33.804 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:33 smithi028 podman[125172]: 2024-05-20 20:21:33.771854591 +0000 UTC m=+5.906184763 container died 468766d7c39cf9a5e7d72035e7c3dd35bf2fd639a4fcceaa259272481d55177f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True) 2024-05-20T20:21:33.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:33.547+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:34.092 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:33 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:33.825+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:34.375 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:33.975+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:34.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:34.153+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:34.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:33 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:33.964+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:34.689 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:34 smithi028 podman[125172]: 2024-05-20 20:21:34.370091217 +0000 UTC m=+6.504421452 container cleanup 468766d7c39cf9a5e7d72035e7c3dd35bf2fd639a4fcceaa259272481d55177f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-05-20T20:21:34.689 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:34 smithi028 bash[125172]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3 2024-05-20T20:21:34.689 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:34 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:34.587+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:35.054 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:34 smithi028 podman[125190]: 2024-05-20 20:21:34.688261791 +0000 UTC m=+0.876210888 container remove 468766d7c39cf9a5e7d72035e7c3dd35bf2fd639a4fcceaa259272481d55177f (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-20T20:21:35.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:34 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:34.825+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:35.375 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:34.944+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:35.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:35.153+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:35.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:34.950+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:35.670 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:35.569+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:35.670 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:35 smithi028 podman[125279]: 2024-05-20 20:21:35.239386267 +0000 UTC m=+0.036978404 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:21:35.671 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:35 smithi028 podman[125279]: 2024-05-20 20:21:35.347698584 +0000 UTC m=+0.145290752 container create e4d0421f11413849f4d0e447a20f96facd9707bb4ab0f8d7e715e2d8cfaaa90c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-05-20T20:21:36.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:35 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:35.855+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:36.054 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:35 smithi028 podman[125279]: 2024-05-20 20:21:35.669347259 +0000 UTC m=+0.466939471 container init e4d0421f11413849f4d0e447a20f96facd9707bb4ab0f8d7e715e2d8cfaaa90c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-20T20:21:36.054 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:35 smithi028 podman[125279]: 2024-05-20 20:21:35.682494531 +0000 UTC m=+0.480086709 container start e4d0421f11413849f4d0e447a20f96facd9707bb4ab0f8d7e715e2d8cfaaa90c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:21:36.054 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:35 smithi028 podman[125279]: 2024-05-20 20:21:35.775086344 +0000 UTC m=+0.572678519 container attach e4d0421f11413849f4d0e447a20f96facd9707bb4ab0f8d7e715e2d8cfaaa90c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:21:36.278 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:35 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:35.940+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:36.375 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:35.968+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:36.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:36.188+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:36.489 INFO:journalctl@ceph.osd.3.smithi028.stdout:May 20 20:21:36 smithi028 podman[125279]: 2024-05-20 20:21:36.151970165 +0000 UTC m=+0.949562360 container died e4d0421f11413849f4d0e447a20f96facd9707bb4ab0f8d7e715e2d8cfaaa90c (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-05-20T20:21:36.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:36 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:36.594+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:36.873 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.3.service' 2024-05-20T20:21:36.988 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:21:36.988 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-20T20:21:36.988 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-20T20:21:36.988 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.4 2024-05-20T20:21:37.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:36 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:36.806+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:37.306 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:36 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:36.944+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:37.306 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:37 smithi033 systemd[1]: Stopping Ceph osd.4 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:21:37.307 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:37 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:37.111+0000 7f63ddff7700 -1 osd.4 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.320428+0000 front 2024-05-20T20:21:13.320453+0000 (oldest deadline 2024-05-20T20:21:36.820638+0000) 2024-05-20T20:21:37.307 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:37 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:37.306+0000 7f63e1a03700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:21:37.307 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:37 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:37.306+0000 7f63e1a03700 -1 osd.4 102 *** Got signal Terminated *** 2024-05-20T20:21:37.375 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:36.973+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:37.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:37.220+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:37.565 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:37 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:37.306+0000 7f63e1a03700 -1 osd.4 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:21:38.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:37.780+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:38.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:37.780+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:38.054 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:37.622+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:38.054 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:37 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:37.622+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:38.270 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:37.990+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:38.270 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:37.990+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:38.377 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:38 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:38.114+0000 7f63ddff7700 -1 osd.4 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.320428+0000 front 2024-05-20T20:21:13.320453+0000 (oldest deadline 2024-05-20T20:21:36.820638+0000) 2024-05-20T20:21:38.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:38 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:38.040+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:38.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:37 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:37.925+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:38.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:37 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:37.925+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:38.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:38.270+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:39.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:38.741+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:39.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:38.741+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:39.054 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:38.615+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:39.054 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:38 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:38.615+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:39.241 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:38.959+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:39.242 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:38.959+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:39.377 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:39 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:39.112+0000 7f63ddff7700 -1 osd.4 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.320428+0000 front 2024-05-20T20:21:13.320453+0000 (oldest deadline 2024-05-20T20:21:36.820638+0000) 2024-05-20T20:21:39.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:39 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:39.032+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:39.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:38 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:38.901+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:39.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:38 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:38.901+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:39.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:39.241+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:40.055 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:39 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:39.581+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:40.055 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:39 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:39.581+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:40.055 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:39 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:39.701+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:40.056 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:39 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:39.701+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:40.260 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:39.973+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:40.260 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:39.973+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:40.377 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:40 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:40.089+0000 7f63ddff7700 -1 osd.4 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.320428+0000 front 2024-05-20T20:21:13.320453+0000 (oldest deadline 2024-05-20T20:21:36.820638+0000) 2024-05-20T20:21:40.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:40 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:40.060+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:40.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:39 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:39.942+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:40.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:39 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:39.942+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:40.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:40.261+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:40.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:40.261+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:40.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:40 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:40.543+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:40.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:40 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:40.543+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:40.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:40 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:40.689+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:40.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:40 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:40.689+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:41.247 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:41.023+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:41.248 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:41.023+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:41.248 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:40.927+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:41.377 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:41 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:41.055+0000 7f63ddff7700 -1 osd.4 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.320428+0000 front 2024-05-20T20:21:13.320453+0000 (oldest deadline 2024-05-20T20:21:36.820638+0000) 2024-05-20T20:21:41.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:41 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:41.034+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:41.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:40 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:40.988+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:41.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:40 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:40.988+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:41.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:41.247+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:41.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:41.247+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:41.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:41 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:41.497+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:41.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:41 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:41.497+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:41.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:41 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:41.693+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:41.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:41 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:41.693+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:42.278 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:41.926+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:42.279 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:41.994+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:42.279 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:41.994+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:42.377 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:42 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4[47397]: 2024-05-20T20:21:42.046+0000 7f63ddff7700 -1 osd.4 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.320428+0000 front 2024-05-20T20:21:13.320453+0000 (oldest deadline 2024-05-20T20:21:36.820638+0000) 2024-05-20T20:21:42.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:42 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:42.020+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:42.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:41 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:41.952+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:42.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:41 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:41.952+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:42.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:42.278+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:42.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:42.278+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:42.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:42 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:42.459+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:42.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:42 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:42.459+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:42.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:42 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:42.693+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:42.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:42 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:42.693+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:43.322 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:42.920+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:43.322 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:42.953+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:43.322 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:42.953+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:43.334 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:43 smithi033 podman[73668]: 2024-05-20 20:21:43.019495702 +0000 UTC m=+5.814772129 container died 1bde32d71d4da52732ad78d3be93dc966f75b5baf88905fad377446b8f078a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:21:43.334 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:43 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:43.032+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:43.334 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:42 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:42.925+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:43.334 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:42 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:42.925+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:43.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:43 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:43.321+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:43.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:43 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:43.321+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:43.627 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:43 smithi033 podman[73668]: 2024-05-20 20:21:43.415132253 +0000 UTC m=+6.210408668 container cleanup 1bde32d71d4da52732ad78d3be93dc966f75b5baf88905fad377446b8f078a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-05-20T20:21:43.627 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:43 smithi033 bash[73668]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4 2024-05-20T20:21:43.696 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:43 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:43.427+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:43.696 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:43 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:43.427+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:43.968 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:43 smithi033 podman[73686]: 2024-05-20 20:21:43.675059701 +0000 UTC m=+0.659262291 container remove 1bde32d71d4da52732ad78d3be93dc966f75b5baf88905fad377446b8f078a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:21:44.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:43 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:43.696+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:44.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:43 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:43.696+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:44.229 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:44 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:44.059+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:44.229 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:43 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:43.967+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:44.229 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:43 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:43.967+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:44.230 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:44 smithi033 podman[73777]: 2024-05-20 20:21:43.951061929 +0000 UTC m=+0.017608767 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:21:44.230 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:44 smithi033 podman[73777]: 2024-05-20 20:21:44.075844534 +0000 UTC m=+0.142391365 container create d3c9e6c35362a12ec4ddcde331549c1f622964f425ee0ca84353ca60316e8306 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514) 2024-05-20T20:21:44.335 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:43 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:43.957+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:44.335 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:43 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:43.957+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:44.335 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:43 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:43.909+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:44.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:44 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:44.333+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:44.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:44 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:44.333+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:44.627 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:44 smithi033 podman[73777]: 2024-05-20 20:21:44.370420375 +0000 UTC m=+0.436967215 container init d3c9e6c35362a12ec4ddcde331549c1f622964f425ee0ca84353ca60316e8306 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True) 2024-05-20T20:21:44.627 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:44 smithi033 podman[73777]: 2024-05-20 20:21:44.374133151 +0000 UTC m=+0.440679983 container start d3c9e6c35362a12ec4ddcde331549c1f622964f425ee0ca84353ca60316e8306 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4-deactivate, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2) 2024-05-20T20:21:44.627 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:44 smithi033 podman[73777]: 2024-05-20 20:21:44.470212231 +0000 UTC m=+0.536759072 container attach d3c9e6c35362a12ec4ddcde331549c1f622964f425ee0ca84353ca60316e8306 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-05-20T20:21:44.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:44 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:44.445+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:44.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:44 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:44.445+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:44.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:44 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:44.690+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:44.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:44 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:44.690+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:44.955 INFO:journalctl@ceph.osd.4.smithi033.stdout:May 20 20:21:44 smithi033 podman[73777]: 2024-05-20 20:21:44.694555755 +0000 UTC m=+0.761102589 container died d3c9e6c35362a12ec4ddcde331549c1f622964f425ee0ca84353ca60316e8306 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-20T20:21:45.299 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:44 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:44.956+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:45.300 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:44 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:44.956+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:45.300 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:44 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:44.924+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:45.332 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:44 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:44.952+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:45.332 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:44 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:44.952+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:45.332 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:45 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:45.104+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:45.332 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:45 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:45.104+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:45.357 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.4.service' 2024-05-20T20:21:45.403 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:21:45.403 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-20T20:21:45.403 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-20T20:21:45.404 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.5 2024-05-20T20:21:45.625 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:45 smithi154 systemd[1]: Stopping Ceph osd.5 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:21:45.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:45.299+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:45.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:45.299+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:45.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:45.706+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:45.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:45.706+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:45.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:45.487+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:45.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:45.487+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:45.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:45.487+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:46.125 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:45.898+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:46.125 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:45.898+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:46.125 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:45.760+0000 7feb49a0e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:21:46.125 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:45.760+0000 7feb49a0e700 -1 osd.5 102 *** Got signal Terminated *** 2024-05-20T20:21:46.125 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:45.760+0000 7feb49a0e700 -1 osd.5 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:21:46.126 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:45.979+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:46.126 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:45 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:45.979+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:46.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:45 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:45.973+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:46.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:45 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:45.973+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:46.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:46 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:46.105+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:46.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:46 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:46.105+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:46.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:46 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:46.259+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:46.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:46 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:46.259+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:46.753 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:46 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:46.498+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:46.753 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:46 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:46.498+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:46.753 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:46 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:46.498+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:47.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:46 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:46.753+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:47.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:46 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:46.753+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:47.271 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:47 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:47.007+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:47.271 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:47 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:47.007+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:47.271 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:46 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:46.913+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:47.271 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:46 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:46.913+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:47.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:47 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:47.117+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:47.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:47 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:47.117+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:47.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:47 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:47.016+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:47.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:47 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:47.016+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:47.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:47 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:47.270+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:47.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:47 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:47.270+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:47.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:47 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:47.523+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:47.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:47 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:47.523+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:47.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:47 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:47.523+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:47.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:47 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:47.729+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:47.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:47 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:47.729+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:47.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:47 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:47.729+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:48.311 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:48 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:48.039+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:48.312 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:48 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:48.039+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:48.312 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:47 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:47.941+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:48.312 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:47 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:47.941+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:48.376 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:48 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:48.084+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:48.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:48 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:48.084+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:48.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:48 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:48.062+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:48.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:48 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:48.062+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:48.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:48 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:48.311+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:48.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:48 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:48.311+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:48.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:48 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:48.519+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:48.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:48 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:48.519+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:48.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:48 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:48.519+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:48.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:48 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:48.681+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:48.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:48 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:48.681+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:48.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:48 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:48.681+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:49.324 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:48 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:48.949+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:49.324 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:48 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:48.949+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:49.324 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:49 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:49.029+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:49.325 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:49 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:49.029+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:49.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:49 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:49.112+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:49.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:49 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:49.112+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:49.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:49 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:49.036+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:49.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:49 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:49.036+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:49.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:49 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:49.036+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:49.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:49 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:49.323+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:49.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:49 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:49.323+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:49.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:49 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:49.323+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:49.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:49 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:49.483+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:49.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:49 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:49.483+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:49.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:49 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:49.483+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:49.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:49 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:49.674+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:49.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:49 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:49.674+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:49.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:49 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:49.674+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:50.300 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:50.048+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.667683+0000 front 2024-05-20T20:21:05.667873+0000 (oldest deadline 2024-05-20T20:21:31.567503+0000) 2024-05-20T20:21:50.300 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:50.048+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:13.268715+0000 front 2024-05-20T20:21:13.268840+0000 (oldest deadline 2024-05-20T20:21:37.269604+0000) 2024-05-20T20:21:50.300 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5[48339]: 2024-05-20T20:21:50.048+0000 7feb41ffa700 -1 osd.5 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:24.171550+0000 front 2024-05-20T20:21:24.171749+0000 (oldest deadline 2024-05-20T20:21:49.471668+0000) 2024-05-20T20:21:50.301 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:49 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:49.930+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:50.301 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:49 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:49.930+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:50.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:50.066+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:50.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:50.066+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:50.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:50.007+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:50.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:50.007+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:50.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:50.007+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:50.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:50.299+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:50.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:50.299+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:50.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:50.299+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:50.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:50 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:50.444+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:50.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:50 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:50.444+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:50.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:50 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:50.444+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:50.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:50 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:50.692+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:50.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:50 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:50.692+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:50.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:50 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:50.692+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:51.313 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:50.933+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:51.314 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:50 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:50.933+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:51.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:51 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:51.057+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:51.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:51 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:51.057+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:51.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:50.961+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:51.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:50.961+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:51.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:50 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:50.961+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:51.625 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:51 smithi154 podman[74795]: 2024-05-20 20:21:51.415727747 +0000 UTC m=+5.817033382 container died 47bf7165bac1ac0cb39fa74d153b6c59150b432b979af97b26fc685339b53bde (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS) 2024-05-20T20:21:51.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:51 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:51.313+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:51.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:51 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:51.313+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:51.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:51 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:51.313+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:51.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:51 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:51.491+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:51.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:51 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:51.492+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:51.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:51 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:51.492+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:51.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:51 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:51.700+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:51.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:51 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:51.700+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:51.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:51 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:51.700+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:52.071 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:51 smithi154 podman[74795]: 2024-05-20 20:21:51.803377813 +0000 UTC m=+6.204683446 container cleanup 47bf7165bac1ac0cb39fa74d153b6c59150b432b979af97b26fc685339b53bde (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:21:52.071 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:51 smithi154 bash[74795]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5 2024-05-20T20:21:52.072 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:51 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:51.968+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:52.072 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:51 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:51.968+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:52.375 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:52 smithi154 podman[74810]: 2024-05-20 20:21:52.071403829 +0000 UTC m=+0.658778149 container remove 47bf7165bac1ac0cb39fa74d153b6c59150b432b979af97b26fc685339b53bde (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T20:21:52.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:52 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:52.295+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:52.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:52 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:52.295+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:52.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:52 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:52.295+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:52.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:52 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:52.054+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:52.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:52 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:52.054+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:52.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:51 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:51.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:52.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:51 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:51.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:52.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:51 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:51.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:52.741 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:52 smithi154 podman[74902]: 2024-05-20 20:21:52.313877899 +0000 UTC m=+0.017624034 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:21:52.741 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:52 smithi154 podman[74902]: 2024-05-20 20:21:52.414008146 +0000 UTC m=+0.117754315 container create c4972c363e8b5458b1f1e13f45da771dd660ba19e08a095dbd6619887a791bfb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-20T20:21:52.741 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:52 smithi154 podman[74902]: 2024-05-20 20:21:52.641504093 +0000 UTC m=+0.345250220 container init c4972c363e8b5458b1f1e13f45da771dd660ba19e08a095dbd6619887a791bfb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5-deactivate, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-05-20T20:21:52.741 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:52 smithi154 podman[74902]: 2024-05-20 20:21:52.644595526 +0000 UTC m=+0.348341658 container start c4972c363e8b5458b1f1e13f45da771dd660ba19e08a095dbd6619887a791bfb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-05-20T20:21:52.741 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:52 smithi154 podman[74902]: 2024-05-20 20:21:52.741727661 +0000 UTC m=+0.445473803 container attach c4972c363e8b5458b1f1e13f45da771dd660ba19e08a095dbd6619887a791bfb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-05-20T20:21:52.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:52.494+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:52.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:52.494+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:52.804 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:52.494+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:52.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:52.688+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:52.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:52.688+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:52.805 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:52.688+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:53.007 INFO:journalctl@ceph.osd.5.smithi154.stdout:May 20 20:21:52 smithi154 podman[74902]: 2024-05-20 20:21:52.944852399 +0000 UTC m=+0.648598529 container died c4972c363e8b5458b1f1e13f45da771dd660ba19e08a095dbd6619887a791bfb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:21:53.262 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:53 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:53.007+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:53.262 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:53 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:53.007+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:53.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:53 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:53.031+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:53.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:53 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:53.031+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:53.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:52 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:52.956+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:53.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:52 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:52.956+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:53.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:52 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:52.956+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:53.562 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:53 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:53.279+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:53.562 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:53 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:53.279+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:53.562 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:53 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:53.279+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:53.581 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.5.service' 2024-05-20T20:21:53.621 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:21:53.621 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-20T20:21:53.621 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-20T20:21:53.622 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.6 2024-05-20T20:21:53.755 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:53.493+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:53.756 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:53.493+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:53.756 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:53.493+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:53.757 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:53.727+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:53.757 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:53.727+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:53.757 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:53 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:53.727+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:54.053 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:53 smithi028 systemd[1]: Stopping Ceph osd.6 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:21:54.313 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:54 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:54.037+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:54.313 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:54 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:54.037+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:54.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:54 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:54.030+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:54.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:54 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:54.030+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:54.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:53 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:53.980+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:54.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:53 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:53.980+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:54.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:53 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:53.980+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:54.531 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:54.237+0000 7f56c0600700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:21:54.532 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:54.237+0000 7f56c0600700 -1 osd.6 102 *** Got signal Terminated *** 2024-05-20T20:21:54.532 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:54.237+0000 7f56c0600700 -1 osd.6 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:21:54.532 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:54.530+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:54.532 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:54.530+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:54.532 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:54.530+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:54.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:54 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:54.313+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:54.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:54 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:54.313+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:54.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:54 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:54.313+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:54.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:54.740+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:54.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:54.740+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:54.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:54 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:54.740+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:55.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:55 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:55.078+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:55.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:55 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:55.078+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:55.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:55 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:55.300+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:55.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:55 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:55.300+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:55.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:55 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:55.300+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:55.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:55 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:55.300+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:21:55.376 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:55 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:55.019+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:55.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:55 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:55.019+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:55.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:55 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:55.019+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:21:55.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:54 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:54.970+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:55.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:54 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:54.970+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:55.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:54 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:54.970+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:55.807 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:55 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:55.722+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:55.807 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:55 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:55.722+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:55.807 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:55 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:55.722+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:55.808 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:55 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:55.529+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:55.808 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:55 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:55.529+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:55.808 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:55 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:55.529+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:56.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:56 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:56.042+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:56.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:56 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:56.042+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:56.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:56 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:56.281+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:56.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:56 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:56.281+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:56.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:56 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:56.281+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:56.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:56 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:56.281+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:21:56.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:56 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:56.066+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:56.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:56 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:56.066+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:56.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:56 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:56.066+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:21:56.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:55 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:55.992+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:56.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:55 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:55.992+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:56.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:55 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:55.992+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:56.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:56.716+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:56.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:56.716+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:56.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:56.716+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:56.805 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:56.519+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:56.805 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:56.519+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:56.805 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:56 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:56.519+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:57.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:57 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:57.078+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:57.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:57 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:57.078+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:57.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:57 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:57.078+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:21:57.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:57 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:57.316+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:57.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:57 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:57.316+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:57.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:57 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:57.316+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:57.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:57 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:57.316+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:21:57.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:56 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:56.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:57.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:56 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:56.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:57.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:56 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:56.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:57.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:57 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:57.114+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:57.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:57 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:57.114+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:57.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:57 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:57.114+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:21:57.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:57.734+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:57.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:57.734+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:57.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:57.734+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:57.804 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:57.734+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:21:57.805 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:57.549+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:57.805 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:57.549+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:57.805 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:57.549+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:57.805 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:57.549+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.713181+0000 front 2024-05-20T20:21:32.713020+0000 (oldest deadline 2024-05-20T20:21:56.714144+0000) 2024-05-20T20:21:58.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:58 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:58.088+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:58.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:58 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:58.088+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:58.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:58 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:58.088+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:21:58.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:58 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:58.268+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:58.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:58 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:58.268+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:58.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:58 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:58.268+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:58.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:58 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:58.268+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:21:58.376 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:58 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:58.083+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:58.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:58 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:58.083+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:58.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:58 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:58.083+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:21:58.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:57 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:57.984+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:58.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:57 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:57.984+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:58.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:57 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:57.984+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:58.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:57 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:57.984+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:21:59.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:58.770+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:21:59.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:58.770+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:21:59.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:58.770+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:21:59.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:58.770+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:21:59.055 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:58.578+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:05.304611+0000 front 2024-05-20T20:21:05.304416+0000 (oldest deadline 2024-05-20T20:21:30.604601+0000) 2024-05-20T20:21:59.055 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:58.578+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.106781+0000 front 2024-05-20T20:21:14.106717+0000 (oldest deadline 2024-05-20T20:21:37.606502+0000) 2024-05-20T20:21:59.055 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:58.578+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:19.908517+0000 front 2024-05-20T20:21:19.908655+0000 (oldest deadline 2024-05-20T20:21:45.208747+0000) 2024-05-20T20:21:59.055 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:58 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6[59559]: 2024-05-20T20:21:58.578+0000 7f56bbe00700 -1 osd.6 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.713181+0000 front 2024-05-20T20:21:32.713020+0000 (oldest deadline 2024-05-20T20:21:56.714144+0000) 2024-05-20T20:21:59.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:59 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:59.056+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:21:59.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:59 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:59.056+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:21:59.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:21:59 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:21:59.056+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:21:59.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:59 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:59.233+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:21:59.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:59 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:59.233+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:21:59.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:59 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:59.233+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:21:59.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:21:59 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:21:59.233+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:21:59.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:59 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:59.103+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:21:59.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:59 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:59.103+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:21:59.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:21:59 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:21:59.103+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:21:59.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:58 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:58.941+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:21:59.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:58 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:58.941+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:21:59.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:58 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:58.941+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:21:59.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:58 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:58.941+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:00.054 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:21:59 smithi028 podman[125384]: 2024-05-20 20:21:59.90734606 +0000 UTC m=+5.812147261 container died dd1372d90864c2acb6ae17b0fe5f8512e2ffeb50768d935990b43312e40d46b7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2) 2024-05-20T20:22:00.055 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:59.778+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:00.055 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:59.778+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:00.055 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:59.778+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:00.055 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:21:59 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:21:59.778+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:00.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:00 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:00.097+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:00.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:00 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:00.097+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:00.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:00 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:00.097+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:00.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:00 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:00.189+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:00.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:00 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:00.189+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:00.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:00 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:00.189+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:00.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:00 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:00.189+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:00.376 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:00 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:00.119+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:00.376 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:00 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:00.119+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:00.376 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:00 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:00.119+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:00.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:59 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:59.922+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:00.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:59 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:59.922+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:00.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:59 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:59.922+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:00.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:21:59 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:21:59.922+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:00.467 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:00 smithi028 podman[125384]: 2024-05-20 20:22:00.439235354 +0000 UTC m=+6.344036574 container cleanup dd1372d90864c2acb6ae17b0fe5f8512e2ffeb50768d935990b43312e40d46b7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-20T20:22:00.467 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:00 smithi028 bash[125384]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6 2024-05-20T20:22:01.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:00 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:00.787+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:01.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:00 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:00.787+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:01.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:00 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:00.787+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:01.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:00 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:00.787+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:01.054 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:00 smithi028 podman[125400]: 2024-05-20 20:22:00.748908559 +0000 UTC m=+0.821750744 container remove dd1372d90864c2acb6ae17b0fe5f8512e2ffeb50768d935990b43312e40d46b7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514) 2024-05-20T20:22:01.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:01 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:01.137+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:01.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:01 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:01.137+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:01.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:01 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:01.137+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:01.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:00 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:00.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:01.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:00 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:00.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:01.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:00 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:00.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:01.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:00 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:00.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:01.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:01 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:01.142+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:01.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:01 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:01.142+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:01.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:01 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:01.142+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:01.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:01 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:01.177+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:01.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:01 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:01.177+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:01.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:01 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:01.177+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:01.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:01 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:01.177+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:01.929 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:01.799+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:01.929 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:01.799+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:01.929 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:01.799+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:01.929 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:01.799+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:01.929 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:01 smithi028 podman[125487]: 2024-05-20 20:22:01.561771851 +0000 UTC m=+0.036620900 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:22:01.929 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:01 smithi028 podman[125487]: 2024-05-20 20:22:01.657841755 +0000 UTC m=+0.132690814 container create a40e1e26645d8bf1a96b61815a04d379220c58f0e31c9774399e4996a76d40a5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-05-20T20:22:01.929 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:01 smithi028 podman[125487]: 2024-05-20 20:22:01.928415251 +0000 UTC m=+0.403264350 container init a40e1e26645d8bf1a96b61815a04d379220c58f0e31c9774399e4996a76d40a5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-05-20T20:22:02.304 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:01 smithi028 podman[125487]: 2024-05-20 20:22:01.940992527 +0000 UTC m=+0.415841640 container start a40e1e26645d8bf1a96b61815a04d379220c58f0e31c9774399e4996a76d40a5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:22:02.304 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:02 smithi028 podman[125487]: 2024-05-20 20:22:02.035207022 +0000 UTC m=+0.510056128 container attach a40e1e26645d8bf1a96b61815a04d379220c58f0e31c9774399e4996a76d40a5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:22:02.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:02 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:02.104+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:02.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:02 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:02.104+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:02.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:02 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:02.104+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:02.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:01 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:01.911+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:02.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:01 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:01.911+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:02.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:01 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:01.911+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:02.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:01 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:01.911+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:02.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:02 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:02.131+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:02.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:02 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:02.131+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:02.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:02 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:02.131+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:02.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:02 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:02.211+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:02.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:02 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:02.211+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:02.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:02 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:02.211+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:02.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:02 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:02.211+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:02.742 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:02 smithi028 podman[125487]: 2024-05-20 20:22:02.407033634 +0000 UTC m=+0.881882752 container died a40e1e26645d8bf1a96b61815a04d379220c58f0e31c9774399e4996a76d40a5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-05-20T20:22:03.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:02 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:02.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:03.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:02 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:02.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:03.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:02 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:02.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:03.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:02 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:02.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:03.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:03.105+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:03.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:03.105+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:03.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:03.105+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:03.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:03.105+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.575389+0000 front 2024-05-20T20:21:39.575462+0000 (oldest deadline 2024-05-20T20:22:03.075552+0000) 2024-05-20T20:22:03.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:02 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:02.866+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:03.128 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:02 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:02.866+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:03.128 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:02 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:02.866+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:03.128 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:02 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:02.866+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:03.230 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.6.service' 2024-05-20T20:22:03.307 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:03 smithi028 podman[125512]: 2024-05-20 20:22:03.189027708 +0000 UTC m=+0.765416941 container remove a40e1e26645d8bf1a96b61815a04d379220c58f0e31c9774399e4996a76d40a5 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-6-deactivate, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514) 2024-05-20T20:22:03.308 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:03 smithi028 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.6.service: Deactivated successfully. 2024-05-20T20:22:03.308 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:03 smithi028 systemd[1]: Stopped Ceph osd.6 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:22:03.308 INFO:journalctl@ceph.osd.6.smithi028.stdout:May 20 20:22:03 smithi028 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.6.service: Consumed 6min 47.104s CPU time. 2024-05-20T20:22:03.340 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:03.340 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-20T20:22:03.340 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-20T20:22:03.341 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.7 2024-05-20T20:22:03.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:03 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:03.108+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:03.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:03 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:03.108+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:03.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:03 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:03.108+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:03.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:03 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:03.204+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:03.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:03 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:03.204+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:03.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:03 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:03.204+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:03.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:03 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:03.204+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:03.627 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 systemd[1]: Stopping Ceph osd.7 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:04.078 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:03.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:04.078 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:03.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:04.079 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:03.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:04.079 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:03 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:03.824+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:04.080 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:03.633+0000 7ff8b1ccf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:03.633+0000 7ff8b1ccf700 -1 osd.7 102 *** Got signal Terminated *** 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:03.633+0000 7ff8b1ccf700 -1 osd.7 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:04.080+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:04.080+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:04.080+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:04.080+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.575389+0000 front 2024-05-20T20:21:39.575462+0000 (oldest deadline 2024-05-20T20:22:03.075552+0000) 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:03.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:03.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:03.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:04.081 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:03 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:03.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:04.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:04.162+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:04.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:04.162+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:04.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:04.162+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:04.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:04.162+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:04.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:04.087+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:04.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:04.087+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:04.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:04.087+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:04.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:04 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:04.087+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:05.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:04 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:04.836+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:05.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:04 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:04.836+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:05.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:04 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:04.836+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:05.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:04 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:04.836+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:05.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:04 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:04.836+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:05.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:05.083+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:05.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:05.083+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:05.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:05.083+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:05.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:05.083+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:05.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:05.126+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:05.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:05.126+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:05.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:05.126+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:05.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:05.126+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:05.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:05 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:05.126+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:05.068+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:05.068+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:05.068+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:05.068+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.575389+0000 front 2024-05-20T20:21:39.575462+0000 (oldest deadline 2024-05-20T20:22:03.075552+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:04.887+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:04.887+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:04.887+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:05.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:04 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:04.887+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:06.278 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:06.023+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:06.279 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:06.023+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:06.279 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:06.023+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:06.279 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:06.023+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.575389+0000 front 2024-05-20T20:21:39.575462+0000 (oldest deadline 2024-05-20T20:22:03.075552+0000) 2024-05-20T20:22:06.280 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:05.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:06.280 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:05.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:06.280 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:05.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:06.280 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:05 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:05.882+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:06.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:05 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:05.820+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:06.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:05 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:05.820+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:06.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:05 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:05.820+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:06.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:05 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:05.820+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:06.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:05 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:05.820+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:06.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:06.052+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:06.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:06.052+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:06.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:06.052+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:06.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:06.052+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:06.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:06.134+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:06.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:06.134+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:06.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:06.134+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:06.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:06.134+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:06.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:06 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:06.134+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:07.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:06.798+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:07.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:06.798+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:07.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:06.798+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:07.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:06.798+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:07.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:06 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:06.798+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:07.061+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:07.061+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:07.061+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:07.061+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.575389+0000 front 2024-05-20T20:21:39.575462+0000 (oldest deadline 2024-05-20T20:22:03.075552+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:06.861+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:06.861+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:06.861+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:06.861+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:07.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:06.861+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:07.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.004+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:07.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.004+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:07.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.004+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:07.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.004+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:07.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:07.116+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:07.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:07.116+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:07.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:07.116+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:07.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:07.116+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:07.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:07.116+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:08.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:07 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:07.767+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:08.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:07 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:07.767+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:08.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:07 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:07.767+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:08.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:07 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:07.767+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:08.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:07 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:07.767+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:08.081+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.469001+0000 front 2024-05-20T20:21:15.469048+0000 (oldest deadline 2024-05-20T20:21:37.769199+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:08.081+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.871375+0000 front 2024-05-20T20:21:22.871447+0000 (oldest deadline 2024-05-20T20:21:45.072080+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:08.081+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.074162+0000 front 2024-05-20T20:21:29.074041+0000 (oldest deadline 2024-05-20T20:21:54.973916+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7[54365]: 2024-05-20T20:22:08.081+0000 7ff8aa2bb700 -1 osd.7 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.575389+0000 front 2024-05-20T20:21:39.575462+0000 (oldest deadline 2024-05-20T20:22:03.075552+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:07.842+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:07.842+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:07.842+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:07.842+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:08.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:07 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:07.842+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:08.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.983+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:08.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.983+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:08.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.983+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:08.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:07 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:07.983+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:08.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:08 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:08.154+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:08.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:08 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:08.154+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:08.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:08 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:08.154+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:08.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:08 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:08.154+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:08.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:08 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:08.154+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:09.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:08 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:08.752+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:09.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:08 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:08.752+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:09.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:08 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:08.752+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:09.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:08 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:08.752+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:09.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:08 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:08.752+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:09.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:08.864+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:09.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:08.864+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:09.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:08.864+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:09.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:08.864+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:09.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:08 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:08.864+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:09.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:09.029+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:09.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:09.029+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:09.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:09.029+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:09.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:09.029+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:09.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:09.165+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:09.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:09.165+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:09.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:09.165+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:09.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:09.165+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:09.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:09 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:09.165+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:09.451 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:09 smithi033 podman[73883]: 2024-05-20 20:22:09.159040455 +0000 UTC m=+5.618284671 container died 8e64544b0484dfaa035eb327f4c20c3ffd99125ca2de5afc6628f73a95a99771 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:22:09.819 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:09 smithi033 podman[73883]: 2024-05-20 20:22:09.5426543 +0000 UTC m=+6.001898514 container cleanup 8e64544b0484dfaa035eb327f4c20c3ffd99125ca2de5afc6628f73a95a99771 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:09.819 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:09 smithi033 bash[73883]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7 2024-05-20T20:22:10.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:09.744+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:10.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:09.744+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:10.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:09.744+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:10.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:09.744+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:10.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:09 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:09.744+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:10.126 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:09 smithi033 podman[73911]: 2024-05-20 20:22:09.81909372 +0000 UTC m=+0.651407035 container remove 8e64544b0484dfaa035eb327f4c20c3ffd99125ca2de5afc6628f73a95a99771 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:10.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:09 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:09.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:10.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:09 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:09.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:10.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:09 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:09.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:10.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:09 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:09.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:10.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:09 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:09.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:10.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:10.014+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:10.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:10.014+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:10.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:10.014+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:10.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:10.014+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:10.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:10.183+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:10.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:10.183+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:10.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:10.183+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:10.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:10.183+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:10.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:10 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:10.183+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:10.455 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:10 smithi033 podman[73999]: 2024-05-20 20:22:10.080748241 +0000 UTC m=+0.017664486 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:22:10.455 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:10 smithi033 podman[73999]: 2024-05-20 20:22:10.203520674 +0000 UTC m=+0.140436916 container create f481ce6b4954e37e109cc132c8e017d44a045c4c16dded9b0e3089b6aff2a5e3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:22:10.455 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:10 smithi033 podman[73999]: 2024-05-20 20:22:10.45576229 +0000 UTC m=+0.392678511 container init f481ce6b4954e37e109cc132c8e017d44a045c4c16dded9b0e3089b6aff2a5e3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7-deactivate, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T20:22:10.762 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:10 smithi033 podman[73999]: 2024-05-20 20:22:10.459085087 +0000 UTC m=+0.396001314 container start f481ce6b4954e37e109cc132c8e017d44a045c4c16dded9b0e3089b6aff2a5e3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T20:22:10.871 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:10 smithi033 podman[73999]: 2024-05-20 20:22:10.547505484 +0000 UTC m=+0.484421715 container attach f481ce6b4954e37e109cc132c8e017d44a045c4c16dded9b0e3089b6aff2a5e3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7-deactivate, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_CLEAN=True) 2024-05-20T20:22:11.042 INFO:journalctl@ceph.osd.7.smithi033.stdout:May 20 20:22:10 smithi033 podman[73999]: 2024-05-20 20:22:10.7628513 +0000 UTC m=+0.699767521 container died f481ce6b4954e37e109cc132c8e017d44a045c4c16dded9b0e3089b6aff2a5e3 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:22:11.042 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:10 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:10.822+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:11.042 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:10 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:10.822+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:11.042 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:10 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:10.822+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:11.042 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:10 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:10.822+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:11.042 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:10 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:10.822+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:11.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:10.788+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:11.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:10.788+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:11.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:10.788+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:11.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:10.788+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:11.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:10 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:10.788+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:11.358 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.7.service' 2024-05-20T20:22:11.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:11.046+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:11.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:11.046+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:11.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:11.046+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:11.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:11.046+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:11.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:11.207+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:11.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:11.207+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:11.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:11.207+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:11.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:11.207+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:11.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:11.207+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:11.398 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:11.399 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-20T20:22:11.399 INFO:tasks.cephadm.osd.8:Stopping osd.8... 2024-05-20T20:22:11.399 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.8 2024-05-20T20:22:11.723 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 systemd[1]: Stopping Ceph osd.8 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:11.723 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:11.722+0000 7ffab0d3e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:12.062 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:11.722+0000 7ffab0d3e700 -1 osd.8 102 *** Got signal Terminated *** 2024-05-20T20:22:12.062 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:11 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:11.722+0000 7ffab0d3e700 -1 osd.8 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:22:12.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:11.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:12.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:11.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:12.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:11.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:12.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:11.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:12.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:11.839+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:12.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:11.837+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:12.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:11.837+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:12.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:11.837+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:12.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:11.837+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:12.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:11.837+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:12.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:11 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:11.837+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:12.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:12.062+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:12.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:12.062+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:12.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:12.062+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:12.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:12.062+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:12.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:12.196+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:12.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:12.196+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:12.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:12.196+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:12.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:12.196+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:12.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:12.196+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:12.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:12 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:12.196+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:13.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:12 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:12.818+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:13.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:12 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:12.818+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:13.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:12 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:12.818+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:13.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:12 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:12.818+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:13.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:12 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:12.818+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:13.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:12 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:12.818+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:13.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:12.851+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:13.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:12.851+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:13.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:12.851+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:13.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:12.851+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:13.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:12.851+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:13.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:12 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:12.851+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:13.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.013+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:13.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.013+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:13.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.013+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:13.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.013+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:13.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.013+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:50.294739+0000 front 2024-05-20T20:21:50.294805+0000 (oldest deadline 2024-05-20T20:22:12.495222+0000) 2024-05-20T20:22:13.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:13.152+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:13.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:13.152+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:13.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:13.152+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:13.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:13.152+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:13.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:13.152+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:13.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:13.152+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:14.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:13.837+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:14.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:13.837+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:14.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:13.837+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:14.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:13.837+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:14.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:13.837+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:14.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:13 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:13.837+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:14.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:13.867+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:14.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:13.867+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:14.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:13.867+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:14.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:13.867+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:14.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:13.867+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:14.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:13 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:13.867+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:14.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.971+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:14.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.971+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:14.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.971+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:14.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.971+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:14.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:13 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:13.971+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:50.294739+0000 front 2024-05-20T20:21:50.294805+0000 (oldest deadline 2024-05-20T20:22:12.495222+0000) 2024-05-20T20:22:14.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:14.122+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:14.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:14.122+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:14.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:14.122+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:14.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:14.122+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:14.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:14.122+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:14.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:14.123+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:15.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:14 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:14.876+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:15.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:14 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:14.876+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:15.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:14 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:14.876+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:15.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:14 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:14.876+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:15.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:14 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:14.876+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:15.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:14 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:14.876+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:15.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:14 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:14.881+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:15.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:14 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:14.881+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:15.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:14 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:14.881+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:15.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:14 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:14.881+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:15.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:14 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:14.881+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:15.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:14 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:14.881+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:15.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:15.148+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:15.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:15.148+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:15.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:15.148+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:15.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:15.148+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:15.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:15.148+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:15.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:15.148+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:15.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:14.989+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:15.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:14.989+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:15.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:14.989+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:15.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:14.989+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:15.376 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:14 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:14.989+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:50.294739+0000 front 2024-05-20T20:21:50.294805+0000 (oldest deadline 2024-05-20T20:22:12.495222+0000) 2024-05-20T20:22:16.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:15.854+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:16.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:15.854+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:16.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:15.854+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:16.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:15.854+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:16.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:15.854+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:16.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:15.854+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:16.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:15 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:15.869+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:16.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:15 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:15.869+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:16.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:15 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:15.869+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:16.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:15 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:15.869+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:16.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:15 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:15.869+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:16.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:15 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:15.869+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:16.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:15.940+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.488355+0000 front 2024-05-20T20:21:15.488376+0000 (oldest deadline 2024-05-20T20:21:40.788530+0000) 2024-05-20T20:22:16.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:15.940+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:20.789074+0000 front 2024-05-20T20:21:20.789046+0000 (oldest deadline 2024-05-20T20:21:45.489188+0000) 2024-05-20T20:22:16.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:15.940+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.891957+0000 front 2024-05-20T20:21:32.891920+0000 (oldest deadline 2024-05-20T20:21:56.991844+0000) 2024-05-20T20:22:16.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:15.940+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:38.093121+0000 front 2024-05-20T20:21:38.093081+0000 (oldest deadline 2024-05-20T20:22:03.992995+0000) 2024-05-20T20:22:16.375 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:15 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8[55198]: 2024-05-20T20:22:15.940+0000 7ffaad332700 -1 osd.8 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:50.294739+0000 front 2024-05-20T20:21:50.294805+0000 (oldest deadline 2024-05-20T20:22:12.495222+0000) 2024-05-20T20:22:16.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:16 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:16.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:16.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:16 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:16.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:16.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:16 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:16.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:16.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:16 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:16.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:16.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:16 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:16.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:16.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:16 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:16.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:17.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:16 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:16.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:17.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:16 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:16.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:17.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:16 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:16.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:17.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:16 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:16.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:17.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:16 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:16.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:17.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:16 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:16.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:17.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:16 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:16.919+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:17.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:16 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:16.919+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:17.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:16 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:16.919+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:17.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:16 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:16.919+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:17.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:16 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:16.919+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:17.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:16 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:16.919+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:17.358 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:17 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:17.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:17.358 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:17 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:17.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:17.358 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:17 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:17.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:17.358 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:17 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:17.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:17.358 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:17 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:17.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:17.358 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:17 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:17.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:17.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:17 smithi154 podman[75021]: 2024-05-20 20:22:17.358400413 +0000 UTC m=+5.758244281 container died 030a52b1dd97fa7e22040a0761fd12aa7a8d7207789a00a5128c342519731fe7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:17.989 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:17 smithi154 podman[75021]: 2024-05-20 20:22:17.730974767 +0000 UTC m=+6.130818625 container cleanup 030a52b1dd97fa7e22040a0761fd12aa7a8d7207789a00a5128c342519731fe7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=) 2024-05-20T20:22:17.990 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:17 smithi154 bash[75021]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8 2024-05-20T20:22:18.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:17.944+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:18.306 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:17.944+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:18.306 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:17.944+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:18.306 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:17.944+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:18.306 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:17.944+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:18.306 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:17 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:17.944+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:18.347 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:17 smithi154 podman[75039]: 2024-05-20 20:22:17.990581234 +0000 UTC m=+0.627522559 container remove 030a52b1dd97fa7e22040a0761fd12aa7a8d7207789a00a5128c342519731fe7 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2) 2024-05-20T20:22:18.347 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:18 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:18.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:18.348 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:18 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:18.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:18.348 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:18 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:18.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:18.348 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:18 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:18.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:18.348 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:18 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:18.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:18.348 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:18 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:18.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:18.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:17 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:17.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:18.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:17 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:17.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:18.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:17 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:17.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:18.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:17 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:17.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:18.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:17 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:17.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:18.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:17 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:17.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:18.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:18 smithi154 podman[75128]: 2024-05-20 20:22:18.250506488 +0000 UTC m=+0.017272226 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:22:18.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:18 smithi154 podman[75128]: 2024-05-20 20:22:18.358035836 +0000 UTC m=+0.124801595 container create 0a3e246f179fabfff26e306f64ff0b8df509e9266aa8fe8800079aced9137bbb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:18.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:18 smithi154 podman[75128]: 2024-05-20 20:22:18.585642256 +0000 UTC m=+0.352408004 container init 0a3e246f179fabfff26e306f64ff0b8df509e9266aa8fe8800079aced9137bbb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-05-20T20:22:18.625 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:18 smithi154 podman[75128]: 2024-05-20 20:22:18.588915531 +0000 UTC m=+0.355681273 container start 0a3e246f179fabfff26e306f64ff0b8df509e9266aa8fe8800079aced9137bbb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8-deactivate, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:22:18.881 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:18 smithi154 podman[75128]: 2024-05-20 20:22:18.693829322 +0000 UTC m=+0.460595075 container attach 0a3e246f179fabfff26e306f64ff0b8df509e9266aa8fe8800079aced9137bbb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-20T20:22:19.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:18 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:18.849+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:19.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:18 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:18.849+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:19.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:18 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:18.849+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:19.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:18 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:18.849+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:19.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:18 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:18.849+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:19.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:18 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:18.849+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:19.188 INFO:journalctl@ceph.osd.8.smithi154.stdout:May 20 20:22:18 smithi154 podman[75128]: 2024-05-20 20:22:18.882374722 +0000 UTC m=+0.649140463 container died 0a3e246f179fabfff26e306f64ff0b8df509e9266aa8fe8800079aced9137bbb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-8-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514) 2024-05-20T20:22:19.188 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:19.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:19.188 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:19.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:19.188 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:19.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:19.188 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:19.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:19.188 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:19.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:19.188 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:19 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:19.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:19.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:18.955+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:19.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:18.955+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:19.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:18.955+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:19.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:18.955+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:19.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:18.955+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:19.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:18 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:18.955+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:19.524 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.8.service' 2024-05-20T20:22:19.561 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:19.562 INFO:tasks.cephadm.osd.8:Stopped osd.8 2024-05-20T20:22:19.562 INFO:tasks.cephadm.osd.9:Stopping osd.9... 2024-05-20T20:22:19.562 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.9 2024-05-20T20:22:19.921 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:19 smithi028 systemd[1]: Stopping Ceph osd.9 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:19.921 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:19 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:19.920+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:20.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:19 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:19.920+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:20.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:19 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:19.920+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:20.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:19 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:19.920+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:20.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:19 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:19.920+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:20.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:19 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:19.920+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:20.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.166+0000 7fbceb400700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:20.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.166+0000 7fbceb400700 -1 osd.9 102 *** Got signal Terminated *** 2024-05-20T20:22:20.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.166+0000 7fbceb400700 -1 osd.9 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:22:20.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:20 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:20.036+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:20.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:20 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:20.036+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:20.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:20 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:20.036+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:20.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:20 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:20.036+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:20.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:20 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:20.036+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:20.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:20 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:20.036+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:20.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:19.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:20.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:19.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:20.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:19.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:20.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:19.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:20.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:19.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:20.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:19.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:21.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:21.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:21.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:21.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:21.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:21.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:20 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:20.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:21.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:21 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:21.068+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:21.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:21 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:21.068+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:21.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:21 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:21.068+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:21.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:21 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:21.068+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:21.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:21 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:21.068+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:21.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:21 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:21.068+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:21.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:20 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:20.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:21.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:20 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:20.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:21.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:20 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:20.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:21.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:20 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:20.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:21.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:20 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:20.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:21.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:20 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:20.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:21.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:20 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:20.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:22.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:21.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:22.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:21.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:22.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:21.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:22.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:21.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:22.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:21.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:22.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:21 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:21.884+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:22.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:22 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:22.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:22.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:22 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:22.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:22.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:22 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:22.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:22.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:22 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:22.093+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:22.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:22 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:22.094+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:22.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:22 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:22.094+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:22.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:21.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:22.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:21.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:22.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:21.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:22.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:21.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:22.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:21.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:22.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:21.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:22.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:21 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:21.889+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:23.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:22 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:22.876+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:23.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:22 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:22.876+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:23.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:22 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:22.876+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:23.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:22 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:22.876+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:23.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:22 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:22.876+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:23.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:22 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:22.877+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:23.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:22 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:22.877+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:23.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:22 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:22.877+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:23.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:22 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:22.877+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:23.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:22 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:22.877+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:23.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:22 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:22.877+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:23.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:22 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:22.877+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:23.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:22 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:22.877+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:23.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:23.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:23.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:23.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:23.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:23.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:23.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:23.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:23.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:23.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:23.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:23.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:24.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:23 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:23.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:24.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:23 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:23.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:24.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:23 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:23.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:24.127 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:23 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:23.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:24.128 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:23 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:23.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:24.128 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:23 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:23.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:24.128 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:23 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:23.857+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:24.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:23.922+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:24.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:23.922+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:24.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:23.922+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:24.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:23.922+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:24.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:23.922+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:24.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:23 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:23.922+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:24.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:24.138+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:24.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:24.138+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:24.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:24.138+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:24.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:24.138+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:24.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:24.138+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:24.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:24 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:24.138+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:25.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:24.914+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.717530+0000 front 2024-05-20T20:21:04.716380+0000 (oldest deadline 2024-05-20T20:21:29.416582+0000) 2024-05-20T20:22:25.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:24.914+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.019243+0000 front 2024-05-20T20:21:14.018919+0000 (oldest deadline 2024-05-20T20:21:36.918932+0000) 2024-05-20T20:22:25.304 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:24.914+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:22.722079+0000 front 2024-05-20T20:21:22.722013+0000 (oldest deadline 2024-05-20T20:21:46.822273+0000) 2024-05-20T20:22:25.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:24.914+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:31.425230+0000 front 2024-05-20T20:21:31.424956+0000 (oldest deadline 2024-05-20T20:21:57.324832+0000) 2024-05-20T20:22:25.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:24.914+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:39.430612+0000 front 2024-05-20T20:21:39.430831+0000 (oldest deadline 2024-05-20T20:22:04.730864+0000) 2024-05-20T20:22:25.305 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:24 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9[66574]: 2024-05-20T20:22:24.914+0000 7fbce1c00700 -1 osd.9 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:45.233267+0000 front 2024-05-20T20:21:45.233358+0000 (oldest deadline 2024-05-20T20:22:11.634288+0000) 2024-05-20T20:22:25.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:24 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:24.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:25.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:24 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:24.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:25.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:24 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:24.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:25.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:24 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:24.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:25.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:24 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:24.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:25.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:24 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:24.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:25.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:24 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:24.898+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:25.526 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:25.145+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:25.526 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:25.145+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:25.526 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:25.145+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:25.527 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:25.145+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:25.527 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:25.145+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:25.527 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:25 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:25.145+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:26.298 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:25 smithi028 podman[125603]: 2024-05-20 20:22:25.973860816 +0000 UTC m=+5.949723974 container died aa0630cefbc90a2cdd3c6eca92e0c386f2092147cc5340062ae681ff2f9ab4ec (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240514) 2024-05-20T20:22:26.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:25 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:25.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:26.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:25 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:25.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:26.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:25 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:25.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:26.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:25 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:25.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:26.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:25 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:25.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:26.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:25 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:25.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:26.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:25 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:25.934+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:26.553 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:26 smithi028 podman[125603]: 2024-05-20 20:22:26.466220908 +0000 UTC m=+6.442084063 container cleanup aa0630cefbc90a2cdd3c6eca92e0c386f2092147cc5340062ae681ff2f9ab4ec (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:22:26.554 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:26 smithi028 bash[125603]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9 2024-05-20T20:22:26.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:26.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:26.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:26.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:26.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:26.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:26.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:26.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:26.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:26.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:26.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:26.131+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:27.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:26 smithi028 podman[125622]: 2024-05-20 20:22:26.642770209 +0000 UTC m=+0.648747168 container remove aa0630cefbc90a2cdd3c6eca92e0c386f2092147cc5340062ae681ff2f9ab4ec (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514) 2024-05-20T20:22:27.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:26.951+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:27.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:26.951+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:27.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:26.951+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:27.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:26.951+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:27.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:26.951+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:27.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:26.951+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:27.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:26 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:26.951+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:27.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:27.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:27.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:27.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:27.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:27.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:27.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:27.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:27.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:27.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:27.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:27 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:27.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:27.780 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:27 smithi028 podman[125709]: 2024-05-20 20:22:27.393354885 +0000 UTC m=+0.034368548 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:22:27.780 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:27 smithi028 podman[125709]: 2024-05-20 20:22:27.510009019 +0000 UTC m=+0.151022707 container create 3f1e50733a94b82b4ac89e892b08f21797d149050b77d9e411fa3c6a5a6743cf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-05-20T20:22:27.780 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:27 smithi028 podman[125709]: 2024-05-20 20:22:27.779994285 +0000 UTC m=+0.421008023 container init 3f1e50733a94b82b4ac89e892b08f21797d149050b77d9e411fa3c6a5a6743cf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:22:28.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:27 smithi028 podman[125709]: 2024-05-20 20:22:27.793590167 +0000 UTC m=+0.434603838 container start 3f1e50733a94b82b4ac89e892b08f21797d149050b77d9e411fa3c6a5a6743cf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9-deactivate, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-20T20:22:28.054 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:27 smithi028 podman[125709]: 2024-05-20 20:22:27.86199941 +0000 UTC m=+0.503013168 container attach 3f1e50733a94b82b4ac89e892b08f21797d149050b77d9e411fa3c6a5a6743cf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-05-20T20:22:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:27.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:27.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:27.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:27.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:27.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:27.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:28.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:27 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:27.929+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:28.544 INFO:journalctl@ceph.osd.9.smithi028.stdout:May 20 20:22:28 smithi028 podman[125709]: 2024-05-20 20:22:28.257489921 +0000 UTC m=+0.898503649 container died 3f1e50733a94b82b4ac89e892b08f21797d149050b77d9e411fa3c6a5a6743cf (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-9-deactivate, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:22:28.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:28.144+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:28.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:28.144+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:28.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:28.144+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:28.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:28.144+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:28.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:28.144+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:28.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:28 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:28.144+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:28.962 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.9.service' 2024-05-20T20:22:29.082 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:29.082 INFO:tasks.cephadm.osd.9:Stopped osd.9 2024-05-20T20:22:29.082 INFO:tasks.cephadm.osd.10:Stopping osd.10... 2024-05-20T20:22:29.083 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.10 2024-05-20T20:22:29.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:28.932+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:29.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:28.932+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:29.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:28.932+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:29.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:28.932+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:29.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:28.932+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:29.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:28.932+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:29.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:28 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:28.932+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:29.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 systemd[1]: Stopping Ceph osd.10 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:29.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:29.142+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:29.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:29.142+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:29.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:29.142+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:29.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:29.142+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:29.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:29.142+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:29.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:29 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:29.142+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:29.877 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.419+0000 7f2b33e66700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:29.877 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.419+0000 7f2b33e66700 -1 osd.10 102 *** Got signal Terminated *** 2024-05-20T20:22:29.877 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.419+0000 7f2b33e66700 -1 osd.10 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:22:30.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:30.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:30.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:30.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:30.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:30.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:30.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:29 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:29.937+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:30.128+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:30.128+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:30.128+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:30.128+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:30.128+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:30.128+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:30.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:30.128+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:31.378 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:30.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:31.379 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:30.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:31.379 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:30.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:31.381 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:30.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:31.382 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:30.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:31.382 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:30.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:31.383 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:30 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:30.959+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:31.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:31.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:31.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:31.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:31.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:31.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:31.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:31.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:31.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:31.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:31.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:31.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:31.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:31 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:31.137+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:32.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:32.113+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:32.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:32.113+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:32.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:32.113+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:32.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:32.113+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:32.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:32.113+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:32.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:32.113+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:32.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:32 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:32.113+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:32.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:31.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:32.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:31.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:32.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:31.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:32.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:31.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:32.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:31.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:32.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:31.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:32.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:31 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:31.936+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:33.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:32.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:33.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:32.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:33.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:32.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:33.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:32.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:33.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:32.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:33.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:32.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:33.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:32 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:32.977+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:33.625 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:33.157+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:33.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:33.157+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:33.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:33.157+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:33.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:33.157+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:33.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:33.157+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:33.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:33.157+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:33.626 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:33 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:33.157+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:34.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:34.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:34.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:34.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:34.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:34.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:34.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:34.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:34.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:34.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:34.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:34.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:34.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:34 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:34.124+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:34.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:34.008+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:02.089111+0000 front 2024-05-20T20:21:02.088913+0000 (oldest deadline 2024-05-20T20:21:27.889447+0000) 2024-05-20T20:22:34.376 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:34.008+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:14.890966+0000 front 2024-05-20T20:21:14.890991+0000 (oldest deadline 2024-05-20T20:21:37.191147+0000) 2024-05-20T20:22:34.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:34.008+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.994374+0000 front 2024-05-20T20:21:23.994399+0000 (oldest deadline 2024-05-20T20:21:48.694511+0000) 2024-05-20T20:22:34.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:34.008+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:32.796081+0000 front 2024-05-20T20:21:32.796270+0000 (oldest deadline 2024-05-20T20:21:57.396591+0000) 2024-05-20T20:22:34.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:34.008+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:40.798004+0000 front 2024-05-20T20:21:40.798036+0000 (oldest deadline 2024-05-20T20:22:06.698027+0000) 2024-05-20T20:22:34.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:34.008+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:46.698684+0000 front 2024-05-20T20:21:46.698710+0000 (oldest deadline 2024-05-20T20:22:11.998558+0000) 2024-05-20T20:22:34.377 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:34 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10[60982]: 2024-05-20T20:22:34.008+0000 7f2b3045a700 -1 osd.10 102 heartbeat_check: no reply from 172.21.15.28:6822 osd.6 since back 2024-05-20T20:21:55.400310+0000 front 2024-05-20T20:21:55.400364+0000 (oldest deadline 2024-05-20T20:22:20.700000+0000) 2024-05-20T20:22:35.269 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:35 smithi033 podman[74106]: 2024-05-20 20:22:35.012832211 +0000 UTC m=+5.721037367 container died 60e1404b42b2019caa493316400bf8ab05ee325680d28ed4679550a6e4aea7ef (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-05-20T20:22:35.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:35.078+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:35.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:35.078+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:35.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:35.078+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:35.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:35.078+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:35.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:35.078+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:35.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:35.078+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:35.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:35.078+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:35.627 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:35 smithi033 podman[74106]: 2024-05-20 20:22:35.369212087 +0000 UTC m=+6.077417226 container cleanup 60e1404b42b2019caa493316400bf8ab05ee325680d28ed4679550a6e4aea7ef (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-20T20:22:35.627 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:35 smithi033 bash[74106]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10 2024-05-20T20:22:36.005 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:35 smithi033 podman[74122]: 2024-05-20 20:22:35.645126994 +0000 UTC m=+0.627511073 container remove 60e1404b42b2019caa493316400bf8ab05ee325680d28ed4679550a6e4aea7ef (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-lusov-quiesce-4ebd1d5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:36.323 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:36 smithi033 podman[74214]: 2024-05-20 20:22:35.907999408 +0000 UTC m=+0.017973482 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:22:36.323 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:36 smithi033 podman[74214]: 2024-05-20 20:22:36.020973759 +0000 UTC m=+0.130947840 container create 8a2a6da972fa3575c038c32840fa702b1ec60c4ec7345339da8d8be6025b7013 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10-deactivate, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-20T20:22:36.324 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:36 smithi033 podman[74214]: 2024-05-20 20:22:36.240681221 +0000 UTC m=+0.350655283 container init 8a2a6da972fa3575c038c32840fa702b1ec60c4ec7345339da8d8be6025b7013 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-20T20:22:36.324 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:36 smithi033 podman[74214]: 2024-05-20 20:22:36.24372775 +0000 UTC m=+0.353701812 container start 8a2a6da972fa3575c038c32840fa702b1ec60c4ec7345339da8d8be6025b7013 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:22:36.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:36.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:36.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:36.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:36.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:36.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:36.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:36.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:36.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:36.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:36.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:36.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:36.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:36 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:36.103+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:36.627 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:36 smithi033 podman[74214]: 2024-05-20 20:22:36.323846429 +0000 UTC m=+0.433820503 container attach 8a2a6da972fa3575c038c32840fa702b1ec60c4ec7345339da8d8be6025b7013 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5) 2024-05-20T20:22:36.627 INFO:journalctl@ceph.osd.10.smithi033.stdout:May 20 20:22:36 smithi033 podman[74214]: 2024-05-20 20:22:36.546657881 +0000 UTC m=+0.656631950 container died 8a2a6da972fa3575c038c32840fa702b1ec60c4ec7345339da8d8be6025b7013 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-10-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:37.127 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.10.service' 2024-05-20T20:22:37.168 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:37.168 INFO:tasks.cephadm.osd.10:Stopped osd.10 2024-05-20T20:22:37.168 INFO:tasks.cephadm.osd.11:Stopping osd.11... 2024-05-20T20:22:37.168 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.11 2024-05-20T20:22:37.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:37.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:37.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:37.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:37.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:37.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:37.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:37.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 systemd[1]: Stopping Ceph osd.11 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:37.875 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.491+0000 7f9b4d06a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:37.875 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.491+0000 7f9b4d06a700 -1 osd.11 102 *** Got signal Terminated *** 2024-05-20T20:22:37.875 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:37 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:37.491+0000 7f9b4d06a700 -1 osd.11 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T20:22:38.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:38.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:38.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:38.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:38.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:38.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:38.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:38.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:38.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:38.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:38.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:38.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:38.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:38 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:38.069+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:39.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:39.029+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:39.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:39.029+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:39.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:39.029+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:39.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:39.029+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:39.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:39.029+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:39.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:39.029+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:39.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:39 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:39.029+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:40.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:40.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:40.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:40.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:40.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:40.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:40.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:40.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:40.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:40.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:40.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:40.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:40.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:40 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:40.055+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:41.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:41.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:41.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:41.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:41.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:41.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:41.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:41.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:41 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:41.058+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6820 osd.8 since back 2024-05-20T20:22:15.863262+0000 front 2024-05-20T20:22:15.863302+0000 (oldest deadline 2024-05-20T20:22:40.563384+0000) 2024-05-20T20:22:42.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6806 osd.0 since back 2024-05-20T20:21:04.251314+0000 front 2024-05-20T20:21:04.251050+0000 (oldest deadline 2024-05-20T20:21:27.051962+0000) 2024-05-20T20:22:42.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6804 osd.1 since back 2024-05-20T20:21:15.154710+0000 front 2024-05-20T20:21:15.154883+0000 (oldest deadline 2024-05-20T20:21:39.254504+0000) 2024-05-20T20:22:42.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6804 osd.2 since back 2024-05-20T20:21:23.256089+0000 front 2024-05-20T20:21:23.256068+0000 (oldest deadline 2024-05-20T20:21:49.156236+0000) 2024-05-20T20:22:42.375 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.28:6814 osd.3 since back 2024-05-20T20:21:29.156927+0000 front 2024-05-20T20:21:29.156914+0000 (oldest deadline 2024-05-20T20:21:54.456835+0000) 2024-05-20T20:22:42.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6812 osd.4 since back 2024-05-20T20:21:42.058519+0000 front 2024-05-20T20:21:42.058697+0000 (oldest deadline 2024-05-20T20:22:04.358431+0000) 2024-05-20T20:22:42.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6812 osd.5 since back 2024-05-20T20:21:48.960095+0000 front 2024-05-20T20:21:48.960114+0000 (oldest deadline 2024-05-20T20:22:11.860162+0000) 2024-05-20T20:22:42.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.33:6820 osd.7 since back 2024-05-20T20:22:04.762144+0000 front 2024-05-20T20:22:04.762033+0000 (oldest deadline 2024-05-20T20:22:29.462046+0000) 2024-05-20T20:22:42.376 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:42 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11[61916]: 2024-05-20T20:22:42.046+0000 7f9b4965e700 -1 osd.11 102 heartbeat_check: no reply from 172.21.15.154:6820 osd.8 since back 2024-05-20T20:22:15.863262+0000 front 2024-05-20T20:22:15.863302+0000 (oldest deadline 2024-05-20T20:22:40.563384+0000) 2024-05-20T20:22:43.458 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:43 smithi154 podman[75244]: 2024-05-20 20:22:43.187451532 +0000 UTC m=+5.817479664 container died 7390fe069fd846a265338e7a0b6c79f4125be0a67dc63a4431f1158ee8ccab95 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-20T20:22:43.807 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:43 smithi154 podman[75244]: 2024-05-20 20:22:43.540529615 +0000 UTC m=+6.170557738 container cleanup 7390fe069fd846a265338e7a0b6c79f4125be0a67dc63a4431f1158ee8ccab95 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:22:43.808 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:43 smithi154 bash[75244]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11 2024-05-20T20:22:44.125 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:43 smithi154 podman[75262]: 2024-05-20 20:22:43.80829116 +0000 UTC m=+0.620310371 container remove 7390fe069fd846a265338e7a0b6c79f4125be0a67dc63a4431f1158ee8ccab95 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T20:22:44.427 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:44 smithi154 podman[75351]: 2024-05-20 20:22:44.066300841 +0000 UTC m=+0.017525368 image pull 559f9f2978373b56e1ed37e8f2207b076e9eb3d6ccfe36983da65ed32ed7d489 quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3 2024-05-20T20:22:44.427 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:44 smithi154 podman[75351]: 2024-05-20 20:22:44.175922777 +0000 UTC m=+0.127147287 container create e19496215b4cd57ff08adb30b51dc0ab4e50e4f03dd673df912a976c16506a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:44.427 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:44 smithi154 podman[75351]: 2024-05-20 20:22:44.369766001 +0000 UTC m=+0.320990510 container init e19496215b4cd57ff08adb30b51dc0ab4e50e4f03dd673df912a976c16506a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:22:44.427 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:44 smithi154 podman[75351]: 2024-05-20 20:22:44.372718062 +0000 UTC m=+0.323942574 container start e19496215b4cd57ff08adb30b51dc0ab4e50e4f03dd673df912a976c16506a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-lusov-quiesce-4ebd1d5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-05-20T20:22:44.427 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:44 smithi154 podman[75351]: 2024-05-20 20:22:44.427997844 +0000 UTC m=+0.379222368 container attach e19496215b4cd57ff08adb30b51dc0ab4e50e4f03dd673df912a976c16506a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:22:44.875 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:44 smithi154 podman[75351]: 2024-05-20 20:22:44.674229618 +0000 UTC m=+0.625454140 container died e19496215b4cd57ff08adb30b51dc0ab4e50e4f03dd673df912a976c16506a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-20T20:22:45.204 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.11.service' 2024-05-20T20:22:45.228 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:45 smithi154 podman[75377]: 2024-05-20 20:22:45.181359817 +0000 UTC m=+0.500867395 container remove e19496215b4cd57ff08adb30b51dc0ab4e50e4f03dd673df912a976c16506a26 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-osd-11-deactivate, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-20T20:22:45.228 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:45 smithi154 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.11.service: Deactivated successfully. 2024-05-20T20:22:45.228 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:45 smithi154 systemd[1]: Stopped Ceph osd.11 for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:22:45.229 INFO:journalctl@ceph.osd.11.smithi154.stdout:May 20 20:22:45 smithi154 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@osd.11.service: Consumed 4min 13.955s CPU time. 2024-05-20T20:22:45.243 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:45.244 INFO:tasks.cephadm.osd.11:Stopped osd.11 2024-05-20T20:22:45.244 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-05-20T20:22:45.244 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.a 2024-05-20T20:22:45.804 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:45 smithi028 systemd[1]: Stopping Ceph mds.a for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:46.304 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a[73673]: 2024-05-20T20:22:45.897+0000 7f5aa0400700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:46.304 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:45 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a[73673]: 2024-05-20T20:22:45.897+0000 7f5aa0400700 -1 mds.a *** got signal Terminated *** 2024-05-20T20:22:50.265 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:49 smithi028 podman[125813]: 2024-05-20 20:22:49.909806308 +0000 UTC m=+4.156833190 container stop ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-20T20:22:50.265 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:49 smithi028 podman[125813]: 2024-05-20 20:22:49.936416661 +0000 UTC m=+4.183443510 container died ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:22:50.514 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:50 smithi028 podman[125813]: 2024-05-20 20:22:50.475020955 +0000 UTC m=+4.722047854 container cleanup ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-05-20T20:22:50.514 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:50 smithi028 bash[125813]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a 2024-05-20T20:22:50.804 INFO:journalctl@ceph.mds.a.smithi028.stdout:May 20 20:22:50 smithi028 podman[125836]: 2024-05-20 20:22:50.718215078 +0000 UTC m=+0.783925596 container remove ee8975d9c7e7cdc59c827e04e5ac1bacf41734e409eaad06b3fb48feb8435f68 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-a, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T20:22:51.249 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.a.service' 2024-05-20T20:22:51.364 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:51.365 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-05-20T20:22:51.365 INFO:tasks.cephadm.mds.d:Stopping mds.d... 2024-05-20T20:22:51.365 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.d 2024-05-20T20:22:51.805 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:22:51 smithi028 systemd[1]: Stopping Ceph mds.d for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:52.304 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:22:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d[72494]: 2024-05-20T20:22:52.020+0000 7f029cc00700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.d -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:52.304 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:22:52 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d[72494]: 2024-05-20T20:22:52.021+0000 7f029cc00700 -1 mds.d *** got signal Terminated *** 2024-05-20T20:22:56.035 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:22:55 smithi028 podman[125945]: 2024-05-20 20:22:55.745047559 +0000 UTC m=+3.852993657 container died e1a49fd899fd61b85d94a711b7d515f3cebda5d02a124547901d5ad56479c9dd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:22:56.304 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:22:56 smithi028 podman[125945]: 2024-05-20 20:22:56.212332603 +0000 UTC m=+4.320278735 container cleanup e1a49fd899fd61b85d94a711b7d515f3cebda5d02a124547901d5ad56479c9dd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux ) 2024-05-20T20:22:56.304 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:22:56 smithi028 bash[125945]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d 2024-05-20T20:22:56.804 INFO:journalctl@ceph.mds.d.smithi028.stdout:May 20 20:22:56 smithi028 podman[125958]: 2024-05-20 20:22:56.447103922 +0000 UTC m=+0.681664053 container remove e1a49fd899fd61b85d94a711b7d515f3cebda5d02a124547901d5ad56479c9dd (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-d, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD) 2024-05-20T20:22:56.988 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.d.service' 2024-05-20T20:22:57.099 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:22:57.099 INFO:tasks.cephadm.mds.d:Stopped mds.d 2024-05-20T20:22:57.100 INFO:tasks.cephadm.mds.g:Stopping mds.g... 2024-05-20T20:22:57.100 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.g 2024-05-20T20:22:57.554 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:22:57 smithi028 systemd[1]: Stopping Ceph mds.g for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:22:58.054 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:22:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g[73275]: 2024-05-20T20:22:57.756+0000 7ff3ee200700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.g -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:22:58.054 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:22:57 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g[73275]: 2024-05-20T20:22:57.756+0000 7ff3ee200700 -1 mds.g *** got signal Terminated *** 2024-05-20T20:22:59.554 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:22:59 smithi028 podman[126066]: 2024-05-20 20:22:59.294014911 +0000 UTC m=+1.667397977 container died 42c50cb5224609c9b5bce74f9c94e63a837ea4a51398e965e80e8633b101b39d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-20T20:23:00.054 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:22:59 smithi028 podman[126066]: 2024-05-20 20:22:59.81616689 +0000 UTC m=+2.189549969 container cleanup 42c50cb5224609c9b5bce74f9c94e63a837ea4a51398e965e80e8633b101b39d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-05-20T20:23:00.054 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:22:59 smithi028 bash[126066]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g 2024-05-20T20:23:00.553 INFO:journalctl@ceph.mds.g.smithi028.stdout:May 20 20:23:00 smithi028 podman[126080]: 2024-05-20 20:23:00.142647343 +0000 UTC m=+0.827869826 container remove 42c50cb5224609c9b5bce74f9c94e63a837ea4a51398e965e80e8633b101b39d (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-g, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True) 2024-05-20T20:23:00.707 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.g.service' 2024-05-20T20:23:00.820 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:00.820 INFO:tasks.cephadm.mds.g:Stopped mds.g 2024-05-20T20:23:00.820 INFO:tasks.cephadm.mds.j:Stopping mds.j... 2024-05-20T20:23:00.820 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.j 2024-05-20T20:23:01.054 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:23:00 smithi028 systemd[1]: Stopping Ceph mds.j for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:01.804 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:23:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j[72079]: 2024-05-20T20:23:01.376+0000 7f570dc00700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.j -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:01.804 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:23:01 smithi028 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j[72079]: 2024-05-20T20:23:01.376+0000 7f570dc00700 -1 mds.j *** got signal Terminated *** 2024-05-20T20:23:05.652 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:23:05 smithi028 podman[126187]: 2024-05-20 20:23:05.397450096 +0000 UTC m=+4.155189673 container died 287c92d37fbb637efe9dc88f8d03598e74e004422cb196a2fb6bfe46a7374fa8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T20:23:05.907 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:23:05 smithi028 podman[126187]: 2024-05-20 20:23:05.869734848 +0000 UTC m=+4.627474425 container cleanup 287c92d37fbb637efe9dc88f8d03598e74e004422cb196a2fb6bfe46a7374fa8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-20T20:23:05.908 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:23:05 smithi028 bash[126187]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j 2024-05-20T20:23:06.491 INFO:journalctl@ceph.mds.j.smithi028.stdout:May 20 20:23:06 smithi028 podman[126199]: 2024-05-20 20:23:06.071202822 +0000 UTC m=+0.657384171 container remove 287c92d37fbb637efe9dc88f8d03598e74e004422cb196a2fb6bfe46a7374fa8 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-j, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-20T20:23:06.524 DEBUG:teuthology.orchestra.run.smithi028:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.j.service' 2024-05-20T20:23:06.638 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:06.638 INFO:tasks.cephadm.mds.j:Stopped mds.j 2024-05-20T20:23:06.638 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-05-20T20:23:06.639 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.b 2024-05-20T20:23:06.960 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:23:06 smithi033 systemd[1]: Stopping Ceph mds.b for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:07.345 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:23:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: 2024-05-20T20:23:06.960+0000 7f714a513700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:07.345 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:23:06 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b[66623]: 2024-05-20T20:23:06.960+0000 7f714a513700 -1 mds.b *** got signal Terminated *** 2024-05-20T20:23:10.161 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:23:09 smithi033 podman[74331]: 2024-05-20 20:23:09.883322671 +0000 UTC m=+3.036625044 container died ee0c30e87bae3688c5f234237209a238b584b7a3b34c54906fae669efdc97cdb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-05-20T20:23:10.519 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:23:10 smithi033 podman[74331]: 2024-05-20 20:23:10.260376973 +0000 UTC m=+3.413679336 container cleanup ee0c30e87bae3688c5f234237209a238b584b7a3b34c54906fae669efdc97cdb (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-05-20T20:23:10.519 INFO:journalctl@ceph.mds.b.smithi033.stdout:May 20 20:23:10 smithi033 bash[74331]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-b 2024-05-20T20:23:10.703 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.b.service' 2024-05-20T20:23:10.739 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:10.739 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-05-20T20:23:10.739 INFO:tasks.cephadm.mds.e:Stopping mds.e... 2024-05-20T20:23:10.740 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.e 2024-05-20T20:23:11.063 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:23:10 smithi033 systemd[1]: Stopping Ceph mds.e for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:11.376 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:23:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e[65989]: 2024-05-20T20:23:11.063+0000 7f0ef2a22700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.e -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:11.377 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:23:11 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e[65989]: 2024-05-20T20:23:11.063+0000 7f0ef2a22700 -1 mds.e *** got signal Terminated *** 2024-05-20T20:23:14.149 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:23:13 smithi033 podman[74463]: 2024-05-20 20:23:13.886632107 +0000 UTC m=+2.922895929 container died 4896299ff43d00ea5d50e3658f5221c80128dbaba94b48ded6bdc4547d357f8b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:23:14.508 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:23:14 smithi033 podman[74463]: 2024-05-20 20:23:14.232460417 +0000 UTC m=+3.268724226 container cleanup 4896299ff43d00ea5d50e3658f5221c80128dbaba94b48ded6bdc4547d357f8b (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-20T20:23:14.508 INFO:journalctl@ceph.mds.e.smithi033.stdout:May 20 20:23:14 smithi033 bash[74463]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-e 2024-05-20T20:23:14.670 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.e.service' 2024-05-20T20:23:14.704 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:14.704 INFO:tasks.cephadm.mds.e:Stopped mds.e 2024-05-20T20:23:14.705 INFO:tasks.cephadm.mds.h:Stopping mds.h... 2024-05-20T20:23:14.705 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.h 2024-05-20T20:23:15.036 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:23:14 smithi033 systemd[1]: Stopping Ceph mds.h for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:15.376 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:23:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h[66295]: 2024-05-20T20:23:15.036+0000 7f6e71cf1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.h -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:15.377 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:23:15 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h[66295]: 2024-05-20T20:23:15.036+0000 7f6e71cf1700 -1 mds.h *** got signal Terminated *** 2024-05-20T20:23:18.144 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:23:17 smithi033 podman[74585]: 2024-05-20 20:23:17.878368873 +0000 UTC m=+2.951353890 container died ea89059c62bc0516dd41e833137271943068a4f40f657ed09a5a67e26d2ba7df (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h, GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-05-20T20:23:18.512 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:23:18 smithi033 podman[74585]: 2024-05-20 20:23:18.244953234 +0000 UTC m=+3.317938240 container cleanup ea89059c62bc0516dd41e833137271943068a4f40f657ed09a5a67e26d2ba7df (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T20:23:18.512 INFO:journalctl@ceph.mds.h.smithi033.stdout:May 20 20:23:18 smithi033 bash[74585]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-h 2024-05-20T20:23:18.684 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.h.service' 2024-05-20T20:23:18.717 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:18.717 INFO:tasks.cephadm.mds.h:Stopped mds.h 2024-05-20T20:23:18.717 INFO:tasks.cephadm.mds.k:Stopping mds.k... 2024-05-20T20:23:18.717 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.k 2024-05-20T20:23:19.039 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:18 smithi033 systemd[1]: Stopping Ceph mds.k for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:19.377 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-k[65669]: 2024-05-20T20:23:19.039+0000 7f44cc102700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.k -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:19.377 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:19 smithi033 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-k[65669]: 2024-05-20T20:23:19.039+0000 7f44cc102700 -1 mds.k *** got signal Terminated *** 2024-05-20T20:23:22.127 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:21 smithi033 podman[74707]: 2024-05-20 20:23:21.870490426 +0000 UTC m=+2.922316102 container died 3613a2b3e18a38b9b1615c73dbfbbc33f2a3ed355406667f5c8f76f801009e55 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-k, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5) 2024-05-20T20:23:22.390 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:22 smithi033 podman[74707]: 2024-05-20 20:23:22.198206443 +0000 UTC m=+3.250032125 container cleanup 3613a2b3e18a38b9b1615c73dbfbbc33f2a3ed355406667f5c8f76f801009e55 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-k, ceph=True, RELEASE=wip-lusov-quiesce-4ebd1d5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2) 2024-05-20T20:23:22.390 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:22 smithi033 bash[74707]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-k 2024-05-20T20:23:22.637 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.k.service' 2024-05-20T20:23:22.659 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:22 smithi033 podman[74720]: 2024-05-20 20:23:22.390698874 +0000 UTC m=+0.510570935 container remove 3613a2b3e18a38b9b1615c73dbfbbc33f2a3ed355406667f5c8f76f801009e55 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-k, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-05-20T20:23:22.660 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:22 smithi033 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.k.service: Deactivated successfully. 2024-05-20T20:23:22.660 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:22 smithi033 systemd[1]: Stopped Ceph mds.k for 3fbec9b2-16e2-11ef-bc9a-c7b262605968. 2024-05-20T20:23:22.660 INFO:journalctl@ceph.mds.k.smithi033.stdout:May 20 20:23:22 smithi033 systemd[1]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.k.service: Consumed 2.543s CPU time. 2024-05-20T20:23:22.670 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:22.670 INFO:tasks.cephadm.mds.k:Stopped mds.k 2024-05-20T20:23:22.671 INFO:tasks.cephadm.mds.c:Stopping mds.c... 2024-05-20T20:23:22.671 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.c 2024-05-20T20:23:23.054 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:23:22 smithi154 systemd[1]: Stopping Ceph mds.c for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:23.318 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:23:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c[67158]: 2024-05-20T20:23:23.054+0000 7fdb04e0a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:23.318 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:23:23 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c[67158]: 2024-05-20T20:23:23.054+0000 7fdb04e0a700 -1 mds.c *** got signal Terminated *** 2024-05-20T20:23:26.180 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:23:25 smithi154 podman[75456]: 2024-05-20 20:23:25.881904792 +0000 UTC m=+3.016291336 container died f549f9ff21e38a6b3357d4936649858a7bdaa760402f1fd415a09109f5f402e2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c, GIT_CLEAN=True, RELEASE=wip-lusov-quiesce-4ebd1d5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, ceph=True) 2024-05-20T20:23:26.513 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:23:26 smithi154 podman[75456]: 2024-05-20 20:23:26.279864967 +0000 UTC m=+3.414251553 container cleanup f549f9ff21e38a6b3357d4936649858a7bdaa760402f1fd415a09109f5f402e2 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c, GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T20:23:26.513 INFO:journalctl@ceph.mds.c.smithi154.stdout:May 20 20:23:26 smithi154 bash[75456]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-c 2024-05-20T20:23:26.656 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.c.service' 2024-05-20T20:23:26.691 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:26.691 INFO:tasks.cephadm.mds.c:Stopped mds.c 2024-05-20T20:23:26.691 INFO:tasks.cephadm.mds.f:Stopping mds.f... 2024-05-20T20:23:26.692 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.f 2024-05-20T20:23:27.125 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:23:26 smithi154 systemd[1]: Stopping Ceph mds.f for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:27.125 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:23:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f[66852]: 2024-05-20T20:23:26.991+0000 7f51b09e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.f -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:27.125 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:23:26 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f[66852]: 2024-05-20T20:23:26.991+0000 7f51b09e6700 -1 mds.f *** got signal Terminated *** 2024-05-20T20:23:30.134 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:23:29 smithi154 podman[75590]: 2024-05-20 20:23:29.879007655 +0000 UTC m=+2.970105263 container died d4e4b2e108eccf63c70eb9311a57344546a741a8d98ea9fe5350cfbcd47db071 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-20T20:23:30.500 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:23:30 smithi154 podman[75590]: 2024-05-20 20:23:30.233220559 +0000 UTC m=+3.324318167 container cleanup d4e4b2e108eccf63c70eb9311a57344546a741a8d98ea9fe5350cfbcd47db071 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-05-20T20:23:30.500 INFO:journalctl@ceph.mds.f.smithi154.stdout:May 20 20:23:30 smithi154 bash[75590]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-f 2024-05-20T20:23:30.636 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.f.service' 2024-05-20T20:23:30.671 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:30.672 INFO:tasks.cephadm.mds.f:Stopped mds.f 2024-05-20T20:23:30.672 INFO:tasks.cephadm.mds.i:Stopping mds.i... 2024-05-20T20:23:30.672 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.i 2024-05-20T20:23:30.874 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:23:30 smithi154 systemd[1]: Stopping Ceph mds.i for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:31.375 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:23:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i[67476]: 2024-05-20T20:23:30.986+0000 7fe30cf9d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.i -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:31.375 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:23:30 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i[67476]: 2024-05-20T20:23:30.986+0000 7fe30cf9d700 -1 mds.i *** got signal Terminated *** 2024-05-20T20:23:34.125 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:23:33 smithi154 podman[75708]: 2024-05-20 20:23:33.866605573 +0000 UTC m=+2.979066162 container died a1ee9ca81892bccd270aa3ae79969fe8846cc4d3628298b1fba4e93d9289df46 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.vendor=CentOS) 2024-05-20T20:23:34.489 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:23:34 smithi154 podman[75708]: 2024-05-20 20:23:34.282294612 +0000 UTC m=+3.394755193 container cleanup a1ee9ca81892bccd270aa3ae79969fe8846cc4d3628298b1fba4e93d9289df46 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-20T20:23:34.489 INFO:journalctl@ceph.mds.i.smithi154.stdout:May 20 20:23:34 smithi154 bash[75708]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-i 2024-05-20T20:23:34.705 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.i.service' 2024-05-20T20:23:34.736 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:34.736 INFO:tasks.cephadm.mds.i:Stopped mds.i 2024-05-20T20:23:34.736 INFO:tasks.cephadm.mds.l:Stopping mds.l... 2024-05-20T20:23:34.737 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.l 2024-05-20T20:23:35.058 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:23:34 smithi154 systemd[1]: Stopping Ceph mds.l for 3fbec9b2-16e2-11ef-bc9a-c7b262605968... 2024-05-20T20:23:35.375 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:23:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l[66553]: 2024-05-20T20:23:35.058+0000 7f2d5db92700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.l -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T20:23:35.375 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:23:35 smithi154 ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l[66553]: 2024-05-20T20:23:35.058+0000 7f2d5db92700 -1 mds.l *** got signal Terminated *** 2024-05-20T20:23:38.126 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:23:37 smithi154 podman[75829]: 2024-05-20 20:23:37.868714231 +0000 UTC m=+2.911961889 container died 07de68c6d6f34fe50bce2dbb31c9ee819d4679ac545aee7c05050cd9ff474fe1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-20T20:23:38.376 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:23:38 smithi154 podman[75829]: 2024-05-20 20:23:38.208703565 +0000 UTC m=+3.251951217 container cleanup 07de68c6d6f34fe50bce2dbb31c9ee819d4679ac545aee7c05050cd9ff474fe1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l, RELEASE=wip-lusov-quiesce-4ebd1d5, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-05-20T20:23:38.376 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:23:38 smithi154 bash[75829]: ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l 2024-05-20T20:23:38.627 INFO:journalctl@ceph.mds.l.smithi154.stdout:May 20 20:23:38 smithi154 podman[75841]: 2024-05-20 20:23:38.376576089 +0000 UTC m=+0.505917714 container remove 07de68c6d6f34fe50bce2dbb31c9ee819d4679ac545aee7c05050cd9ff474fe1 (image=quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:2eb2a5f16dc82fc2fb58caa939fa48240b0c05b2a3ce0f75aa77a9d2415604e3, name=ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968-mds-l, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-lusov-quiesce-4ebd1d5, org.label-schema.schema-version=1.0) 2024-05-20T20:23:38.631 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3fbec9b2-16e2-11ef-bc9a-c7b262605968@mds.l.service' 2024-05-20T20:23:38.662 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T20:23:38.662 INFO:tasks.cephadm.mds.l:Stopped mds.l 2024-05-20T20:23:38.663 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 --force --keep-logs 2024-05-20T20:23:39.002 INFO:teuthology.orchestra.run.smithi028.stdout:Deleting cluster with fsid: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T20:23:44.190 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 --force --keep-logs 2024-05-20T20:23:44.389 INFO:teuthology.orchestra.run.smithi033.stdout:Deleting cluster with fsid: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T20:23:46.940 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 --force --keep-logs 2024-05-20T20:23:47.131 INFO:teuthology.orchestra.run.smithi154.stdout:Deleting cluster with fsid: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T20:23:49.460 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T20:23:49.552 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T20:23:49.585 DEBUG:teuthology.orchestra.run.smithi154:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T20:23:49.611 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-20T20:23:49.613 DEBUG:teuthology.misc:Transferring archived files from smithi028:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi028/crash 2024-05-20T20:23:49.614 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash -- . 2024-05-20T20:23:49.708 INFO:teuthology.orchestra.run.smithi028.stderr:tar: /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash: Cannot open: No such file or directory 2024-05-20T20:23:49.708 INFO:teuthology.orchestra.run.smithi028.stderr:tar: Error is not recoverable: exiting now 2024-05-20T20:23:49.713 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi033/crash 2024-05-20T20:23:49.714 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash -- . 2024-05-20T20:23:49.753 INFO:teuthology.orchestra.run.smithi033.stderr:tar: /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash: Cannot open: No such file or directory 2024-05-20T20:23:49.753 INFO:teuthology.orchestra.run.smithi033.stderr:tar: Error is not recoverable: exiting now 2024-05-20T20:23:49.755 DEBUG:teuthology.misc:Transferring archived files from smithi154:/var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi154/crash 2024-05-20T20:23:49.756 DEBUG:teuthology.orchestra.run.smithi154:> sudo tar c -f - -C /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash -- . 2024-05-20T20:23:49.824 INFO:teuthology.orchestra.run.smithi154.stderr:tar: /var/lib/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/crash: Cannot open: No such file or directory 2024-05-20T20:23:49.825 INFO:teuthology.orchestra.run.smithi154.stderr:tar: Error is not recoverable: exiting now 2024-05-20T20:23:49.826 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-20T20:23:49.826 DEBUG:teuthology.orchestra.run.smithi028:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v 'filesystem is online with fewer MDS than max_mds' | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'do not have an application enabled' | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v 'slow metadata IO' | egrep -v SLOW_OPS | egrep -v 'slow request' | head -n 1 2024-05-20T20:23:49.997 INFO:tasks.cephadm:Compressing logs... 2024-05-20T20:23:49.997 DEBUG:teuthology.orchestra.run.smithi028:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-20T20:23:50.003 DEBUG:teuthology.orchestra.run.smithi033:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-20T20:23:50.005 DEBUG:teuthology.orchestra.run.smithi154:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-20T20:23:50.029 INFO:teuthology.orchestra.run.smithi154.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log 2024-05-20T20:23:50.029 INFO:teuthology.orchestra.run.smithi154.stderr:: No such file or directory 2024-05-20T20:23:50.029 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.2.log 2024-05-20T20:23:50.029 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.f.log 2024-05-20T20:23:50.029 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log: /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log 2024-05-20T20:23:50.029 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.f.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.5.log 2024-05-20T20:23:50.031 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.c.log 2024-05-20T20:23:50.031 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log: /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.c.log 2024-05-20T20:23:50.031 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.c.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.z.log 2024-05-20T20:23:50.032 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.i.log 2024-05-20T20:23:50.033 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.z.log: 87.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log.gz 2024-05-20T20:23:50.033 INFO:teuthology.orchestra.run.smithi154.stderr: 90.4% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log.gz 2024-05-20T20:23:50.033 INFO:teuthology.orchestra.run.smithi033.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log 2024-05-20T20:23:50.034 INFO:teuthology.orchestra.run.smithi033.stderr:: No such file or directory 2024-05-20T20:23:50.034 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log 2024-05-20T20:23:50.034 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.7.log 2024-05-20T20:23:50.034 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.b.log 2024-05-20T20:23:50.034 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.4.log 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.1.log 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.y.log 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.h.log 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.y.log: 90.4% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log.gz 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.h.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.10.log 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 87.1% --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr: -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log.gz 2024-05-20T20:23:50.037 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.10.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.e.log 2024-05-20T20:23:50.038 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log: 83.6% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log.gz 2024-05-20T20:23:50.038 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log 2024-05-20T20:23:50.042 INFO:teuthology.orchestra.run.smithi154.stderr: 92.4% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.z.log.gz 2024-05-20T20:23:50.043 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log 2024-05-20T20:23:50.043 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.i.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.11.log 2024-05-20T20:23:50.043 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.8.log 2024-05-20T20:23:50.043 INFO:teuthology.orchestra.run.smithi154.stderr: 83.6% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log.gz 2024-05-20T20:23:50.046 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.e.log: 92.3% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.y.log.gz 2024-05-20T20:23:50.051 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.b.log 2024-05-20T20:23:50.056 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.11.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log 2024-05-20T20:23:50.063 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.k.log 2024-05-20T20:23:50.065 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.b.log: 91.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log.gz 2024-05-20T20:23:50.070 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-20T20:23:50.075 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.8.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.l.log 2024-05-20T20:23:50.084 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-20T20:23:50.086 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.k.log: /var/log/ceph/cephadm.log: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-20T20:23:50.093 INFO:teuthology.orchestra.run.smithi028.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log: No such file or directory 2024-05-20T20:23:50.093 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-05-20T20:23:50.095 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log 2024-05-20T20:23:50.096 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.g.log 2024-05-20T20:23:50.097 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.j.log 2024-05-20T20:23:50.098 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.l.log: 91.0%/var/log/ceph/cephadm.log: -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log.gz 2024-05-20T20:23:50.099 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.g.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.3.log 2024-05-20T20:23:50.099 INFO:teuthology.orchestra.run.smithi154.stderr: 92.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-20T20:23:50.100 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.j.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.0.log 2024-05-20T20:23:50.100 INFO:teuthology.orchestra.run.smithi028.stderr: 87.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.log.gz 2024-05-20T20:23:50.101 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.x.log 2024-05-20T20:23:50.102 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.a.log 2024-05-20T20:23:50.103 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.6.log 2024-05-20T20:23:50.103 INFO:teuthology.orchestra.run.smithi028.stderr: 94.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.g.log.gz 2024-05-20T20:23:50.104 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log 2024-05-20T20:23:50.106 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.a.log 2024-05-20T20:23:50.107 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log 2024-05-20T20:23:50.108 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.a.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.d.log 2024-05-20T20:23:50.109 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.9.log 2024-05-20T20:23:50.111 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.d.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-20T20:23:50.115 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.9.log: /var/log/ceph/cephadm.log: 94.3% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.d.log.gz 2024-05-20T20:23:50.116 INFO:teuthology.orchestra.run.smithi154.stderr: 93.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.c.log.gz 2024-05-20T20:23:50.123 INFO:teuthology.orchestra.run.smithi033.stderr: 93.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.e.log.gz 2024-05-20T20:23:50.137 INFO:teuthology.orchestra.run.smithi028.stderr: 90.2% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.audit.log.gz 2024-05-20T20:23:50.142 INFO:teuthology.orchestra.run.smithi033.stderr: 93.3% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.h.log.gz 2024-05-20T20:23:50.153 INFO:teuthology.orchestra.run.smithi028.stderr: 84.0% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph.cephadm.log.gz 2024-05-20T20:23:50.171 INFO:teuthology.orchestra.run.smithi154.stderr: 93.0% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.i.log.gz 2024-05-20T20:23:50.183 INFO:teuthology.orchestra.run.smithi154.stderr: 94.6% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.l.log.gz 2024-05-20T20:23:50.187 INFO:teuthology.orchestra.run.smithi033.stderr: 94.7% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.k.log.gz 2024-05-20T20:23:50.192 INFO:teuthology.orchestra.run.smithi028.stderr: 94.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-20T20:23:50.230 INFO:teuthology.orchestra.run.smithi028.stderr: 90.9% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-volume.log.gz 2024-05-20T20:23:50.276 INFO:teuthology.orchestra.run.smithi154.stderr: 93.9% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.f.log.gz 2024-05-20T20:23:50.336 INFO:teuthology.orchestra.run.smithi028.stderr: 94.5% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.a.log.gz 2024-05-20T20:23:50.493 INFO:teuthology.orchestra.run.smithi028.stderr: 94.4% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.j.log.gz 2024-05-20T20:23:50.537 INFO:teuthology.orchestra.run.smithi033.stderr: 91.5% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.b.log.gz 2024-05-20T20:23:50.698 INFO:teuthology.orchestra.run.smithi154.stderr: 91.6% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.c.log.gz 2024-05-20T20:23:51.304 INFO:teuthology.orchestra.run.smithi028.stderr: 90.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mgr.x.log.gz 2024-05-20T20:23:51.983 INFO:teuthology.orchestra.run.smithi033.stderr: 93.4% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mds.b.log.gz 2024-05-20T20:23:54.018 INFO:teuthology.orchestra.run.smithi028.stderr: 90.1% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-mon.a.log.gz 2024-05-20T20:24:30.550 INFO:teuthology.orchestra.run.smithi154.stderr: 92.9% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.8.log.gz 2024-05-20T20:24:37.583 INFO:teuthology.orchestra.run.smithi154.stderr: 92.9% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.2.log.gz 2024-05-20T20:24:39.336 INFO:teuthology.orchestra.run.smithi033.stderr: 93.0% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.4.log.gz 2024-05-20T20:24:39.449 INFO:teuthology.orchestra.run.smithi154.stderr: 92.9% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.5.log.gz 2024-05-20T20:24:40.548 INFO:teuthology.orchestra.run.smithi033.stderr: 92.7% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.7.log.gz 2024-05-20T20:24:41.577 INFO:teuthology.orchestra.run.smithi154.stderr: 92.8% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.11.log.gz 2024-05-20T20:24:41.579 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-05-20T20:24:41.579 INFO:teuthology.orchestra.run.smithi154.stderr:real 0m51.560s 2024-05-20T20:24:41.579 INFO:teuthology.orchestra.run.smithi154.stderr:user 3m3.688s 2024-05-20T20:24:41.579 INFO:teuthology.orchestra.run.smithi154.stderr:sys 0m6.147s 2024-05-20T20:24:42.002 INFO:teuthology.orchestra.run.smithi033.stderr: 92.8% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.10.log.gz 2024-05-20T20:24:44.383 INFO:teuthology.orchestra.run.smithi033.stderr: 92.8% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.1.log.gz 2024-05-20T20:24:44.385 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-05-20T20:24:44.386 INFO:teuthology.orchestra.run.smithi033.stderr:real 0m54.365s 2024-05-20T20:24:44.386 INFO:teuthology.orchestra.run.smithi033.stderr:user 3m22.060s 2024-05-20T20:24:44.386 INFO:teuthology.orchestra.run.smithi033.stderr:sys 0m6.227s 2024-05-20T20:25:56.539 INFO:teuthology.orchestra.run.smithi028.stderr: 92.8% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.9.log.gz 2024-05-20T20:26:16.287 INFO:teuthology.orchestra.run.smithi028.stderr: 92.8% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.6.log.gz 2024-05-20T20:26:26.142 INFO:teuthology.orchestra.run.smithi028.stderr: 92.7% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.0.log.gz 2024-05-20T20:26:27.737 INFO:teuthology.orchestra.run.smithi028.stderr: 92.7% -- replaced with /var/log/ceph/3fbec9b2-16e2-11ef-bc9a-c7b262605968/ceph-osd.3.log.gz 2024-05-20T20:26:27.741 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-05-20T20:26:27.742 INFO:teuthology.orchestra.run.smithi028.stderr:real 2m37.695s 2024-05-20T20:26:27.742 INFO:teuthology.orchestra.run.smithi028.stderr:user 5m4.779s 2024-05-20T20:26:27.742 INFO:teuthology.orchestra.run.smithi028.stderr:sys 0m34.321s 2024-05-20T20:26:27.743 INFO:tasks.cephadm:Archiving logs... 2024-05-20T20:26:27.744 DEBUG:teuthology.misc:Transferring archived files from smithi028:/var/log/ceph to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi028/log 2024-05-20T20:26:27.745 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-20T20:26:57.952 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/log/ceph to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi033/log 2024-05-20T20:26:57.957 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-20T20:27:29.075 DEBUG:teuthology.misc:Transferring archived files from smithi154:/var/log/ceph to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi154/log 2024-05-20T20:27:29.076 DEBUG:teuthology.orchestra.run.smithi154:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-20T20:27:58.645 INFO:tasks.cephadm:Removing cluster... 2024-05-20T20:27:58.646 DEBUG:teuthology.orchestra.run.smithi028:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 --force 2024-05-20T20:27:58.985 INFO:teuthology.orchestra.run.smithi028.stdout:Deleting cluster with fsid: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T20:28:00.219 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 --force 2024-05-20T20:28:00.419 INFO:teuthology.orchestra.run.smithi033.stdout:Deleting cluster with fsid: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T20:28:00.968 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3fbec9b2-16e2-11ef-bc9a-c7b262605968 --force 2024-05-20T20:28:01.159 INFO:teuthology.orchestra.run.smithi154.stdout:Deleting cluster with fsid: 3fbec9b2-16e2-11ef-bc9a-c7b262605968 2024-05-20T20:28:01.669 INFO:tasks.cephadm:Removing cephadm ... 2024-05-20T20:28:01.670 DEBUG:teuthology.orchestra.run.smithi028:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-20T20:28:01.731 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-20T20:28:01.747 DEBUG:teuthology.orchestra.run.smithi154:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-20T20:28:01.762 INFO:tasks.cephadm:Teardown complete 2024-05-20T20:28:01.763 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-05-20T20:28:01.775 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-05-20T20:28:01.776 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-05-20T20:28:01.784 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-05-20T20:28:01.789 DEBUG:teuthology.orchestra.run.smithi154:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-05-20T20:28:01.942 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-20T20:28:01.953 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-20T20:28:01.953 DEBUG:teuthology.orchestra.run.smithi028:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T20:28:01.961 DEBUG:teuthology.orchestra.run.smithi033:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T20:28:01.964 DEBUG:teuthology.orchestra.run.smithi154:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T20:28:01.976 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-05-20T20:28:01.977 INFO:teuthology.orchestra.run.smithi154.stderr:bash: line 1: ntpq: command not found 2024-05-20T20:28:01.983 INFO:teuthology.orchestra.run.smithi154.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T20:28:01.983 INFO:teuthology.orchestra.run.smithi154.stdout:=============================================================================== 2024-05-20T20:28:01.983 INFO:teuthology.orchestra.run.smithi154.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 +927us[ +974us] +/- 67ms 2024-05-20T20:28:01.983 INFO:teuthology.orchestra.run.smithi154.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 49 -671us[ -671us] +/- 65ms 2024-05-20T20:28:01.983 INFO:teuthology.orchestra.run.smithi154.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 49 -185us[ -185us] +/- 63ms 2024-05-20T20:28:01.984 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T20:28:01.984 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T20:28:01.984 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-05-20T20:28:01.984 INFO:teuthology.orchestra.run.smithi033.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 +934us[ +946us] +/- 67ms 2024-05-20T20:28:01.984 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 47 -663us[ -663us] +/- 65ms 2024-05-20T20:28:01.984 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 50 -164us[ -164us] +/- 63ms 2024-05-20T20:28:01.984 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T20:28:02.010 INFO:teuthology.orchestra.run.smithi028.stderr:bash: line 1: ntpq: command not found 2024-05-20T20:28:02.021 INFO:teuthology.orchestra.run.smithi028.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T20:28:02.022 INFO:teuthology.orchestra.run.smithi028.stdout:=============================================================================== 2024-05-20T20:28:02.022 INFO:teuthology.orchestra.run.smithi028.stdout:^* hv01.front.sepia.ceph.com 3 6 377 48 +942us[ +943us] +/- 67ms 2024-05-20T20:28:02.022 INFO:teuthology.orchestra.run.smithi028.stdout:^+ hv02.front.sepia.ceph.com 3 8 377 248 -682us[ -692us] +/- 62ms 2024-05-20T20:28:02.022 INFO:teuthology.orchestra.run.smithi028.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 116 -126us[ -133us] +/- 62ms 2024-05-20T20:28:02.022 INFO:teuthology.orchestra.run.smithi028.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T20:28:02.023 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-20T20:28:02.035 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-20T20:28:02.035 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-20T20:28:02.046 DEBUG:teuthology.orchestra.run.smithi028:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-20T20:28:02.399 DEBUG:teuthology.orchestra.run.smithi033:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-20T20:28:02.469 DEBUG:teuthology.orchestra.run.smithi154:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-20T20:28:02.538 DEBUG:teuthology.orchestra.run.smithi028:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T20:28:02.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T20:28:02.638 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T20:28:02.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T20:28:02.667 DEBUG:teuthology.orchestra.run.smithi154:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T20:28:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T20:28:02.695 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-20T20:28:02.707 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-20T20:28:02.719 INFO:teuthology.task.internal:Duration was 2765.071886 seconds 2024-05-20T20:28:02.719 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-20T20:28:02.730 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-20T20:28:02.730 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-20T20:28:02.738 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-20T20:28:02.741 DEBUG:teuthology.orchestra.run.smithi154:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-20T20:28:02.778 INFO:teuthology.orchestra.run.smithi154.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T20:28:02.779 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T20:28:02.921 INFO:teuthology.orchestra.run.smithi028.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T20:28:03.431 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-20T20:28:03.431 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi028.front.sepia.ceph.com 2024-05-20T20:28:03.431 DEBUG:teuthology.orchestra.run.smithi028:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-20T20:28:03.498 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com 2024-05-20T20:28:03.498 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-20T20:28:03.527 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi154.front.sepia.ceph.com 2024-05-20T20:28:03.527 DEBUG:teuthology.orchestra.run.smithi154:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-20T20:28:03.547 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-20T20:28:03.548 DEBUG:teuthology.orchestra.run.smithi028:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-20T20:28:03.556 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-20T20:28:03.570 DEBUG:teuthology.orchestra.run.smithi154:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-20T20:28:03.646 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-20T20:28:03.646 DEBUG:teuthology.orchestra.run.smithi028:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-20T20:28:03.651 DEBUG:teuthology.orchestra.run.smithi033:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-20T20:28:03.653 DEBUG:teuthology.orchestra.run.smithi154:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-20T20:28:05.099 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-20T20:28:05.111 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-20T20:28:05.111 DEBUG:teuthology.orchestra.run.smithi028:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-20T20:28:05.204 DEBUG:teuthology.orchestra.run.smithi033:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-20T20:28:05.233 DEBUG:teuthology.orchestra.run.smithi154:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-20T20:28:05.262 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-20T20:28:05.275 DEBUG:teuthology.orchestra.run.smithi028:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-20T20:28:05.283 DEBUG:teuthology.orchestra.run.smithi033:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-20T20:28:05.286 DEBUG:teuthology.orchestra.run.smithi154:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-20T20:28:05.311 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = core 2024-05-20T20:28:05.328 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern = core 2024-05-20T20:28:05.390 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern = core 2024-05-20T20:28:05.436 DEBUG:teuthology.orchestra.run.smithi028:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-20T20:28:05.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T20:28:05.489 DEBUG:teuthology.orchestra.run.smithi033:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-20T20:28:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T20:28:05.507 DEBUG:teuthology.orchestra.run.smithi154:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-20T20:28:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T20:28:05.523 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-20T20:28:05.537 INFO:teuthology.task.internal:Transferring archived files... 2024-05-20T20:28:05.538 DEBUG:teuthology.misc:Transferring archived files from smithi028:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi028 2024-05-20T20:28:05.538 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-20T20:28:05.669 DEBUG:teuthology.misc:Transferring archived files from smithi033:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi033 2024-05-20T20:28:05.670 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-20T20:28:05.710 DEBUG:teuthology.misc:Transferring archived files from smithi154:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/leonidus-2024-05-20_18:41:05-fs-wip-lusov-quiesce-distro-default-smithi/7715929/remote/smithi154 2024-05-20T20:28:05.710 DEBUG:teuthology.orchestra.run.smithi154:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-20T20:28:05.749 INFO:teuthology.task.internal:Removing archive directory... 2024-05-20T20:28:05.749 DEBUG:teuthology.orchestra.run.smithi028:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-20T20:28:05.757 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-20T20:28:05.760 DEBUG:teuthology.orchestra.run.smithi154:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-20T20:28:05.811 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-20T20:28:05.825 INFO:teuthology.task.internal:Not uploading archives. 2024-05-20T20:28:05.825 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-20T20:28:05.837 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-20T20:28:05.837 DEBUG:teuthology.orchestra.run.smithi028:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-20T20:28:05.843 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-20T20:28:05.845 DEBUG:teuthology.orchestra.run.smithi154:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-20T20:28:05.861 INFO:teuthology.orchestra.run.smithi033.stdout: 262161 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 20 20:28 /home/ubuntu/cephtest 2024-05-20T20:28:05.861 INFO:teuthology.orchestra.run.smithi154.stdout: 262161 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 20 20:28 /home/ubuntu/cephtest 2024-05-20T20:28:05.896 INFO:teuthology.orchestra.run.smithi028.stdout: 393718 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 20 20:28 /home/ubuntu/cephtest 2024-05-20T20:28:05.899 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-20T20:28:05.912 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-20T20:28:05.940 INFO:teuthology.run:Summary data: client.0-kernel-sha1: 0fe9556839d28e65b29a64cc05a26a6884e8775d description: fs/workload/{0-centos_9.stream begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mgr mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/secure wsync/no} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts pg_health session_timeout} ranks/multi/{balancer/random export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/yes 5-quiesce/with-quiesce 6-workunit/suites/iogen}} duration: 2765.071885585785 flavor: default owner: scheduled_leonidus@teuthology success: true 2024-05-20T20:28:05.941 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-20T20:28:06.042 INFO:teuthology.run:pass